diff --git a/src/applications/differential/engine/DifferentialDiffExtractionEngine.php b/src/applications/differential/engine/DifferentialDiffExtractionEngine.php index 7b94b1958b..08e0a344e0 100644 --- a/src/applications/differential/engine/DifferentialDiffExtractionEngine.php +++ b/src/applications/differential/engine/DifferentialDiffExtractionEngine.php @@ -1,420 +1,414 @@ viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setAuthorPHID($author_phid) { $this->authorPHID = $author_phid; return $this; } public function getAuthorPHID() { return $this->authorPHID; } public function newDiffFromCommit(PhabricatorRepositoryCommit $commit) { $viewer = $this->getViewer(); // If we already have an unattached diff for this commit, just reuse it. // This stops us from repeatedly generating diffs if something goes wrong // later in the process. See T10968 for context. $existing_diffs = id(new DifferentialDiffQuery()) ->setViewer($viewer) ->withCommitPHIDs(array($commit->getPHID())) ->withHasRevision(false) ->needChangesets(true) ->execute(); if ($existing_diffs) { return head($existing_diffs); } $repository = $commit->getRepository(); $identifier = $commit->getCommitIdentifier(); $monogram = $commit->getMonogram(); $drequest = DiffusionRequest::newFromDictionary( array( 'user' => $viewer, 'repository' => $repository, )); $diff_info = DiffusionQuery::callConduitWithDiffusionRequest( $viewer, $drequest, 'diffusion.rawdiffquery', array( 'commit' => $identifier, )); $file_phid = $diff_info['filePHID']; $diff_file = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withPHIDs(array($file_phid)) ->executeOne(); if (!$diff_file) { throw new Exception( pht( 'Failed to load file ("%s") returned by "%s".', $file_phid, 'diffusion.rawdiffquery')); } $raw_diff = $diff_file->loadFileData(); // TODO: Support adds, deletes and moves under SVN. if (strlen($raw_diff)) { $changes = id(new ArcanistDiffParser())->parseDiff($raw_diff); } else { // This is an empty diff, maybe made with `git commit --allow-empty`. // NOTE: These diffs have the same tree hash as their ancestors, so // they may attach to revisions in an unexpected way. Just let this // happen for now, although it might make sense to special case it // eventually. $changes = array(); } $diff = DifferentialDiff::newFromRawChanges($viewer, $changes) ->setRepositoryPHID($repository->getPHID()) ->setCommitPHID($commit->getPHID()) ->setCreationMethod('commit') ->setSourceControlSystem($repository->getVersionControlSystem()) ->setLintStatus(DifferentialLintStatus::LINT_AUTO_SKIP) ->setUnitStatus(DifferentialUnitStatus::UNIT_AUTO_SKIP) ->setDateCreated($commit->getEpoch()) ->setDescription($monogram); $author_phid = $this->getAuthorPHID(); if ($author_phid !== null) { $diff->setAuthorPHID($author_phid); } $parents = DiffusionQuery::callConduitWithDiffusionRequest( $viewer, $drequest, 'diffusion.commitparentsquery', array( 'commit' => $identifier, )); if ($parents) { $diff->setSourceControlBaseRevision(head($parents)); } // TODO: Attach binary files. return $diff->save(); } public function isDiffChangedBeforeCommit( PhabricatorRepositoryCommit $commit, DifferentialDiff $old, DifferentialDiff $new) { $viewer = $this->getViewer(); $repository = $commit->getRepository(); $identifier = $commit->getCommitIdentifier(); $vs_changesets = array(); foreach ($old->getChangesets() as $changeset) { $path = $changeset->getAbsoluteRepositoryPath($repository, $old); $path = ltrim($path, '/'); $vs_changesets[$path] = $changeset; } $changesets = array(); foreach ($new->getChangesets() as $changeset) { $path = $changeset->getAbsoluteRepositoryPath($repository, $new); $path = ltrim($path, '/'); $changesets[$path] = $changeset; } if (array_fill_keys(array_keys($changesets), true) != array_fill_keys(array_keys($vs_changesets), true)) { return true; } $file_phids = array(); foreach ($vs_changesets as $changeset) { $metadata = $changeset->getMetadata(); $file_phid = idx($metadata, 'new:binary-phid'); if ($file_phid) { $file_phids[$file_phid] = $file_phid; } } $files = array(); if ($file_phids) { $files = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs($file_phids) ->execute(); $files = mpull($files, null, 'getPHID'); } foreach ($changesets as $path => $changeset) { $vs_changeset = $vs_changesets[$path]; $file_phid = idx($vs_changeset->getMetadata(), 'new:binary-phid'); if ($file_phid) { if (!isset($files[$file_phid])) { return true; } $drequest = DiffusionRequest::newFromDictionary( array( 'user' => $viewer, 'repository' => $repository, )); try { $response = DiffusionQuery::callConduitWithDiffusionRequest( $viewer, $drequest, 'diffusion.filecontentquery', array( 'commit' => $identifier, 'path' => $path, )); } catch (Exception $ex) { // TODO: See PHI1044. This call may fail if the diff deleted the // file. If the call fails, just detect a change for now. This should // generally be made cleaner in the future. return true; } $new_file_phid = $response['filePHID']; if (!$new_file_phid) { return true; } $new_file = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withPHIDs(array($new_file_phid)) ->executeOne(); if (!$new_file) { return true; } if ($files[$file_phid]->loadFileData() != $new_file->loadFileData()) { return true; } } else { $context = implode("\n", $changeset->makeChangesWithContext()); $vs_context = implode("\n", $vs_changeset->makeChangesWithContext()); // We couldn't just compare $context and $vs_context because following // diffs will be considered different: // // -(empty line) // -echo 'test'; // (empty line) // // (empty line) // -echo "test"; // -(empty line) $hunk = id(new DifferentialHunk())->setChanges($context); $vs_hunk = id(new DifferentialHunk())->setChanges($vs_context); if ($hunk->makeOldFile() != $vs_hunk->makeOldFile() || $hunk->makeNewFile() != $vs_hunk->makeNewFile()) { return true; } } } return false; } public function updateRevisionWithCommit( DifferentialRevision $revision, PhabricatorRepositoryCommit $commit, array $more_xactions, PhabricatorContentSource $content_source) { $viewer = $this->getViewer(); - $result_data = array(); - $new_diff = $this->newDiffFromCommit($commit); $old_diff = $revision->getActiveDiff(); $changed_uri = null; if ($old_diff) { $old_diff = id(new DifferentialDiffQuery()) ->setViewer($viewer) ->withIDs(array($old_diff->getID())) ->needChangesets(true) ->executeOne(); if ($old_diff) { $has_changed = $this->isDiffChangedBeforeCommit( $commit, $old_diff, $new_diff); if ($has_changed) { - $result_data['vsDiff'] = $old_diff->getID(); - $revision_monogram = $revision->getMonogram(); $old_id = $old_diff->getID(); $new_id = $new_diff->getID(); $changed_uri = "/{$revision_monogram}?vs={$old_id}&id={$new_id}#toc"; $changed_uri = PhabricatorEnv::getProductionURI($changed_uri); } } } $xactions = array(); // If the revision isn't closed or "Accepted", write a warning into the // transaction log. This makes it more clear when users bend the rules. if (!$revision->isClosed() && !$revision->isAccepted()) { $wrong_type = DifferentialRevisionWrongStateTransaction::TRANSACTIONTYPE; $xactions[] = id(new DifferentialTransaction()) ->setTransactionType($wrong_type) ->setNewValue($revision->getModernRevisionStatus()); } $concerning_builds = $this->loadConcerningBuilds($revision); if ($concerning_builds) { $build_list = array(); foreach ($concerning_builds as $build) { $build_list[] = array( 'phid' => $build->getPHID(), 'status' => $build->getBuildStatus(), ); } $wrong_builds = DifferentialRevisionWrongBuildsTransaction::TRANSACTIONTYPE; $xactions[] = id(new DifferentialTransaction()) ->setTransactionType($wrong_builds) ->setNewValue($build_list); } $type_update = DifferentialRevisionUpdateTransaction::TRANSACTIONTYPE; $xactions[] = id(new DifferentialTransaction()) ->setTransactionType($type_update) ->setIgnoreOnNoEffect(true) ->setNewValue($new_diff->getPHID()) ->setMetadataValue('isCommitUpdate', true) ->setMetadataValue('commitPHIDs', array($commit->getPHID())); foreach ($more_xactions as $more_xaction) { $xactions[] = $more_xaction; } $editor = id(new DifferentialTransactionEditor()) ->setActor($viewer) ->setContinueOnMissingFields(true) ->setContentSource($content_source) ->setChangedPriorToCommitURI($changed_uri) ->setIsCloseByCommit(true); $author_phid = $this->getAuthorPHID(); if ($author_phid !== null) { $editor->setActingAsPHID($author_phid); } try { $editor->applyTransactions($revision, $xactions); } catch (PhabricatorApplicationTransactionNoEffectException $ex) { // NOTE: We've marked transactions other than the CLOSE transaction // as ignored when they don't have an effect, so this means that we // lost a race to close the revision. That's perfectly fine, we can // just continue normally. } - - return $result_data; } private function loadConcerningBuilds(DifferentialRevision $revision) { $viewer = $this->getViewer(); $diff = $revision->getActiveDiff(); $buildables = id(new HarbormasterBuildableQuery()) ->setViewer($viewer) ->withBuildablePHIDs(array($diff->getPHID())) ->needBuilds(true) ->withManualBuildables(false) ->execute(); if (!$buildables) { return array(); } $land_key = HarbormasterBuildPlanBehavior::BEHAVIOR_LANDWARNING; $behavior = HarbormasterBuildPlanBehavior::getBehavior($land_key); $key_never = HarbormasterBuildPlanBehavior::LANDWARNING_NEVER; $key_building = HarbormasterBuildPlanBehavior::LANDWARNING_IF_BUILDING; $key_complete = HarbormasterBuildPlanBehavior::LANDWARNING_IF_COMPLETE; $concerning_builds = array(); foreach ($buildables as $buildable) { $builds = $buildable->getBuilds(); foreach ($builds as $build) { $plan = $build->getBuildPlan(); $option = $behavior->getPlanOption($plan); $behavior_value = $option->getKey(); $if_never = ($behavior_value === $key_never); if ($if_never) { continue; } $if_building = ($behavior_value === $key_building); if ($if_building && $build->isComplete()) { continue; } $if_complete = ($behavior_value === $key_complete); if ($if_complete) { if (!$build->isComplete()) { continue; } // TODO: If you "arc land" and a build with "Warn: If Complete" // is still running, you may not see a warning, and push the revision // in good faith. The build may then complete before we get here, so // we now see a completed, failed build. // For now, just err on the side of caution and assume these builds // were in a good state when we prompted the user, even if they're in // a bad state now. // We could refine this with a rule like "if the build finished // within a couple of minutes before the push happened, assume it was // in good faith", but we don't currently have an especially // convenient way to check when the build finished or when the commit // was pushed or discovered, and this would create some issues in // cases where the repository is observed and the fetch pipeline // stalls for a while. continue; } if ($build->isPassed()) { continue; } $concerning_builds[] = $build; } } return $concerning_builds; } } diff --git a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php index 8a1b5e5dd1..b0583725a9 100644 --- a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php +++ b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php @@ -1,452 +1,448 @@ shouldSkipImportStep()) { $viewer = PhabricatorUser::getOmnipotentUser(); $refs_raw = DiffusionQuery::callConduitWithDiffusionRequest( $viewer, DiffusionRequest::newFromDictionary( array( 'repository' => $repository, 'user' => $viewer, )), 'diffusion.querycommits', array( 'repositoryPHID' => $repository->getPHID(), 'phids' => array($commit->getPHID()), 'bypassCache' => true, 'needMessages' => true, )); if (empty($refs_raw['data'])) { throw new Exception( pht( 'Unable to retrieve details for commit "%s"!', $commit->getPHID())); } $ref = DiffusionCommitRef::newFromConduitResult(head($refs_raw['data'])); $this->updateCommitData($ref); } if ($this->shouldQueueFollowupTasks()) { $this->queueTask( $this->getFollowupTaskClass(), array( 'commitID' => $commit->getID(), ), array( // We queue followup tasks at default priority so that the queue // finishes work it has started before starting more work. If // followups are queued at the same priority level, we do all // message parses first, then all change parses, etc. This makes // progress uneven. See T11677 for discussion. 'priority' => PhabricatorWorker::PRIORITY_DEFAULT, )); } } final protected function updateCommitData(DiffusionCommitRef $ref) { $commit = $this->commit; $author = $ref->getAuthor(); $message = $ref->getMessage(); $committer = $ref->getCommitter(); $hashes = $ref->getHashes(); $author_identity = id(new PhabricatorRepositoryIdentityQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withIdentityNames(array($author)) ->executeOne(); if (!$author_identity) { $author_identity = id(new PhabricatorRepositoryIdentity()) ->setAuthorPHID($commit->getPHID()) ->setIdentityName($author) ->setAutomaticGuessedUserPHID( $this->resolveUserPHID($commit, $author)) ->save(); } $committer_identity = null; if ($committer) { $committer_identity = id(new PhabricatorRepositoryIdentityQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withIdentityNames(array($committer)) ->executeOne(); if (!$committer_identity) { $committer_identity = id(new PhabricatorRepositoryIdentity()) ->setAuthorPHID($commit->getPHID()) ->setIdentityName($committer) ->setAutomaticGuessedUserPHID( $this->resolveUserPHID($commit, $committer)) ->save(); } } $data = id(new PhabricatorRepositoryCommitData())->loadOneWhere( 'commitID = %d', $commit->getID()); if (!$data) { $data = new PhabricatorRepositoryCommitData(); } $data->setCommitID($commit->getID()); $data->setAuthorName(id(new PhutilUTF8StringTruncator()) ->setMaximumBytes(255) ->truncateString((string)$author)); $data->setCommitDetail('authorEpoch', $ref->getAuthorEpoch()); $data->setCommitDetail('authorName', $ref->getAuthorName()); $data->setCommitDetail('authorEmail', $ref->getAuthorEmail()); $data->setCommitDetail( 'authorIdentityPHID', $author_identity->getPHID()); $data->setCommitDetail( 'authorPHID', $this->resolveUserPHID($commit, $author)); $data->setCommitMessage($message); if (strlen($committer)) { $data->setCommitDetail('committer', $committer); $data->setCommitDetail('committerName', $ref->getCommitterName()); $data->setCommitDetail('committerEmail', $ref->getCommitterEmail()); $data->setCommitDetail( 'committerPHID', $this->resolveUserPHID($commit, $committer)); $data->setCommitDetail( 'committerIdentityPHID', $committer_identity->getPHID()); $commit->setCommitterIdentityPHID($committer_identity->getPHID()); } $repository = $this->repository; $author_phid = $data->getCommitDetail('authorPHID'); $committer_phid = $data->getCommitDetail('committerPHID'); $user = new PhabricatorUser(); if ($author_phid) { $user = $user->loadOneWhere( 'phid = %s', $author_phid); } $differential_app = 'PhabricatorDifferentialApplication'; $revision_id = null; $low_level_query = null; if (PhabricatorApplication::isClassInstalled($differential_app)) { $low_level_query = id(new DiffusionLowLevelCommitFieldsQuery()) ->setRepository($repository) ->withCommitRef($ref); $field_values = $low_level_query->execute(); $revision_id = idx($field_values, 'revisionID'); if (!empty($field_values['reviewedByPHIDs'])) { $data->setCommitDetail( 'reviewerPHID', reset($field_values['reviewedByPHIDs'])); } $data->setCommitDetail('differential.revisionID', $revision_id); } if ($author_phid != $commit->getAuthorPHID()) { $commit->setAuthorPHID($author_phid); } $commit->setAuthorIdentityPHID($author_identity->getPHID()); $commit->setSummary($data->getSummary()); $commit->save(); // Figure out if we're going to try to "autoclose" related objects (e.g., // close linked tasks and related revisions) and, if not, record why we // aren't. Autoclose can be disabled for various reasons at the repository // or commit levels. $force_autoclose = idx($this->getTaskData(), 'forceAutoclose', false); if ($force_autoclose) { $autoclose_reason = PhabricatorRepository::BECAUSE_AUTOCLOSE_FORCED; } else { $autoclose_reason = $repository->shouldSkipAutocloseCommit($commit); } $data->setCommitDetail('autocloseReason', $autoclose_reason); $should_autoclose = $force_autoclose || $repository->shouldAutocloseCommit($commit); // When updating related objects, we'll act under an omnipotent user to // ensure we can see them, but take actions as either the committer or // author (if we recognize their accounts) or the Diffusion application // (if we do not). $actor = PhabricatorUser::getOmnipotentUser(); $acting_as_phid = nonempty( $committer_phid, $author_phid, id(new PhabricatorDiffusionApplication())->getPHID()); $acting_user = $this->loadActingUser($actor, $acting_as_phid); $conn_w = id(new DifferentialRevision())->establishConnection('w'); // NOTE: The `differential_commit` table has a unique ID on `commitPHID`, // preventing more than one revision from being associated with a commit. // Generally this is good and desirable, but with the advent of hash // tracking we may end up in a situation where we match several different // revisions. We just kind of ignore this and pick one, we might want to // revisit this and do something differently. (If we match several revisions // someone probably did something very silly, though.) $revision = null; if ($revision_id) { $revision_query = id(new DifferentialRevisionQuery()) ->withIDs(array($revision_id)) ->setViewer($actor) ->needReviewers(true) ->needActiveDiffs(true); $revision = $revision_query->executeOne(); if ($revision) { $commit_drev = DiffusionCommitHasRevisionEdgeType::EDGECONST; id(new PhabricatorEdgeEditor()) ->addEdge($commit->getPHID(), $commit_drev, $revision->getPHID()) ->save(); queryfx( $conn_w, 'INSERT IGNORE INTO %T (revisionID, commitPHID) VALUES (%d, %s)', DifferentialRevision::TABLE_COMMIT, $revision->getID(), $commit->getPHID()); $should_close = !$revision->isPublished() && $should_autoclose; if ($should_close) { $type_close = DifferentialRevisionCloseTransaction::TRANSACTIONTYPE; $commit_close_xaction = id(new DifferentialTransaction()) ->setTransactionType($type_close) ->setNewValue(true) ->setMetadataValue('isCommitClose', true); $commit_close_xaction->setMetadataValue( 'commitPHID', $commit->getPHID()); $commit_close_xaction->setMetadataValue( 'committerPHID', $committer_phid); $commit_close_xaction->setMetadataValue( 'committerName', $data->getCommitDetail('committer')); $commit_close_xaction->setMetadataValue( 'authorPHID', $author_phid); $commit_close_xaction->setMetadataValue( 'authorName', $data->getAuthorName()); if ($low_level_query) { $commit_close_xaction->setMetadataValue( 'revisionMatchData', $low_level_query->getRevisionMatchData()); $data->setCommitDetail( 'revisionMatchData', $low_level_query->getRevisionMatchData()); } $extraction_engine = id(new DifferentialDiffExtractionEngine()) ->setViewer($actor) ->setAuthorPHID($acting_as_phid); $content_source = $this->newContentSource(); - $update_data = $extraction_engine->updateRevisionWithCommit( + $extraction_engine->updateRevisionWithCommit( $revision, $commit, array( $commit_close_xaction, ), $content_source); - - foreach ($update_data as $key => $value) { - $data->setCommitDetail($key, $value); - } } } } if ($should_autoclose) { $this->closeTasks( $actor, $acting_as_phid, $repository, $commit, $message, $acting_user); } $data->save(); $commit->writeImportStatusFlag( PhabricatorRepositoryCommit::IMPORTED_MESSAGE); } private function resolveUserPHID( PhabricatorRepositoryCommit $commit, $user_name) { return id(new DiffusionResolveUserQuery()) ->withCommit($commit) ->withName($user_name) ->execute(); } private function closeTasks( PhabricatorUser $actor, $acting_as, PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, $message, PhabricatorUser $acting_user = null) { // If we we were able to identify an author for the commit, we try to act // as that user when loading tasks marked with "Fixes Txxx". This prevents // mistakes where a user accidentally writes the wrong task IDs and affects // tasks they can't see (and thus can't undo the status changes for). // This is just a guard rail, not a security measure. An attacker can still // forge another user's identity trivially by forging author or committer // emails. We also let commits with unrecognized authors act on any task to // make behavior less confusing for new installs. if (!$acting_user) { $acting_user = $actor; } $maniphest = 'PhabricatorManiphestApplication'; if (!PhabricatorApplication::isClassInstalled($maniphest)) { return; } $prefixes = ManiphestTaskStatus::getStatusPrefixMap(); $suffixes = ManiphestTaskStatus::getStatusSuffixMap(); $matches = id(new ManiphestCustomFieldStatusParser()) ->parseCorpus($message); $task_statuses = array(); foreach ($matches as $match) { $prefix = phutil_utf8_strtolower($match['prefix']); $suffix = phutil_utf8_strtolower($match['suffix']); $status = idx($suffixes, $suffix); if (!$status) { $status = idx($prefixes, $prefix); } foreach ($match['monograms'] as $task_monogram) { $task_id = (int)trim($task_monogram, 'tT'); $task_statuses[$task_id] = $status; } } if (!$task_statuses) { return; } $tasks = id(new ManiphestTaskQuery()) ->setViewer($acting_user) ->withIDs(array_keys($task_statuses)) ->needProjectPHIDs(true) ->requireCapabilities( array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, )) ->execute(); foreach ($tasks as $task_id => $task) { $xactions = array(); $edge_type = ManiphestTaskHasCommitEdgeType::EDGECONST; $edge_xaction = id(new ManiphestTransaction()) ->setTransactionType(PhabricatorTransactions::TYPE_EDGE) ->setMetadataValue('edge:type', $edge_type) ->setNewValue( array( '+' => array( $commit->getPHID() => $commit->getPHID(), ), )); $status = $task_statuses[$task_id]; if ($status) { if ($task->getStatus() != $status) { $xactions[] = id(new ManiphestTransaction()) ->setTransactionType( ManiphestTaskStatusTransaction::TRANSACTIONTYPE) ->setMetadataValue('commitPHID', $commit->getPHID()) ->setNewValue($status); $edge_xaction->setMetadataValue('commitPHID', $commit->getPHID()); } } $xactions[] = $edge_xaction; $content_source = $this->newContentSource(); $editor = id(new ManiphestTransactionEditor()) ->setActor($actor) ->setActingAsPHID($acting_as) ->setContinueOnNoEffect(true) ->setContinueOnMissingFields(true) ->setUnmentionablePHIDMap( array($commit->getPHID() => $commit->getPHID())) ->setContentSource($content_source); $editor->applyTransactions($task, $xactions); } } private function loadActingUser(PhabricatorUser $viewer, $user_phid) { if (!$user_phid) { return null; } $user_type = PhabricatorPeopleUserPHIDType::TYPECONST; if (phid_get_type($user_phid) != $user_type) { return null; } $user = id(new PhabricatorPeopleQuery()) ->setViewer($viewer) ->withPHIDs(array($user_phid)) ->executeOne(); if (!$user) { return null; } return $user; } }