diff --git a/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php b/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php index a8e9000787..65d503d114 100644 --- a/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php +++ b/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php @@ -1,61 +1,63 @@ 'required string', ); } protected function getGitResult(ConduitAPIRequest $request) { $drequest = $this->getDiffusionRequest(); $repository = $drequest->getRepository(); $commit = $request->getValue('commit'); // NOTE: We can't use DiffusionLowLevelGitRefQuery here because // `git for-each-ref` does not support `--contains`. if ($repository->isWorkingCopyBare()) { list($contains) = $repository->execxLocalCommand( 'branch --verbose --no-abbrev --contains %s', $commit); return DiffusionGitBranch::parseLocalBranchOutput( $contains); } else { list($contains) = $repository->execxLocalCommand( 'branch -r --verbose --no-abbrev --contains %s', $commit); return DiffusionGitBranch::parseRemoteBranchOutput( $contains, DiffusionBranchInformation::DEFAULT_GIT_REMOTE); } } protected function getMercurialResult(ConduitAPIRequest $request) { $drequest = $this->getDiffusionRequest(); $repository = $drequest->getRepository(); $commit = $request->getValue('commit'); + // TODO: This should use `branches`. + list($contains) = $repository->execxLocalCommand( 'log --template %s --limit 1 --rev %s --', '{branch}', $commit); return array( trim($contains) => $commit, ); } } diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index 7ce6297d57..4f1b80534c 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -1,460 +1,668 @@ remoteProtocol = $remote_protocol; return $this; } public function getRemoteProtocol() { return $this->remoteProtocol; } public function setRemoteAddress($remote_address) { $this->remoteAddress = $remote_address; return $this; } public function getRemoteAddress() { return $this->remoteAddress; } private function getRemoteAddressForLog() { // If whatever we have here isn't a valid IPv4 address, just store `null`. // Older versions of PHP return `-1` on failure instead of `false`. $remote_address = $this->getRemoteAddress(); $remote_address = max(0, ip2long($remote_address)); $remote_address = nonempty($remote_address, null); return $remote_address; } private function getTransactionKey() { if (!$this->transactionKey) { $entropy = Filesystem::readRandomBytes(64); $this->transactionKey = PhabricatorHash::digestForIndex($entropy); } return $this->transactionKey; } public function setSubversionTransactionInfo($transaction, $repository) { $this->subversionTransaction = $transaction; $this->subversionRepository = $repository; return $this; } public function setStdin($stdin) { $this->stdin = $stdin; return $this; } public function getStdin() { return $this->stdin; } public function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } /* -( Hook Execution )----------------------------------------------------- */ public function execute() { $ref_updates = $this->findRefUpdates(); $all_updates = $ref_updates; $caught = null; try { try { $this->rejectDangerousChanges($ref_updates); } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting dangerous changes, flag everything that we've // seen as rejected so it's clear that none of it was accepted. foreach ($all_updates as $update) { $update->setRejectCode( PhabricatorRepositoryPushLog::REJECT_DANGEROUS); } throw $ex; } // TODO: Fire ref herald rules. $content_updates = $this->findContentUpdates($ref_updates); $all_updates = array_merge($all_updates, $content_updates); // TODO: Fire content Herald rules. // TODO: Fire external hooks. // If we make it this far, we're accepting these changes. Mark all the // logs as accepted. foreach ($all_updates as $update) { $update->setRejectCode(PhabricatorRepositoryPushLog::REJECT_ACCEPT); } } catch (Exception $ex) { // We'll throw this again in a minute, but we want to save all the logs // first. $caught = $ex; } // Save all the logs no matter what the outcome was. foreach ($all_updates as $update) { $update->save(); } if ($caught) { throw $caught; } return 0; } private function findRefUpdates() { $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionRefUpdates(); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } private function rejectDangerousChanges(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $repository = $this->getRepository(); if ($repository->shouldAllowDangerousChanges()) { return; } $flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; foreach ($ref_updates as $ref_update) { if (!$ref_update->hasChangeFlags($flag_dangerous)) { // This is not a dangerous change. continue; } // We either have a branch deletion or a non fast-forward branch update. // Format a message and reject the push. $message = pht( "DANGEROUS CHANGE: %s\n". "Dangerous change protection is enabled for this repository.\n". "Edit the repository configuration before making dangerous changes.", $ref_update->getDangerousChangeDescription()); throw new DiffusionCommitHookRejectException($message); } } private function findContentUpdates(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionContentUpdates($ref_updates); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } /* -( Git )---------------------------------------------------------------- */ private function findGitRefUpdates() { $ref_updates = array(); // First, parse stdin, which lists all the ref changes. The input looks // like this: // // $stdin = $this->getStdin(); $lines = phutil_split_lines($stdin, $retain_endings = false); foreach ($lines as $line) { $parts = explode(' ', $line, 3); if (count($parts) != 3) { throw new Exception(pht('Expected "old new ref", got "%s".', $line)); } $ref_old = $parts[0]; $ref_new = $parts[1]; $ref_raw = $parts[2]; if (preg_match('(^refs/heads/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; } else if (preg_match('(^refs/tags/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG; } else { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_UNKNOWN; } $ref_update = $this->newPushLog() ->setRefType($ref_type) ->setRefName($ref_raw) ->setRefOld($ref_old) ->setRefNew($ref_new); $ref_updates[] = $ref_update; } $this->findGitMergeBases($ref_updates); $this->findGitChangeFlags($ref_updates); return $ref_updates; } private function findGitMergeBases(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $futures = array(); foreach ($ref_updates as $key => $ref_update) { // If the old hash is "00000...", the ref is being created (either a new // branch, or a new tag). If the new hash is "00000...", the ref is being // deleted. If both are nonempty, the ref is being updated. For updates, // we'll figure out the `merge-base` of the old and new objects here. This // lets us reject non-FF changes cheaply; later, we'll figure out exactly // which commits are new. $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); if (($ref_old === self::EMPTY_HASH) || ($ref_new === self::EMPTY_HASH)) { continue; } $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'merge-base %s %s', $ref_old, $ref_new); } foreach (Futures($futures)->limit(8) as $key => $future) { // If 'old' and 'new' have no common ancestors (for example, a force push // which completely rewrites a ref), `git merge-base` will exit with // an error and no output. It would be nice to find a positive test // for this instead, but I couldn't immediately come up with one. See // T4224. Assume this means there are no ancestors. list($err, $stdout) = $future->resolve(); if ($err) { $merge_base = null; } else { $merge_base = rtrim($stdout, "\n"); } $ref_update->setMergeBase($merge_base); } return $ref_updates; } private function findGitChangeFlags(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); foreach ($ref_updates as $key => $ref_update) { $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); $ref_type = $ref_update->getRefType(); $ref_flags = 0; $dangerous = null; if ($ref_old === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($ref_new === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push deletes the branch '%s'.", $ref_update->getRefName()); } } else { $merge_base = $ref_update->getMergeBase(); if ($merge_base == $ref_old) { // This is a fast-forward update to an existing branch. // These are safe. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; // For now, we don't consider deleting or moving tags to be a // "dangerous" update. It's way harder to get wrong and should be easy // to recover from once we have better logging. Only add the dangerous // flag if this ref is a branch. if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( - "DANGEROUS CHANGE: The change you're attempting to push updates ". - "the branch '%s' from '%s' to '%s', but this is not a ". - "fast-forward. Pushes which rewrite published branch history ". - "are dangerous.", + "The change you're attempting to push updates the branch '%s' ". + "from '%s' to '%s', but this is not a fast-forward. Pushes ". + "which rewrite published branch history are dangerous.", $ref_update->getRefName(), $ref_update->getRefOldShort(), $ref_update->getRefNewShort()); } } } $ref_update->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } } return $ref_updates; } private function findGitContentUpdates(array $ref_updates) { $flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; $futures = array(); foreach ($ref_updates as $key => $ref_update) { if ($ref_update->hasChangeFlags($flag_delete)) { // Deleting a branch or tag can never create any new commits. continue; } // NOTE: This piece of magic finds all new commits, by walking backward // from the new value to the value of *any* existing ref in the // repository. Particularly, this will cover the cases of a new branch, a // completely moved tag, etc. $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'log --format=%s %s --not --all', '%H', $ref_update->getRefNew()); } $content_updates = array(); foreach (Futures($futures)->limit(8) as $key => $future) { list($stdout) = $future->resolvex(); if (!strlen(trim($stdout))) { // This change doesn't have any new commits. One common case of this // is creating a new tag which points at an existing commit. continue; } $commits = phutil_split_lines($stdout, $retain_newlines = false); foreach ($commits as $commit) { $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } } return $content_updates; } /* -( Mercurial )---------------------------------------------------------- */ private function findMercurialRefUpdates() { - // TODO: Implement. - return array(); + $hg_node = getenv('HG_NODE'); + if (!$hg_node) { + throw new Exception(pht('Expected HG_NODE in environment!')); + } + + // NOTE: We need to make sure this is passed to subprocesses, or they won't + // be able to see new commits. Mercurial uses this as a marker to determine + // whether the pending changes are visible or not. + $_ENV['HG_PENDING'] = getenv('HG_PENDING'); + $repository = $this->getRepository(); + + $futures = array(); + + foreach (array('old', 'new') as $key) { + $futures[$key] = $repository->getLocalCommandFuture( + 'heads --template %s', + '{node}\1{branches}\2'); + } + // Wipe HG_PENDING out of the old environment so we see the pre-commit + // state of the repository. + $futures['old']->updateEnv('HG_PENDING', null); + + $futures['commits'] = $repository->getLocalCommandFuture( + "log --rev %s --rev tip --template %s", + hgsprintf('%s', $hg_node), + '{node}\1{branches}\2'); + + // Resolve all of the futures now. We don't need the 'commits' future yet, + // but it simplifies the logic to just get it out of the way. + foreach (Futures($futures) as $future) { + $future->resolvex(); + } + + list($commit_raw) = $futures['commits']->resolvex(); + $commit_map = $this->parseMercurialCommits($commit_raw); + + list($old_raw) = $futures['old']->resolvex(); + $old_refs = $this->parseMercurialHeads($old_raw); + + list($new_raw) = $futures['new']->resolvex(); + $new_refs = $this->parseMercurialHeads($new_raw); + + $all_refs = array_keys($old_refs + $new_refs); + + $ref_updates = array(); + foreach ($all_refs as $ref) { + $old_heads = idx($old_refs, $ref, array()); + $new_heads = idx($new_refs, $ref, array()); + + sort($old_heads); + sort($new_heads); + + if ($old_heads === $new_heads) { + // No changes to this branch, so skip it. + continue; + } + + if (!$new_heads) { + if ($old_heads) { + // It looks like this push deletes a branch, but that isn't possible + // in Mercurial, so something is going wrong here. Bail out. + throw new Exception( + pht( + 'Mercurial repository has no new head for branch "%s" after '. + 'push. This is unexpected; rejecting change.')); + } else { + // Obviously, this should never be possible either, as it makes + // no sense. Explode. + throw new Exception( + pht( + 'Mercurial repository has no new or old heads for branch "%s" '. + 'after push. This makes no sense; rejecting change.')); + } + } + + $stray_heads = array(); + if (count($old_heads) > 1) { + // HORRIBLE: In Mercurial, branches can have multiple heads. If the + // old branch had multiple heads, we need to figure out which new + // heads descend from which old heads, so we can tell whether you're + // actively creating new heads (dangerous) or just working in a + // repository that's already full of garbage (strongly discouraged but + // not as inherently dangerous). These cases should be very uncommon. + + $dfutures = array(); + foreach ($old_heads as $old_head) { + $dfutures[$old_head] = $repository->getLocalCommandFuture( + 'log --rev %s --template %s', + hgsprintf('(descendants(%s) and head())', $old_head), + '{node}\1'); + } + + $head_map = array(); + foreach (Futures($dfutures) as $future_head => $dfuture) { + list($stdout) = $dfuture->resolvex(); + $head_map[$future_head] = array_filter(explode("\1", $stdout)); + } + + // Now, find all the new stray heads this push creates, if any. These + // are new heads which do not descend from the old heads. + $seen = array_fuse(array_mergev($head_map)); + foreach ($new_heads as $new_head) { + if (empty($seen[$new_head])) { + $head_map[self::EMPTY_HASH][] = $new_head; + } + } + } else if ($old_heads) { + $head_map[head($old_heads)] = $new_heads; + } else { + $head_map[self::EMPTY_HASH] = $new_heads; + } + + foreach ($head_map as $old_head => $child_heads) { + foreach ($child_heads as $new_head) { + if ($new_head === $old_head) { + continue; + } + + $ref_flags = 0; + $dangerous = null; + if ($old_head == self::EMPTY_HASH) { + $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; + } else { + $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; + } + + $splits_existing_head = (count($child_heads) > 1); + $creates_duplicate_head = ($old_head == self::EMPTY_HASH) && + (count($head_map) > 1); + + if ($splits_existing_head || $creates_duplicate_head) { + $readable_child_heads = array(); + foreach ($child_heads as $child_head) { + $readable_child_heads[] = substr($child_head, 0, 12); + } + + $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; + + if ($splits_existing_head) { + // We're splitting an existing head into two or more heads. + // This is dangerous, and a super bad idea. Note that we're only + // raising this if you're actively splitting a branch head. If a + // head split in the past, we don't consider appends to it + // to be dangerous. + $dangerous = pht( + "The change you're attempting to push splits the head of ". + "branch '%s' into multiple heads: %s. This is inadvisable ". + "and dangerous.", + $ref, + implode(', ', $readable_child_heads)); + } else { + // We're adding a second (or more) head to a branch. The new + // head is not a descendant of any old head. + $dangerous = pht( + "The change you're attempting to push creates new, divergent ". + "heads for the branch '%s': %s. This is inadvisable and ". + "dangerous.", + $ref, + implode(', ', $readable_child_heads)); + } + } + + $ref_update = $this->newPushLog() + ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH) + ->setRefName($ref) + ->setRefOld($old_head) + ->setRefNew($new_head) + ->setChangeFlags($ref_flags); + + if ($dangerous !== null) { + $ref_update->attachDangerousChangeDescription($dangerous); + } + + $ref_updates[] = $ref_update; + } + } + } + + return $ref_updates; + } + + private function parseMercurialCommits($raw) { + $commits_lines = explode("\2", $raw); + $commits_lines = array_filter($commits_lines); + $commit_map = array(); + foreach ($commits_lines as $commit_line) { + list($node, $branches_raw) = explode("\1", $commit_line); + + if (!strlen($branches_raw)) { + $branches = array('default'); + } else { + $branches = explode(' ', $branches_raw); + } + + $commit_map[$node] = $branches; + } + + return $commit_map; + } + + private function parseMercurialHeads($raw) { + $heads_map = $this->parseMercurialCommits($raw); + + $heads = array(); + foreach ($heads_map as $commit => $branches) { + foreach ($branches as $branch) { + $heads[$branch][] = $commit; + } + } + + return $heads; } private function findMercurialContentUpdates(array $ref_updates) { // TODO: Implement. return array(); } /* -( Subversion )--------------------------------------------------------- */ private function findSubversionRefUpdates() { // TODO: Implement. return array(); } private function findSubversionContentUpdates(array $ref_updates) { // TODO: Implement. return array(); } /* -( Internals )---------------------------------------------------------- */ private function newPushLog() { // NOTE: By default, we create these with REJECT_BROKEN as the reject // code. This indicates a broken hook, and covers the case where we // encounter some unexpected exception and consequently reject the changes. return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer()) ->attachRepository($this->getRepository()) ->setRepositoryPHID($this->getRepository()->getPHID()) ->setEpoch(time()) ->setRemoteAddress($this->getRemoteAddressForLog()) ->setRemoteProtocol($this->getRemoteProtocol()) ->setTransactionKey($this->getTransactionKey()) ->setRejectCode(PhabricatorRepositoryPushLog::REJECT_BROKEN) ->setRejectDetails(null); } } diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index 0bf9007d03..62cb4eeb3f 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -1,853 +1,853 @@ repair = $repair; return $this; } /* -( Pulling Repositories )----------------------------------------------- */ /** * @task pull */ public function run() { $argv = $this->getArgv(); array_unshift($argv, __CLASS__); $args = new PhutilArgumentParser($argv); $args->parse( array( array( 'name' => 'no-discovery', 'help' => 'Pull only, without discovering commits.', ), array( 'name' => 'not', 'param' => 'repository', 'repeat' => true, 'help' => 'Do not pull __repository__.', ), array( 'name' => 'repositories', 'wildcard' => true, 'help' => 'Pull specific __repositories__ instead of all.', ), )); $no_discovery = $args->getArg('no-discovery'); $repo_names = $args->getArg('repositories'); $exclude_names = $args->getArg('not'); // Each repository has an individual pull frequency; after we pull it, // wait that long to pull it again. When we start up, try to pull everything // serially. $retry_after = array(); $min_sleep = 15; while (true) { $repositories = $this->loadRepositories($repo_names); if ($exclude_names) { $exclude = $this->loadRepositories($exclude_names); $repositories = array_diff_key($repositories, $exclude); } // Shuffle the repositories, then re-key the array since shuffle() // discards keys. This is mostly for startup, we'll use soft priorities // later. shuffle($repositories); $repositories = mpull($repositories, null, 'getID'); // If any repositories have the NEEDS_UPDATE flag set, pull them // as soon as possible. $type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE; $need_update_messages = id(new PhabricatorRepositoryStatusMessage()) ->loadAllWhere('statusType = %s', $type_need_update); foreach ($need_update_messages as $message) { $retry_after[$message->getRepositoryID()] = time(); } // If any repositories were deleted, remove them from the retry timer map // so we don't end up with a retry timer that never gets updated and // causes us to sleep for the minimum amount of time. $retry_after = array_select_keys( $retry_after, array_keys($repositories)); // Assign soft priorities to repositories based on how frequently they // should pull again. asort($retry_after); $repositories = array_select_keys( $repositories, array_keys($retry_after)) + $repositories; foreach ($repositories as $id => $repository) { $after = idx($retry_after, $id, 0); if ($after > time()) { continue; } $tracked = $repository->isTracked(); if (!$tracked) { continue; } $callsign = $repository->getCallsign(); try { $this->log("Updating repository '{$callsign}'."); id(new PhabricatorRepositoryPullEngine()) ->setRepository($repository) ->pullRepository(); if (!$no_discovery) { // TODO: It would be nice to discover only if we pulled something, // but this isn't totally trivial. It's slightly more complicated // with hosted repositories, too. $lock_name = get_class($this).':'.$callsign; $lock = PhabricatorGlobalLock::newLock($lock_name); $lock->lock(); $repository->writeStatusMessage( PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, null); try { $this->discoverRepository($repository); $repository->writeStatusMessage( PhabricatorRepositoryStatusMessage::TYPE_FETCH, PhabricatorRepositoryStatusMessage::CODE_OKAY); } catch (Exception $ex) { $repository->writeStatusMessage( PhabricatorRepositoryStatusMessage::TYPE_FETCH, PhabricatorRepositoryStatusMessage::CODE_ERROR, array( 'message' => pht( 'Error updating working copy: %s', $ex->getMessage()), )); $lock->unlock(); throw $ex; } $lock->unlock(); } $sleep_for = $repository->getDetail('pull-frequency', $min_sleep); $retry_after[$id] = time() + $sleep_for; } catch (PhutilLockException $ex) { $retry_after[$id] = time() + $min_sleep; $this->log("Failed to acquire lock."); } catch (Exception $ex) { $retry_after[$id] = time() + $min_sleep; $proxy = new PhutilProxyException( "Error while fetching changes to the '{$callsign}' repository.", $ex); phlog($proxy); } $this->stillWorking(); } if ($retry_after) { $sleep_until = max(min($retry_after), time() + $min_sleep); } else { $sleep_until = time() + $min_sleep; } $this->sleep($sleep_until - time()); } } /** * @task pull */ protected function loadRepositories(array $names) { $query = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()); if ($names) { $query->withCallsigns($names); } $repos = $query->execute(); if ($names) { $by_callsign = mpull($repos, null, 'getCallsign'); foreach ($names as $name) { if (empty($by_callsign[$name])) { throw new Exception( "No repository exists with callsign '{$name}'!"); } } } return $repos; } public function discoverRepository(PhabricatorRepository $repository) { $vcs = $repository->getVersionControlSystem(); $result = null; $refs = null; switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $result = $this->executeGitDiscover($repository); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $refs = $this->getDiscoveryEngine($repository) ->discoverCommits(); break; default: throw new Exception("Unknown VCS '{$vcs}'!"); } if ($refs !== null) { foreach ($refs as $ref) { $this->recordCommit( $repository, $ref->getIdentifier(), $ref->getEpoch(), $ref->getBranch()); } } $this->checkIfRepositoryIsFullyImported($repository); try { $this->pushToMirrors($repository); } catch (Exception $ex) { // TODO: We should report these into the UI properly, but for // now just complain. These errors are much less severe than // pull errors. phlog($ex); } if ($refs !== null) { return (bool)count($refs); } else { return $result; } } private function getDiscoveryEngine(PhabricatorRepository $repository) { $id = $repository->getID(); if (empty($this->discoveryEngines[$id])) { $engine = id(new PhabricatorRepositoryDiscoveryEngine()) ->setRepository($repository) ->setVerbose($this->getVerbose()) ->setRepairMode($this->repair); $this->discoveryEngines[$id] = $engine; } return $this->discoveryEngines[$id]; } private function isKnownCommit( PhabricatorRepository $repository, $target) { if ($this->getCache($repository, $target)) { return true; } if ($this->repair) { // In repair mode, rediscover the entire repository, ignoring the // database state. We can hit the local cache above, but if we miss it // stop the script from going to the database cache. return false; } $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( 'repositoryID = %d AND commitIdentifier = %s', $repository->getID(), $target); if (!$commit) { return false; } $this->setCache($repository, $target); while (count($this->commitCache) > 2048) { array_shift($this->commitCache); } return true; } private function isKnownCommitOnAnyAutocloseBranch( PhabricatorRepository $repository, $target) { $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( 'repositoryID = %d AND commitIdentifier = %s', $repository->getID(), $target); if (!$commit) { $callsign = $repository->getCallsign(); $console = PhutilConsole::getConsole(); $console->writeErr( "WARNING: Repository '%s' is missing commits ('%s' is missing from ". "history). Run '%s' to repair the repository.\n", $callsign, $target, "bin/repository discover --repair {$callsign}"); return false; } $data = $commit->loadCommitData(); if (!$data) { return false; } if ($repository->shouldAutocloseCommit($commit, $data)) { return true; } return false; } private function recordCommit( PhabricatorRepository $repository, $commit_identifier, $epoch, $branch = null) { $commit = new PhabricatorRepositoryCommit(); $commit->setRepositoryID($repository->getID()); $commit->setCommitIdentifier($commit_identifier); $commit->setEpoch($epoch); $data = new PhabricatorRepositoryCommitData(); if ($branch) { $data->setCommitDetail('seenOnBranches', array($branch)); } try { $commit->openTransaction(); $commit->save(); $data->setCommitID($commit->getID()); $data->save(); $commit->saveTransaction(); $this->insertTask($repository, $commit); queryfx( $repository->establishConnection('w'), 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) VALUES (%d, 1, %d, %d) ON DUPLICATE KEY UPDATE size = size + 1, lastCommitID = IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', PhabricatorRepository::TABLE_SUMMARY, $repository->getID(), $commit->getID(), $epoch); if ($this->repair) { // Normally, the query should throw a duplicate key exception. If we // reach this in repair mode, we've actually performed a repair. $this->log("Repaired commit '{$commit_identifier}'."); } $this->setCache($repository, $commit_identifier); PhutilEventEngine::dispatchEvent( new PhabricatorEvent( PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT, array( 'repository' => $repository, 'commit' => $commit, ))); } catch (AphrontQueryDuplicateKeyException $ex) { $commit->killTransaction(); // Ignore. This can happen because we discover the same new commit // more than once when looking at history, or because of races or // data inconsistency or cosmic radiation; in any case, we're still // in a good state if we ignore the failure. $this->setCache($repository, $commit_identifier); } } private function updateCommit( PhabricatorRepository $repository, $commit_identifier, $branch) { $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( 'repositoryID = %d AND commitIdentifier = %s', $repository->getID(), $commit_identifier); if (!$commit) { // This can happen if the phabricator DB doesn't have the commit info, // or the commit is so big that phabricator couldn't parse it. In this // case we just ignore it. return; } $data = id(new PhabricatorRepositoryCommitData())->loadOneWhere( 'commitID = %d', $commit->getID()); if (!$data) { $data = new PhabricatorRepositoryCommitData(); $data->setCommitID($commit->getID()); } $branches = $data->getCommitDetail('seenOnBranches', array()); $branches[] = $branch; $data->setCommitDetail('seenOnBranches', $branches); $data->save(); $this->insertTask( $repository, $commit, array( 'only' => true )); } private function insertTask( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, $data = array()) { $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; break; default: throw new Exception("Unknown repository type '{$vcs}'!"); } $data['commitID'] = $commit->getID(); PhabricatorWorker::scheduleTask($class, $data); } private function setCache( PhabricatorRepository $repository, $commit_identifier) { $key = $this->getCacheKey($repository, $commit_identifier); $this->commitCache[$key] = true; } private function getCache( PhabricatorRepository $repository, $commit_identifier) { $key = $this->getCacheKey($repository, $commit_identifier); return idx($this->commitCache, $key, false); } private function getCacheKey( PhabricatorRepository $repository, $commit_identifier) { return $repository->getID().':'.$commit_identifier; } private function checkIfRepositoryIsFullyImported( PhabricatorRepository $repository) { // Check if the repository has the "Importing" flag set. We want to clear // the flag if we can. $importing = $repository->getDetail('importing'); if (!$importing) { // This repository isn't marked as "Importing", so we're done. return; } // Look for any commit which hasn't imported. $unparsed_commit = queryfx_one( $repository->establishConnection('r'), 'SELECT * FROM %T WHERE repositoryID = %d AND importStatus != %d LIMIT 1', id(new PhabricatorRepositoryCommit())->getTableName(), $repository->getID(), PhabricatorRepositoryCommit::IMPORTED_ALL); if ($unparsed_commit) { // We found a commit which still needs to import, so we can't clear the // flag. return; } // Clear the "importing" flag. $repository->openTransaction(); $repository->beginReadLocking(); $repository = $repository->reload(); $repository->setDetail('importing', false); $repository->save(); $repository->endReadLocking(); $repository->saveTransaction(); } /* -( Git Implementation )------------------------------------------------- */ /** * @task git */ private function executeGitDiscover( PhabricatorRepository $repository) { if (!$repository->isHosted()) { $this->verifyOrigin($repository); } $refs = id(new DiffusionLowLevelGitRefQuery()) ->setRepository($repository) ->withIsOriginBranch(true) ->execute(); $branches = mpull($refs, 'getCommitIdentifier', 'getShortName'); if (!$branches) { // This repository has no branches at all, so we don't need to do // anything. Generally, this means the repository is empty. return; } $callsign = $repository->getCallsign(); $tracked_something = false; $this->log("Discovering commits in repository '{$callsign}'..."); foreach ($branches as $name => $commit) { $this->log("Examining branch '{$name}', at {$commit}."); if (!$repository->shouldTrackBranch($name)) { $this->log("Skipping, branch is untracked."); continue; } $tracked_something = true; if ($this->isKnownCommit($repository, $commit)) { $this->log("Skipping, HEAD is known."); continue; } $this->log("Looking for new commits."); $this->executeGitDiscoverCommit($repository, $commit, $name, false); } if (!$tracked_something) { $repo_name = $repository->getName(); $repo_callsign = $repository->getCallsign(); throw new Exception( "Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ". "Verify that your branch filtering settings are correct."); } $this->log("Discovering commits on autoclose branches..."); foreach ($branches as $name => $commit) { $this->log("Examining branch '{$name}', at {$commit}'."); if (!$repository->shouldTrackBranch($name)) { $this->log("Skipping, branch is untracked."); continue; } if (!$repository->shouldAutocloseBranch($name)) { $this->log("Skipping, branch is not autoclose."); continue; } if ($this->isKnownCommitOnAnyAutocloseBranch($repository, $commit)) { $this->log("Skipping, commit is known on an autoclose branch."); continue; } $this->log("Looking for new autoclose commits."); $this->executeGitDiscoverCommit($repository, $commit, $name, true); } } /** * @task git */ private function executeGitDiscoverCommit( PhabricatorRepository $repository, $commit, $branch, $autoclose) { $discover = array($commit); $insert = array($commit); $seen_parent = array(); $stream = new PhabricatorGitGraphStream($repository, $commit); while (true) { $target = array_pop($discover); $parents = $stream->getParents($target); foreach ($parents as $parent) { if (isset($seen_parent[$parent])) { // We end up in a loop here somehow when we parse Arcanist if we // don't do this. TODO: Figure out why and draw a pretty diagram // since it's not evident how parsing a DAG with this causes the // loop to stop terminating. continue; } $seen_parent[$parent] = true; if ($autoclose) { $known = $this->isKnownCommitOnAnyAutocloseBranch( $repository, $parent); } else { $known = $this->isKnownCommit($repository, $parent); } if (!$known) { $this->log("Discovered commit '{$parent}'."); $discover[] = $parent; $insert[] = $parent; } } if (empty($discover)) { break; } } $n = count($insert); if ($autoclose) { $this->log("Found {$n} new autoclose commits on branch '{$branch}'."); } else { $this->log("Found {$n} new commits on branch '{$branch}'."); } while (true) { $target = array_pop($insert); $epoch = $stream->getCommitDate($target); $epoch = trim($epoch); if ($autoclose) { $this->updateCommit($repository, $target, $branch); } else { $this->recordCommit($repository, $target, $epoch, $branch); } if (empty($insert)) { break; } } } /** * Verify that the "origin" remote exists, and points at the correct URI. * * This catches or corrects some types of misconfiguration, and also repairs * an issue where Git 1.7.1 does not create an "origin" for `--bare` clones. * See T4041. * * @param PhabricatorRepository Repository to verify. * @return void */ private function verifyOrigin(PhabricatorRepository $repository) { list($remotes) = $repository->execxLocalCommand( 'remote show -n origin'); $matches = null; if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) { throw new Exception( "Expected 'Fetch URL' in 'git remote show -n origin'."); } $remote_uri = $matches[1]; $expect_remote = $repository->getRemoteURI(); if ($remote_uri == "origin") { // If a remote does not exist, git pretends it does and prints out a // made up remote where the URI is the same as the remote name. This is // definitely not correct. // Possibly, we should use `git remote --verbose` instead, which does not // suffer from this problem (but is a little more complicated to parse). $valid = false; $exists = false; } else { $valid = self::isSameGitOrigin($remote_uri, $expect_remote); $exists = true; } if (!$valid) { if (!$exists) { // If there's no "origin" remote, just create it regardless of how // strongly we own the working copy. There is almost no conceivable // scenario in which this could do damage. $this->log( pht( 'Remote "origin" does not exist. Creating "origin", with '. 'URI "%s".', $expect_remote)); $repository->execxLocalCommand( - 'remote add origin %s', - $expect_remote); + 'remote add origin %P', + $repository->getRemoteURIEnvelope()); // NOTE: This doesn't fetch the origin (it just creates it), so we won't // know about origin branches until the next "pull" happens. That's fine // for our purposes, but might impact things in the future. } else { if ($repository->canDestroyWorkingCopy()) { // Bad remote, but we can try to repair it. $this->log( pht( 'Remote "origin" exists, but is pointed at the wrong URI, "%s". '. 'Resetting origin URI to "%s.', $remote_uri, $expect_remote)); $repository->execxLocalCommand( - 'remote set-url origin %s', - $expect_remote); + 'remote set-url origin %P', + $repository->getRemoteURIEnvelope()); } else { // Bad remote and we aren't comfortable repairing it. $message = pht( 'Working copy at "%s" has a mismatched origin URI, "%s". '. 'The expected origin URI is "%s". Fix your configuration, or '. 'set the remote URI correctly. To avoid breaking anything, '. 'Phabricator will not automatically fix this.', $repository->getLocalPath(), $remote_uri, $expect_remote); throw new Exception($message); } } } } /** * @task git */ public static function isSameGitOrigin($remote, $expect) { $remote_path = self::getPathFromGitURI($remote); $expect_path = self::getPathFromGitURI($expect); $remote_match = self::executeGitNormalizePath($remote_path); $expect_match = self::executeGitNormalizePath($expect_path); return ($remote_match == $expect_match); } private static function getPathFromGitURI($raw_uri) { $uri = new PhutilURI($raw_uri); if ($uri->getProtocol()) { return $uri->getPath(); } $uri = new PhutilGitURI($raw_uri); if ($uri->getDomain()) { return $uri->getPath(); } return $raw_uri; } /** * @task git */ private static function executeGitNormalizePath($path) { // Strip away "/" and ".git", so similar paths correctly match. $path = trim($path, '/'); $path = preg_replace('/\.git$/', '', $path); return $path; } private function pushToMirrors(PhabricatorRepository $repository) { if (!$repository->canMirror()) { return; } $mirrors = id(new PhabricatorRepositoryMirrorQuery()) ->setViewer($this->getViewer()) ->withRepositoryPHIDs(array($repository->getPHID())) ->execute(); // TODO: This is a little bit janky, but we don't have first-class // infrastructure for running remote commands against an arbitrary remote // right now. Just make an emphemeral copy of the repository and muck with // it a little bit. In the medium term, we should pull this command stuff // out and use it here and for "Land to ...". $proxy = clone $repository; $proxy->makeEphemeral(); $proxy->setDetail('hosting-enabled', false); foreach ($mirrors as $mirror) { $proxy->setDetail('remote-uri', $mirror->getRemoteURI()); $proxy->setCredentialPHID($mirror->getCredentialPHID()); $this->log(pht('Pushing to remote "%s"...', $mirror->getRemoteURI())); if (!$proxy->isGit()) { throw new Exception('Unsupported VCS!'); } $future = $proxy->getRemoteCommandFuture( 'push --verbose --mirror -- %P', $proxy->getRemoteURIEnvelope()); $future ->setCWD($proxy->getLocalPath()) ->resolvex(); } } }