diff --git a/src/applications/diffusion/data/DiffusionRepositoryRef.php b/src/applications/diffusion/data/DiffusionRepositoryRef.php index 45c92c8c0a..3e68d6f1c0 100644 --- a/src/applications/diffusion/data/DiffusionRepositoryRef.php +++ b/src/applications/diffusion/data/DiffusionRepositoryRef.php @@ -1,66 +1,83 @@ rawFields = $raw_fields; return $this; } public function getRawFields() { return $this->rawFields; } public function setCommitIdentifier($commit_identifier) { $this->commitIdentifier = $commit_identifier; return $this; } public function getCommitIdentifier() { return $this->commitIdentifier; } public function setShortName($short_name) { $this->shortName = $short_name; return $this; } public function getShortName() { return $this->shortName; } + public function setRefType($ref_type) { + $this->refType = $ref_type; + return $this; + } + + public function getRefType() { + return $this->refType; + } + + public function isBranch() { + $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH; + return ($this->getRefType() === $type_branch); + } + /* -( Serialization )------------------------------------------------------ */ public function toDictionary() { return array( 'shortName' => $this->shortName, 'commitIdentifier' => $this->commitIdentifier, + 'refType' => $this->refType, 'rawFields' => $this->rawFields, ); } public static function newFromDictionary(array $dict) { return id(new DiffusionRepositoryRef()) ->setShortName($dict['shortName']) ->setCommitIdentifier($dict['commitIdentifier']) + ->setRefType($dict['refType']) ->setRawFields($dict['rawFields']); } public static function loadAllFromDictionaries(array $dictionaries) { $refs = array(); foreach ($dictionaries as $dictionary) { $refs[] = self::newFromDictionary($dictionary); } return $refs; } } diff --git a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php index 038d833670..de3866594d 100644 --- a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php +++ b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php @@ -1,155 +1,171 @@ refTypes = $ref_types; return $this; } protected function executeQuery() { + $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH; + $type_tag = PhabricatorRepositoryRefCursor::TYPE_TAG; + $type_ref = PhabricatorRepositoryRefCursor::TYPE_REF; + $ref_types = $this->refTypes; - if ($ref_types) { - $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH; - $type_tag = PhabricatorRepositoryRefCursor::TYPE_TAG; + if (!$ref_types) { + $ref_types = array($type_branch, $type_tag, $type_ref); + } - $ref_types = array_fuse($ref_types); + $ref_types = array_fuse($ref_types); - $with_branches = isset($ref_types[$type_branch]); - $with_tags = isset($ref_types[$type_tag]); - } else { - $with_branches = true; - $with_tags = true; - } + $with_branches = isset($ref_types[$type_branch]); + $with_tags = isset($ref_types[$type_tag]); + $with_refs = isset($refs_types[$type_ref]); $repository = $this->getRepository(); $prefixes = array(); - if ($with_branches) { - if ($repository->isWorkingCopyBare()) { - $prefix = 'refs/heads/'; - } else { - $remote = DiffusionGitBranch::DEFAULT_GIT_REMOTE; - $prefix = 'refs/remotes/'.$remote.'/'; - } - $prefixes[] = $prefix; + if ($repository->isWorkingCopyBare()) { + $branch_prefix = 'refs/heads/'; + } else { + $remote = DiffusionGitBranch::DEFAULT_GIT_REMOTE; + $branch_prefix = 'refs/remotes/'.$remote.'/'; } - if ($with_tags) { - $prefixes[] = 'refs/tags/'; - } + $tag_prefix = 'refs/tags/'; - $order = '-creatordate'; - $futures = array(); - foreach ($prefixes as $prefix) { - $futures[$prefix] = $repository->getLocalCommandFuture( - 'for-each-ref --sort=%s --format=%s %s', - $order, - $this->getFormatString(), - $prefix); + if ($with_refs || count($ref_types) > 1) { + // If we're loading refs or more than one type of ref, just query + // everything. + $prefix = 'refs/'; + } else { + if ($with_branches) { + $prefix = $branch_prefix; + } + if ($with_tags) { + $prefix = $tag_prefix; + } } - // Resolve all the futures first. We want to iterate over them in prefix - // order, not resolution order. - foreach (new FutureIterator($futures) as $prefix => $future) { - $future->resolvex(); + $branch_len = strlen($branch_prefix); + $tag_len = strlen($tag_prefix); + + list($stdout) = $repository->execxLocalCommand( + 'for-each-ref --sort=%s --format=%s -- %s', + '-creatordate', + $this->getFormatString(), + $prefix); + + $stdout = rtrim($stdout); + if (!strlen($stdout)) { + return array(); } + // NOTE: Although git supports --count, we can't apply any offset or + // limit logic until the very end because we may encounter a HEAD which + // we want to discard. + + $lines = explode("\n", $stdout); $results = array(); - foreach ($futures as $prefix => $future) { - list($stdout) = $future->resolvex(); + foreach ($lines as $line) { + $fields = $this->extractFields($line); + + $refname = $fields['refname']; + if (!strncmp($refname, $branch_prefix, $branch_len)) { + $short = substr($refname, $branch_len); + $type = $type_branch; + } else if (!strncmp($refname, $tag_prefix, $tag_len)) { + $short = substr($refname, $tag_len); + $type = $type_tag; + } else { + $short = $refname; + $type = $type_ref; + } - $stdout = rtrim($stdout); - if (!strlen($stdout)) { + // If this isn't a type of ref we care about, skip it. + if (empty($ref_types[$type])) { continue; } - // NOTE: Although git supports --count, we can't apply any offset or - // limit logic until the very end because we may encounter a HEAD which - // we want to discard. - - $lines = explode("\n", $stdout); - foreach ($lines as $line) { - $fields = $this->extractFields($line); - - $creator = $fields['creator']; - $matches = null; - if (preg_match('/^(.*) ([0-9]+) ([0-9+-]+)$/', $creator, $matches)) { - $fields['author'] = $matches[1]; - $fields['epoch'] = (int)$matches[2]; - } else { - $fields['author'] = null; - $fields['epoch'] = null; - } - - $commit = nonempty($fields['*objectname'], $fields['objectname']); - - $short = substr($fields['refname'], strlen($prefix)); - if ($short == 'HEAD') { - continue; - } - - $ref = id(new DiffusionRepositoryRef()) - ->setShortName($short) - ->setCommitIdentifier($commit) - ->setRawFields($fields); - - $results[] = $ref; + // If this is the local HEAD, skip it. + if ($short == 'HEAD') { + continue; } + + $creator = $fields['creator']; + $matches = null; + if (preg_match('/^(.*) ([0-9]+) ([0-9+-]+)$/', $creator, $matches)) { + $fields['author'] = $matches[1]; + $fields['epoch'] = (int)$matches[2]; + } else { + $fields['author'] = null; + $fields['epoch'] = null; + } + + $commit = nonempty($fields['*objectname'], $fields['objectname']); + + $ref = id(new DiffusionRepositoryRef()) + ->setRefType($type) + ->setShortName($short) + ->setCommitIdentifier($commit) + ->setRawFields($fields); + + $results[] = $ref; } return $results; } /** * List of git `--format` fields we want to grab. */ private function getFields() { return array( 'objectname', 'objecttype', 'refname', '*objectname', '*objecttype', 'subject', 'creator', ); } /** * Get a string for `--format` which enumerates all the fields we want. */ private function getFormatString() { $fields = $this->getFields(); foreach ($fields as $key => $field) { $fields[$key] = '%('.$field.')'; } return implode('%01', $fields); } /** * Parse a line back into fields. */ private function extractFields($line) { $fields = $this->getFields(); $parts = explode("\1", $line, count($fields)); $dict = array(); foreach ($fields as $index => $field) { $dict[$field] = idx($parts, $index); } return $dict; } } diff --git a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php index 9e0e13c720..8f81352580 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php @@ -1,705 +1,699 @@ repairMode = $repair_mode; return $this; } public function getRepairMode() { return $this->repairMode; } /** * @task discovery */ public function discoverCommits() { $repository = $this->getRepository(); $lock = $this->newRepositoryLock($repository, 'repo.look', false); try { $lock->lock(); } catch (PhutilLockException $ex) { throw new DiffusionDaemonLockException( pht( 'Another process is currently discovering repository "%s", '. 'skipping discovery.', $repository->getDisplayName())); } try { $result = $this->discoverCommitsWithLock(); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); return $result; } private function discoverCommitsWithLock() { $repository = $this->getRepository(); $viewer = $this->getViewer(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $refs = $this->discoverSubversionCommits(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $refs = $this->discoverMercurialCommits(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $refs = $this->discoverGitCommits(); break; default: throw new Exception(pht("Unknown VCS '%s'!", $vcs)); } if ($this->isInitialImport($refs)) { $this->log( pht( 'Discovered more than %s commit(s) in an empty repository, '. 'marking repository as importing.', new PhutilNumber(PhabricatorRepository::IMPORT_THRESHOLD))); $repository->markImporting(); } // Clear the working set cache. $this->workingSet = array(); // Record discovered commits and mark them in the cache. foreach ($refs as $ref) { $this->recordCommit( $repository, $ref->getIdentifier(), $ref->getEpoch(), $ref->getCanCloseImmediately(), $ref->getParents()); $this->commitCache[$ref->getIdentifier()] = true; } $version = $this->getObservedVersion($repository); if ($version !== null) { id(new DiffusionRepositoryClusterEngine()) ->setViewer($viewer) ->setRepository($repository) ->synchronizeWorkingCopyAfterDiscovery($version); } return $refs; } /* -( Discovering Git Repositories )--------------------------------------- */ /** * @task git */ private function discoverGitCommits() { $repository = $this->getRepository(); if (!$repository->isHosted()) { $this->verifyGitOrigin($repository); } - // TODO: This should also import tags, but some of the logic is still - // branch-specific today. - - $branches = id(new DiffusionLowLevelGitRefQuery()) + $heads = id(new DiffusionLowLevelGitRefQuery()) ->setRepository($repository) - ->withRefTypes( - array( - PhabricatorRepositoryRefCursor::TYPE_BRANCH, - )) ->execute(); - if (!$branches) { - // This repository has no branches at all, so we don't need to do + if (!$heads) { + // This repository has no heads at all, so we don't need to do // anything. Generally, this means the repository is empty. return array(); } - $branches = $this->sortBranches($branches); - $branches = mpull($branches, 'getCommitIdentifier', 'getShortName'); + $heads = $this->sortRefs($heads); + $head_commits = mpull($heads, 'getCommitIdentifier'); $this->log( pht( 'Discovering commits in repository "%s".', $repository->getDisplayName())); - $this->fillCommitCache(array_values($branches)); + $this->fillCommitCache($head_commits); $refs = array(); - foreach ($branches as $name => $commit) { - $this->log(pht('Examining branch "%s", at "%s".', $name, $commit)); + foreach ($heads as $ref) { + $name = $ref->getShortName(); + $commit = $ref->getCommitIdentifier(); - if (!$repository->shouldTrackBranch($name)) { - $this->log(pht('Skipping, branch is untracked.')); + $this->log(pht('Examining ref "%s", at "%s".', $name, $commit)); + + if (!$repository->shouldTrackRef($ref)) { + $this->log(pht('Skipping, ref is untracked.')); continue; } if ($this->isKnownCommit($commit)) { $this->log(pht('Skipping, HEAD is known.')); continue; } $this->log(pht('Looking for new commits.')); - $branch_refs = $this->discoverStreamAncestry( + $head_refs = $this->discoverStreamAncestry( new PhabricatorGitGraphStream($repository, $commit), $commit, - $repository->shouldAutocloseBranch($name)); + $repository->shouldAutocloseRef($ref)); - $this->didDiscoverRefs($branch_refs); + $this->didDiscoverRefs($head_refs); - $refs[] = $branch_refs; + $refs[] = $head_refs; } return array_mergev($refs); } /* -( Discovering Subversion Repositories )-------------------------------- */ /** * @task svn */ private function discoverSubversionCommits() { $repository = $this->getRepository(); if (!$repository->isHosted()) { $this->verifySubversionRoot($repository); } $upper_bound = null; $limit = 1; $refs = array(); do { // Find all the unknown commits on this path. Note that we permit // importing an SVN subdirectory rather than the entire repository, so // commits may be nonsequential. if ($upper_bound === null) { $at_rev = 'HEAD'; } else { $at_rev = ($upper_bound - 1); } try { list($xml, $stderr) = $repository->execxRemoteCommand( 'log --xml --quiet --limit %d %s', $limit, $repository->getSubversionBaseURI($at_rev)); } catch (CommandException $ex) { $stderr = $ex->getStdErr(); if (preg_match('/(path|File) not found/', $stderr)) { // We've gone all the way back through history and this path was not // affected by earlier commits. break; } throw $ex; } $xml = phutil_utf8ize($xml); $log = new SimpleXMLElement($xml); foreach ($log->logentry as $entry) { $identifier = (int)$entry['revision']; $epoch = (int)strtotime((string)$entry->date[0]); $refs[$identifier] = id(new PhabricatorRepositoryCommitRef()) ->setIdentifier($identifier) ->setEpoch($epoch) ->setCanCloseImmediately(true); if ($upper_bound === null) { $upper_bound = $identifier; } else { $upper_bound = min($upper_bound, $identifier); } } // Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially // import large repositories fairly quickly, while pulling only as much // data as we need in the common case (when we've already imported the // repository and are just grabbing one commit at a time). $limit = min($limit * 2, 256); } while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound)); krsort($refs); while ($refs && $this->isKnownCommit(last($refs)->getIdentifier())) { array_pop($refs); } $refs = array_reverse($refs); $this->didDiscoverRefs($refs); return $refs; } private function verifySubversionRoot(PhabricatorRepository $repository) { list($xml) = $repository->execxRemoteCommand( 'info --xml %s', $repository->getSubversionPathURI()); $xml = phutil_utf8ize($xml); $xml = new SimpleXMLElement($xml); $remote_root = (string)($xml->entry[0]->repository[0]->root[0]); $expect_root = $repository->getSubversionPathURI(); $normal_type_svn = PhabricatorRepositoryURINormalizer::TYPE_SVN; $remote_normal = id(new PhabricatorRepositoryURINormalizer( $normal_type_svn, $remote_root))->getNormalizedPath(); $expect_normal = id(new PhabricatorRepositoryURINormalizer( $normal_type_svn, $expect_root))->getNormalizedPath(); if ($remote_normal != $expect_normal) { throw new Exception( pht( 'Repository "%s" does not have a correctly configured remote URI. '. 'The remote URI for a Subversion repository MUST point at the '. 'repository root. The root for this repository is "%s", but the '. 'configured URI is "%s". To resolve this error, set the remote URI '. 'to point at the repository root. If you want to import only part '. 'of a Subversion repository, use the "Import Only" option.', $repository->getDisplayName(), $remote_root, $expect_root)); } } /* -( Discovering Mercurial Repositories )--------------------------------- */ /** * @task hg */ private function discoverMercurialCommits() { $repository = $this->getRepository(); $branches = id(new DiffusionLowLevelMercurialBranchesQuery()) ->setRepository($repository) ->execute(); $this->fillCommitCache(mpull($branches, 'getCommitIdentifier')); $refs = array(); foreach ($branches as $branch) { // NOTE: Mercurial branches may have multiple heads, so the names may // not be unique. $name = $branch->getShortName(); $commit = $branch->getCommitIdentifier(); $this->log(pht('Examining branch "%s" head "%s".', $name, $commit)); if (!$repository->shouldTrackBranch($name)) { $this->log(pht('Skipping, branch is untracked.')); continue; } if ($this->isKnownCommit($commit)) { $this->log(pht('Skipping, this head is a known commit.')); continue; } $this->log(pht('Looking for new commits.')); $branch_refs = $this->discoverStreamAncestry( new PhabricatorMercurialGraphStream($repository, $commit), $commit, $close_immediately = true); $this->didDiscoverRefs($branch_refs); $refs[] = $branch_refs; } return array_mergev($refs); } /* -( Internals )---------------------------------------------------------- */ private function discoverStreamAncestry( PhabricatorRepositoryGraphStream $stream, $commit, $close_immediately) { $discover = array($commit); $graph = array(); $seen = array(); // Find all the reachable, undiscovered commits. Build a graph of the // edges. while ($discover) { $target = array_pop($discover); if (empty($graph[$target])) { $graph[$target] = array(); } $parents = $stream->getParents($target); foreach ($parents as $parent) { if ($this->isKnownCommit($parent)) { continue; } $graph[$target][$parent] = true; if (empty($seen[$parent])) { $seen[$parent] = true; $discover[] = $parent; } } } // Now, sort them topographically. $commits = $this->reduceGraph($graph); $refs = array(); foreach ($commits as $commit) { $refs[] = id(new PhabricatorRepositoryCommitRef()) ->setIdentifier($commit) ->setEpoch($stream->getCommitDate($commit)) ->setCanCloseImmediately($close_immediately) ->setParents($stream->getParents($commit)); } return $refs; } private function reduceGraph(array $edges) { foreach ($edges as $commit => $parents) { $edges[$commit] = array_keys($parents); } $graph = new PhutilDirectedScalarGraph(); $graph->addNodes($edges); $commits = $graph->getTopographicallySortedNodes(); // NOTE: We want the most ancestral nodes first, so we need to reverse the // list we get out of AbstractDirectedGraph. $commits = array_reverse($commits); return $commits; } private function isKnownCommit($identifier) { if (isset($this->commitCache[$identifier])) { return true; } if (isset($this->workingSet[$identifier])) { return true; } if ($this->repairMode) { // In repair mode, rediscover the entire repository, ignoring the // database state. We can hit the local cache above, but if we miss it // stop the script from going to the database cache. return false; } $this->fillCommitCache(array($identifier)); return isset($this->commitCache[$identifier]); } private function fillCommitCache(array $identifiers) { if (!$identifiers) { return; } $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( 'repositoryID = %d AND commitIdentifier IN (%Ls)', $this->getRepository()->getID(), $identifiers); foreach ($commits as $commit) { $this->commitCache[$commit->getCommitIdentifier()] = true; } while (count($this->commitCache) > self::MAX_COMMIT_CACHE_SIZE) { array_shift($this->commitCache); } } /** * Sort branches so we process closeable branches first. This makes the * whole import process a little cheaper, since we can close these commits * the first time through rather than catching them in the refs step. * * @task internal * - * @param list List of branch heads. - * @return list Sorted list of branch heads. + * @param list List of refs. + * @return list Sorted list of refs. */ - private function sortBranches(array $branches) { + private function sortRefs(array $refs) { $repository = $this->getRepository(); - $head_branches = array(); - $tail_branches = array(); - foreach ($branches as $branch) { - $name = $branch->getShortName(); - - if ($repository->shouldAutocloseBranch($name)) { - $head_branches[] = $branch; + $head_refs = array(); + $tail_refs = array(); + foreach ($refs as $ref) { + if ($repository->shouldAutocloseRef($ref)) { + $head_refs[] = $ref; } else { - $tail_branches[] = $branch; + $tail_refs[] = $ref; } } - return array_merge($head_branches, $tail_branches); + return array_merge($head_refs, $tail_refs); } private function recordCommit( PhabricatorRepository $repository, $commit_identifier, $epoch, $close_immediately, array $parents) { $commit = new PhabricatorRepositoryCommit(); $commit->setRepositoryID($repository->getID()); $commit->setCommitIdentifier($commit_identifier); $commit->setEpoch($epoch); if ($close_immediately) { $commit->setImportStatus(PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE); } $data = new PhabricatorRepositoryCommitData(); $conn_w = $repository->establishConnection('w'); try { // If this commit has parents, look up their IDs. The parent commits // should always exist already. $parent_ids = array(); if ($parents) { $parent_rows = queryfx_all( $conn_w, 'SELECT id, commitIdentifier FROM %T WHERE commitIdentifier IN (%Ls) AND repositoryID = %d', $commit->getTableName(), $parents, $repository->getID()); $parent_map = ipull($parent_rows, 'id', 'commitIdentifier'); foreach ($parents as $parent) { if (empty($parent_map[$parent])) { throw new Exception( pht('Unable to identify parent "%s"!', $parent)); } $parent_ids[] = $parent_map[$parent]; } } else { // Write an explicit 0 so we can distinguish between "really no // parents" and "data not available". if (!$repository->isSVN()) { $parent_ids = array(0); } } $commit->openTransaction(); $commit->save(); $data->setCommitID($commit->getID()); $data->save(); foreach ($parent_ids as $parent_id) { queryfx( $conn_w, 'INSERT IGNORE INTO %T (childCommitID, parentCommitID) VALUES (%d, %d)', PhabricatorRepository::TABLE_PARENTS, $commit->getID(), $parent_id); } $commit->saveTransaction(); $this->insertTask($repository, $commit); queryfx( $conn_w, 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) VALUES (%d, 1, %d, %d) ON DUPLICATE KEY UPDATE size = size + 1, lastCommitID = IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', PhabricatorRepository::TABLE_SUMMARY, $repository->getID(), $commit->getID(), $epoch); if ($this->repairMode) { // Normally, the query should throw a duplicate key exception. If we // reach this in repair mode, we've actually performed a repair. $this->log(pht('Repaired commit "%s".', $commit_identifier)); } PhutilEventEngine::dispatchEvent( new PhabricatorEvent( PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT, array( 'repository' => $repository, 'commit' => $commit, ))); } catch (AphrontDuplicateKeyQueryException $ex) { $commit->killTransaction(); // Ignore. This can happen because we discover the same new commit // more than once when looking at history, or because of races or // data inconsistency or cosmic radiation; in any case, we're still // in a good state if we ignore the failure. } } private function didDiscoverRefs(array $refs) { foreach ($refs as $ref) { $this->workingSet[$ref->getIdentifier()] = true; } } private function insertTask( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit, $data = array()) { $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; break; default: throw new Exception(pht("Unknown repository type '%s'!", $vcs)); } $data['commitID'] = $commit->getID(); PhabricatorWorker::scheduleTask($class, $data); } private function isInitialImport(array $refs) { $commit_count = count($refs); if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) { // If we fetched a small number of commits, assume it's an initial // commit or a stack of a few initial commits. return false; } $viewer = $this->getViewer(); $repository = $this->getRepository(); $any_commits = id(new DiffusionCommitQuery()) ->setViewer($viewer) ->withRepository($repository) ->setLimit(1) ->execute(); if ($any_commits) { // If the repository already has commits, this isn't an import. return false; } return true; } private function getObservedVersion(PhabricatorRepository $repository) { if ($repository->isHosted()) { return null; } if ($repository->isGit()) { return $this->getGitObservedVersion($repository); } return null; } private function getGitObservedVersion(PhabricatorRepository $repository) { $refs = id(new DiffusionLowLevelGitRefQuery()) ->setRepository($repository) ->execute(); if (!$refs) { return null; } // In Git, the observed version is the most recently discovered commit // at any repository HEAD. It's possible for this to regress temporarily // if a branch is pushed and then deleted. This is acceptable because it // doesn't do anything meaningfully bad and will fix itself on the next // push. $ref_identifiers = mpull($refs, 'getCommitIdentifier'); $ref_identifiers = array_fuse($ref_identifiers); $version = queryfx_one( $repository->establishConnection('w'), 'SELECT MAX(id) version FROM %T WHERE repositoryID = %d AND commitIdentifier IN (%Ls)', id(new PhabricatorRepositoryCommit())->getTableName(), $repository->getID(), $ref_identifiers); if (!$version) { return null; } return (int)$version['version']; } } diff --git a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php index 3d7dce90fa..7e54bf4620 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php @@ -1,593 +1,593 @@ getRepository(); $lock = $this->newRepositoryLock($repository, 'repo.pull', true); try { $lock->lock(); } catch (PhutilLockException $ex) { throw new DiffusionDaemonLockException( pht( 'Another process is currently updating repository "%s", '. 'skipping pull.', $repository->getDisplayName())); } try { $result = $this->pullRepositoryWithLock(); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); return $result; } private function pullRepositoryWithLock() { $repository = $this->getRepository(); $viewer = PhabricatorUser::getOmnipotentUser(); $is_hg = false; $is_git = false; $is_svn = false; $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // We never pull a local copy of non-hosted Subversion repositories. if (!$repository->isHosted()) { $this->skipPull( pht( 'Repository "%s" is a non-hosted Subversion repository, which '. 'does not require a local working copy to be pulled.', $repository->getDisplayName())); return; } $is_svn = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $is_git = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $is_hg = true; break; default: $this->abortPull(pht('Unknown VCS "%s"!', $vcs)); break; } $local_path = $repository->getLocalPath(); if ($local_path === null) { $this->abortPull( pht( 'No local path is configured for repository "%s".', $repository->getDisplayName())); } try { $dirname = dirname($local_path); if (!Filesystem::pathExists($dirname)) { Filesystem::createDirectory($dirname, 0755, $recursive = true); } if (!Filesystem::pathExists($local_path)) { $this->logPull( pht( 'Creating a new working copy for repository "%s".', $repository->getDisplayName())); if ($is_git) { $this->executeGitCreate(); } else if ($is_hg) { $this->executeMercurialCreate(); } else { $this->executeSubversionCreate(); } } id(new DiffusionRepositoryClusterEngine()) ->setViewer($viewer) ->setRepository($repository) ->synchronizeWorkingCopyBeforeRead(); if (!$repository->isHosted()) { $this->logPull( pht( 'Updating the working copy for repository "%s".', $repository->getDisplayName())); if ($is_git) { $this->verifyGitOrigin($repository); $this->executeGitUpdate(); } else if ($is_hg) { $this->executeMercurialUpdate(); } } if ($repository->isHosted()) { if ($is_git) { $this->installGitHook(); } else if ($is_svn) { $this->installSubversionHook(); } else if ($is_hg) { $this->installMercurialHook(); } foreach ($repository->getHookDirectories() as $directory) { $this->installHookDirectory($directory); } } } catch (Exception $ex) { $this->abortPull( pht( "Pull of '%s' failed: %s", $repository->getDisplayName(), $ex->getMessage()), $ex); } $this->donePull(); return $this; } private function skipPull($message) { $this->log('%s', $message); $this->donePull(); } private function abortPull($message, Exception $ex = null) { $code_error = PhabricatorRepositoryStatusMessage::CODE_ERROR; $this->updateRepositoryInitStatus($code_error, $message); if ($ex) { throw $ex; } else { throw new Exception($message); } } private function logPull($message) { $code_working = PhabricatorRepositoryStatusMessage::CODE_WORKING; $this->updateRepositoryInitStatus($code_working, $message); $this->log('%s', $message); } private function donePull() { $code_okay = PhabricatorRepositoryStatusMessage::CODE_OKAY; $this->updateRepositoryInitStatus($code_okay); } private function updateRepositoryInitStatus($code, $message = null) { $this->getRepository()->writeStatusMessage( PhabricatorRepositoryStatusMessage::TYPE_INIT, $code, array( 'message' => $message, )); } private function installHook($path) { $this->log('%s', pht('Installing commit hook to "%s"...', $path)); $repository = $this->getRepository(); $identifier = $this->getHookContextIdentifier($repository); $root = dirname(phutil_get_library_root('phabricator')); $bin = $root.'/bin/commit-hook'; $full_php_path = Filesystem::resolveBinary('php'); $cmd = csprintf( 'exec %s -f %s -- %s "$@"', $full_php_path, $bin, $identifier); $hook = "#!/bin/sh\nexport TERM=dumb\n{$cmd}\n"; Filesystem::writeFile($path, $hook); Filesystem::changePermissions($path, 0755); } private function installHookDirectory($path) { $readme = pht( "To add custom hook scripts to this repository, add them to this ". "directory.\n\nPhabricator will run any executables in this directory ". "after running its own checks, as though they were normal hook ". "scripts."); Filesystem::createDirectory($path, 0755); Filesystem::writeFile($path.'/README', $readme); } private function getHookContextIdentifier(PhabricatorRepository $repository) { $identifier = $repository->getPHID(); $instance = PhabricatorEnv::getEnvConfig('cluster.instance'); if (strlen($instance)) { $identifier = "{$identifier}:{$instance}"; } return $identifier; } /* -( Pulling Git Working Copies )----------------------------------------- */ /** * @task git */ private function executeGitCreate() { $repository = $this->getRepository(); $path = rtrim($repository->getLocalPath(), '/'); if ($repository->isHosted()) { $repository->execxRemoteCommand( 'init --bare -- %s', $path); } else { $repository->execxRemoteCommand( 'clone --bare -- %P %s', $repository->getRemoteURIEnvelope(), $path); } } /** * @task git */ private function executeGitUpdate() { $repository = $this->getRepository(); list($err, $stdout) = $repository->execLocalCommand( 'rev-parse --show-toplevel'); $message = null; $path = $repository->getLocalPath(); if ($err) { // Try to raise a more tailored error message in the more common case // of the user creating an empty directory. (We could try to remove it, // but might not be able to, and it's much simpler to raise a good // message than try to navigate those waters.) if (is_dir($path)) { $files = Filesystem::listDirectory($path, $include_hidden = true); if (!$files) { $message = pht( "Expected to find a git repository at '%s', but there ". "is an empty directory there. Remove the directory: the daemon ". "will run '%s' for you.", $path, 'git clone'); } else { $message = pht( "Expected to find a git repository at '%s', but there is ". "a non-repository directory (with other stuff in it) there. Move ". "or remove this directory (or reconfigure the repository to use a ". "different directory), and then either clone a repository ". "yourself or let the daemon do it.", $path); } } else if (is_file($path)) { $message = pht( "Expected to find a git repository at '%s', but there is a ". "file there instead. Remove it and let the daemon clone a ". "repository for you.", $path); } else { $message = pht( "Expected to find a git repository at '%s', but did not.", $path); } } else { $repo_path = rtrim($stdout, "\n"); if (empty($repo_path)) { // This can mean one of two things: we're in a bare repository, or // we're inside a git repository inside another git repository. Since // the first is dramatically more likely now that we perform bare // clones and I don't have a great way to test for the latter, assume // we're OK. } else if (!Filesystem::pathsAreEquivalent($repo_path, $path)) { $err = true; $message = pht( "Expected to find repo at '%s', but the actual git repository root ". "for this directory is '%s'. Something is misconfigured. ". "The repository's 'Local Path' should be set to some place where ". "the daemon can check out a working copy, ". "and should not be inside another git repository.", $path, $repo_path); } } if ($err && $repository->canDestroyWorkingCopy()) { phlog( pht( "Repository working copy at '%s' failed sanity check; ". "destroying and re-cloning. %s", $path, $message)); Filesystem::remove($path); $this->executeGitCreate(); } else if ($err) { throw new Exception($message); } $retry = false; do { // This is a local command, but needs credentials. if ($repository->isWorkingCopyBare()) { // For bare working copies, we need this magic incantation. $future = $repository->getRemoteCommandFuture( 'fetch origin %s --prune', - '+refs/heads/*:refs/heads/*'); + '+refs/*:refs/*'); } else { $future = $repository->getRemoteCommandFuture( 'fetch --all --prune'); } $future->setCWD($path); list($err, $stdout, $stderr) = $future->resolve(); if ($err && !$retry && $repository->canDestroyWorkingCopy()) { $retry = true; // Fix remote origin url if it doesn't match our configuration $origin_url = $repository->execLocalCommand( 'config --get remote.origin.url'); $remote_uri = $repository->getRemoteURIEnvelope(); if ($origin_url != $remote_uri->openEnvelope()) { $repository->execLocalCommand( 'remote set-url origin %P', $remote_uri); } } else if ($err) { throw new CommandException( pht('Failed to fetch changes!'), $future->getCommand(), $err, $stdout, $stderr); } else { $retry = false; } } while ($retry); } /** * @task git */ private function installGitHook() { $repository = $this->getRepository(); $root = $repository->getLocalPath(); if ($repository->isWorkingCopyBare()) { $path = '/hooks/pre-receive'; } else { $path = '/.git/hooks/pre-receive'; } $this->installHook($root.$path); } /* -( Pulling Mercurial Working Copies )----------------------------------- */ /** * @task hg */ private function executeMercurialCreate() { $repository = $this->getRepository(); $path = rtrim($repository->getLocalPath(), '/'); if ($repository->isHosted()) { $repository->execxRemoteCommand( 'init -- %s', $path); } else { $remote = $repository->getRemoteURIEnvelope(); // NOTE: Mercurial prior to 3.2.4 has an severe command injection // vulnerability. See: // On vulnerable versions of Mercurial, we refuse to clone remotes which // contain characters which may be interpreted by the shell. $hg_version = PhabricatorRepositoryVersion::getMercurialVersion(); $is_vulnerable = version_compare($hg_version, '3.2.4', '<'); if ($is_vulnerable) { $cleartext = $remote->openEnvelope(); // The use of "%R" here is an attempt to limit collateral damage // for normal URIs because it isn't clear how long this vulnerability // has been around for. $escaped = csprintf('%R', $cleartext); if ((string)$escaped !== (string)$cleartext) { throw new Exception( pht( 'You have an old version of Mercurial (%s) which has a severe '. 'command injection security vulnerability. The remote URI for '. 'this repository (%s) is potentially unsafe. Upgrade Mercurial '. 'to at least 3.2.4 to clone it.', $hg_version, $repository->getMonogram())); } } try { $repository->execxRemoteCommand( 'clone --noupdate -- %P %s', $remote, $path); } catch (Exception $ex) { $message = $ex->getMessage(); $message = $this->censorMercurialErrorMessage($message); throw new Exception($message); } } } /** * @task hg */ private function executeMercurialUpdate() { $repository = $this->getRepository(); $path = $repository->getLocalPath(); // This is a local command, but needs credentials. $remote = $repository->getRemoteURIEnvelope(); $future = $repository->getRemoteCommandFuture('pull -u -- %P', $remote); $future->setCWD($path); try { $future->resolvex(); } catch (CommandException $ex) { $err = $ex->getError(); $stdout = $ex->getStdOut(); // NOTE: Between versions 2.1 and 2.1.1, Mercurial changed the behavior // of "hg pull" to return 1 in case of a successful pull with no changes. // This behavior has been reverted, but users who updated between Feb 1, // 2012 and Mar 1, 2012 will have the erroring version. Do a dumb test // against stdout to check for this possibility. // See: https://github.com/phacility/phabricator/issues/101/ // NOTE: Mercurial has translated versions, which translate this error // string. In a translated version, the string will be something else, // like "aucun changement trouve". There didn't seem to be an easy way // to handle this (there are hard ways but this is not a common problem // and only creates log spam, not application failures). Assume English. // TODO: Remove this once we're far enough in the future that deployment // of 2.1 is exceedingly rare? if ($err == 1 && preg_match('/no changes found/', $stdout)) { return; } else { $message = $ex->getMessage(); $message = $this->censorMercurialErrorMessage($message); throw new Exception($message); } } } /** * Censor response bodies from Mercurial error messages. * * When Mercurial attempts to clone an HTTP repository but does not * receive a response it expects, it emits the response body in the * command output. * * This represents a potential SSRF issue, because an attacker with * permission to create repositories can create one which points at the * remote URI for some local service, then read the response from the * error message. To prevent this, censor response bodies out of error * messages. * * @param string Uncensored Mercurial command output. * @return string Censored Mercurial command output. */ private function censorMercurialErrorMessage($message) { return preg_replace( '/^---%<---.*/sm', pht('')."\n", $message); } /** * @task hg */ private function installMercurialHook() { $repository = $this->getRepository(); $path = $repository->getLocalPath().'/.hg/hgrc'; $identifier = $this->getHookContextIdentifier($repository); $root = dirname(phutil_get_library_root('phabricator')); $bin = $root.'/bin/commit-hook'; $data = array(); $data[] = '[hooks]'; // This hook handles normal pushes. $data[] = csprintf( 'pretxnchangegroup.phabricator = TERM=dumb %s %s %s', $bin, $identifier, 'pretxnchangegroup'); // This one handles creating bookmarks. $data[] = csprintf( 'prepushkey.phabricator = TERM=dumb %s %s %s', $bin, $identifier, 'prepushkey'); $data[] = null; $data = implode("\n", $data); $this->log('%s', pht('Installing commit hook config to "%s"...', $path)); Filesystem::writeFile($path, $data); } /* -( Pulling Subversion Working Copies )---------------------------------- */ /** * @task svn */ private function executeSubversionCreate() { $repository = $this->getRepository(); $path = rtrim($repository->getLocalPath(), '/'); execx('svnadmin create -- %s', $path); } /** * @task svn */ private function installSubversionHook() { $repository = $this->getRepository(); $root = $repository->getLocalPath(); $path = '/hooks/pre-commit'; $this->installHook($root.$path); } } diff --git a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php index da89db75e0..f486eb442c 100644 --- a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php +++ b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php @@ -1,500 +1,487 @@ newRefs = array(); $this->deadRefs = array(); $this->closeCommits = array(); $repository = $this->getRepository(); $branches_may_close = false; $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // No meaningful refs of any type in Subversion. - $branches = array(); - $bookmarks = array(); - $tags = array(); + $maps = array(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $branches = $this->loadMercurialBranchPositions($repository); $bookmarks = $this->loadMercurialBookmarkPositions($repository); - $tags = array(); + $maps = array( + PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches, + PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks, + ); + $branches_may_close = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $branches = $this->loadGitBranchPositions($repository); - $bookmarks = array(); - $tags = $this->loadGitTagPositions($repository); + $maps = $this->loadGitRefPositions($repository); break; default: throw new Exception(pht('Unknown VCS "%s"!', $vcs)); } - $maps = array( - PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches, - PhabricatorRepositoryRefCursor::TYPE_TAG => $tags, - PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks, + // Fill in any missing types with empty lists. + $maps = $maps + array( + PhabricatorRepositoryRefCursor::TYPE_BRANCH => array(), + PhabricatorRepositoryRefCursor::TYPE_TAG => array(), + PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => array(), + PhabricatorRepositoryRefCursor::TYPE_REF => array(), ); $all_cursors = id(new PhabricatorRepositoryRefCursorQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withRepositoryPHIDs(array($repository->getPHID())) ->execute(); $cursor_groups = mgroup($all_cursors, 'getRefType'); $this->hasNoCursors = (!$all_cursors); // Find all the heads of closing refs. $all_closing_heads = array(); foreach ($all_cursors as $cursor) { if ($this->shouldCloseRef($cursor->getRefType(), $cursor->getRefName())) { $all_closing_heads[] = $cursor->getCommitIdentifier(); } } $all_closing_heads = array_unique($all_closing_heads); $all_closing_heads = $this->removeMissingCommits($all_closing_heads); foreach ($maps as $type => $refs) { $cursor_group = idx($cursor_groups, $type, array()); $this->updateCursors($cursor_group, $refs, $type, $all_closing_heads); } if ($this->closeCommits) { $this->setCloseFlagOnCommits($this->closeCommits); } if ($this->newRefs || $this->deadRefs) { $repository->openTransaction(); foreach ($this->newRefs as $ref) { $ref->save(); } foreach ($this->deadRefs as $ref) { $ref->delete(); } $repository->saveTransaction(); $this->newRefs = array(); $this->deadRefs = array(); } + $branches = $maps[PhabricatorRepositoryRefCursor::TYPE_BRANCH]; if ($branches && $branches_may_close) { $this->updateBranchStates($repository, $branches); } } private function updateBranchStates( PhabricatorRepository $repository, array $branches) { assert_instances_of($branches, 'DiffusionRepositoryRef'); $all_cursors = id(new PhabricatorRepositoryRefCursorQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withRepositoryPHIDs(array($repository->getPHID())) ->execute(); $state_map = array(); $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH; foreach ($all_cursors as $cursor) { if ($cursor->getRefType() !== $type_branch) { continue; } $raw_name = $cursor->getRefNameRaw(); $hash = $cursor->getCommitIdentifier(); $state_map[$raw_name][$hash] = $cursor; } foreach ($branches as $branch) { $cursor = idx($state_map, $branch->getShortName(), array()); $cursor = idx($cursor, $branch->getCommitIdentifier()); if (!$cursor) { continue; } $fields = $branch->getRawFields(); $cursor_state = (bool)$cursor->getIsClosed(); $branch_state = (bool)idx($fields, 'closed'); if ($cursor_state != $branch_state) { $cursor->setIsClosed((int)$branch_state)->save(); } } } private function markRefNew(PhabricatorRepositoryRefCursor $cursor) { $this->newRefs[] = $cursor; return $this; } private function markRefDead(PhabricatorRepositoryRefCursor $cursor) { $this->deadRefs[] = $cursor; return $this; } private function markCloseCommits(array $identifiers) { foreach ($identifiers as $identifier) { $this->closeCommits[$identifier] = $identifier; } return $this; } /** * Remove commits which no longer exist in the repository from a list. * * After a force push and garbage collection, we may have branch cursors which * point at commits which no longer exist. This can make commands issued later * fail. See T5839 for discussion. * * @param list List of commit identifiers. * @return list List with nonexistent identifiers removed. */ private function removeMissingCommits(array $identifiers) { if (!$identifiers) { return array(); } $resolved = id(new DiffusionLowLevelResolveRefsQuery()) ->setRepository($this->getRepository()) ->withRefs($identifiers) ->execute(); foreach ($identifiers as $key => $identifier) { if (empty($resolved[$identifier])) { unset($identifiers[$key]); } } return $identifiers; } private function updateCursors( array $cursors, array $new_refs, $ref_type, array $all_closing_heads) { $repository = $this->getRepository(); // NOTE: Mercurial branches may have multiple branch heads; this logic // is complex primarily to account for that. // Group all the cursors by their ref name, like "master". Since Mercurial // branches may have multiple heads, there could be several cursors with // the same name. $cursor_groups = mgroup($cursors, 'getRefNameRaw'); // Group all the new ref values by their name. As above, these groups may // have multiple members in Mercurial. $ref_groups = mgroup($new_refs, 'getShortName'); foreach ($ref_groups as $name => $refs) { $new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier'); $ref_cursors = idx($cursor_groups, $name, array()); $old_commits = mpull($ref_cursors, null, 'getCommitIdentifier'); // We're going to delete all the cursors pointing at commits which are // no longer associated with the refs. This primarily makes the Mercurial // multiple head case easier, and means that when we update a ref we // delete the old one and write a new one. foreach ($ref_cursors as $cursor) { if (isset($new_commits[$cursor->getCommitIdentifier()])) { // This ref previously pointed at this commit, and still does. $this->log( pht( 'Ref %s "%s" still points at %s.', $ref_type, $name, $cursor->getCommitIdentifier())); } else { // This ref previously pointed at this commit, but no longer does. $this->log( pht( 'Ref %s "%s" no longer points at %s.', $ref_type, $name, $cursor->getCommitIdentifier())); // Nuke the obsolete cursor. $this->markRefDead($cursor); } } // Now, we're going to insert new cursors for all the commits which are // associated with this ref that don't currently have cursors. $added_commits = array_diff_key($new_commits, $old_commits); foreach ($added_commits as $identifier) { $this->log( pht( 'Ref %s "%s" now points at %s.', $ref_type, $name, $identifier)); $this->markRefNew( id(new PhabricatorRepositoryRefCursor()) ->setRepositoryPHID($repository->getPHID()) ->setRefType($ref_type) ->setRefName($name) ->setCommitIdentifier($identifier)); } if ($this->shouldCloseRef($ref_type, $name)) { foreach ($added_commits as $identifier) { $new_identifiers = $this->loadNewCommitIdentifiers( $identifier, $all_closing_heads); $this->markCloseCommits($new_identifiers); } } } // Find any cursors for refs which no longer exist. This happens when a // branch, tag or bookmark is deleted. foreach ($cursor_groups as $name => $cursor_group) { if (idx($ref_groups, $name) === null) { foreach ($cursor_group as $cursor) { $this->log( pht( 'Ref %s "%s" no longer exists.', $cursor->getRefType(), $cursor->getRefName())); $this->markRefDead($cursor); } } } } private function shouldCloseRef($ref_type, $ref_name) { if ($ref_type !== PhabricatorRepositoryRefCursor::TYPE_BRANCH) { return false; } if ($this->hasNoCursors) { // If we don't have any cursors, don't close things. Particularly, this // corresponds to the case where you've just updated to this code on an // existing repository: we don't want to requeue message steps for every // commit on a closeable ref. return false; } return $this->getRepository()->shouldAutocloseBranch($ref_name); } /** * Find all ancestors of a new closing branch head which are not ancestors * of any old closing branch head. */ private function loadNewCommitIdentifiers( $new_head, array $all_closing_heads) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: if ($all_closing_heads) { $parts = array(); foreach ($all_closing_heads as $head) { $parts[] = hgsprintf('%s', $head); } // See T5896. Mercurial can not parse an "X or Y or ..." rev list // with more than about 300 items, because it exceeds the maximum // allowed recursion depth. Split all the heads into chunks of // 256, and build a query like this: // // ((1 or 2 or ... or 255) or (256 or 257 or ... 511)) // // If we have more than 65535 heads, we'll do that again: // // (((1 or ...) or ...) or ((65536 or ...) or ...)) $chunk_size = 256; while (count($parts) > $chunk_size) { $chunks = array_chunk($parts, $chunk_size); foreach ($chunks as $key => $chunk) { $chunks[$key] = '('.implode(' or ', $chunk).')'; } $parts = array_values($chunks); } $parts = '('.implode(' or ', $parts).')'; list($stdout) = $this->getRepository()->execxLocalCommand( 'log --template %s --rev %s', '{node}\n', hgsprintf('%s', $new_head).' - '.$parts); } else { list($stdout) = $this->getRepository()->execxLocalCommand( 'log --template %s --rev %s', '{node}\n', hgsprintf('%s', $new_head)); } $stdout = trim($stdout); if (!strlen($stdout)) { return array(); } return phutil_split_lines($stdout, $retain_newlines = false); case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: if ($all_closing_heads) { list($stdout) = $this->getRepository()->execxLocalCommand( 'log --format=%s %s --not %Ls', '%H', $new_head, $all_closing_heads); } else { list($stdout) = $this->getRepository()->execxLocalCommand( 'log --format=%s %s', '%H', $new_head); } $stdout = trim($stdout); if (!strlen($stdout)) { return array(); } return phutil_split_lines($stdout, $retain_newlines = false); default: throw new Exception(pht('Unsupported VCS "%s"!', $vcs)); } } /** * Mark a list of commits as closeable, and queue workers for those commits * which don't already have the flag. */ private function setCloseFlagOnCommits(array $identifiers) { $repository = $this->getRepository(); $commit_table = new PhabricatorRepositoryCommit(); $conn_w = $commit_table->establishConnection('w'); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; break; default: throw new Exception(pht("Unknown repository type '%s'!", $vcs)); } $all_commits = queryfx_all( $conn_w, 'SELECT id, commitIdentifier, importStatus FROM %T WHERE repositoryID = %d AND commitIdentifier IN (%Ls)', $commit_table->getTableName(), $repository->getID(), $identifiers); $closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE; $all_commits = ipull($all_commits, null, 'commitIdentifier'); foreach ($identifiers as $identifier) { $row = idx($all_commits, $identifier); if (!$row) { throw new Exception( pht( 'Commit "%s" has not been discovered yet! Run discovery before '. 'updating refs.', $identifier)); } if (!($row['importStatus'] & $closeable_flag)) { queryfx( $conn_w, 'UPDATE %T SET importStatus = (importStatus | %d) WHERE id = %d', $commit_table->getTableName(), $closeable_flag, $row['id']); $data = array( 'commitID' => $row['id'], 'only' => true, ); PhabricatorWorker::scheduleTask($class, $data); } } return $this; } /* -( Updating Git Refs )-------------------------------------------------- */ /** * @task git */ - private function loadGitBranchPositions(PhabricatorRepository $repository) { - return id(new DiffusionLowLevelGitRefQuery()) + private function loadGitRefPositions(PhabricatorRepository $repository) { + $refs = id(new DiffusionLowLevelGitRefQuery()) ->setRepository($repository) - ->withRefTypes( - array( - PhabricatorRepositoryRefCursor::TYPE_BRANCH, - )) ->execute(); - } - - /** - * @task git - */ - private function loadGitTagPositions(PhabricatorRepository $repository) { - return id(new DiffusionLowLevelGitRefQuery()) - ->setRepository($repository) - ->withRefTypes( - array( - PhabricatorRepositoryRefCursor::TYPE_TAG, - )) - ->execute(); + return mgroup($refs, 'getRefType'); } /* -( Updating Mercurial Refs )-------------------------------------------- */ /** * @task hg */ private function loadMercurialBranchPositions( PhabricatorRepository $repository) { return id(new DiffusionLowLevelMercurialBranchesQuery()) ->setRepository($repository) ->execute(); } /** * @task hg */ private function loadMercurialBookmarkPositions( PhabricatorRepository $repository) { // TODO: Implement support for Mercurial bookmarks. return array(); } } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index b771066ed6..b560be1964 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1,2401 +1,2417 @@ setViewer($actor) ->withClasses(array('PhabricatorDiffusionApplication')) ->executeOne(); $view_policy = $app->getPolicy(DiffusionDefaultViewCapability::CAPABILITY); $edit_policy = $app->getPolicy(DiffusionDefaultEditCapability::CAPABILITY); $push_policy = $app->getPolicy(DiffusionDefaultPushCapability::CAPABILITY); $repository = id(new PhabricatorRepository()) ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setPushPolicy($push_policy) ->setSpacePHID($actor->getDefaultSpacePHID()); // Put the repository in "Importing" mode until we finish // parsing it. $repository->setDetail('importing', true); return $repository; } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'details' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'sort255', 'callsign' => 'sort32?', 'repositorySlug' => 'sort64?', 'versionControlSystem' => 'text32', 'uuid' => 'text64?', 'pushPolicy' => 'policy', 'credentialPHID' => 'phid?', 'almanacServicePHID' => 'phid?', 'localPath' => 'text128?', ), self::CONFIG_KEY_SCHEMA => array( 'callsign' => array( 'columns' => array('callsign'), 'unique' => true, ), 'key_name' => array( 'columns' => array('name(128)'), ), 'key_vcs' => array( 'columns' => array('versionControlSystem'), ), 'key_slug' => array( 'columns' => array('repositorySlug'), 'unique' => true, ), 'key_local' => array( 'columns' => array('localPath'), 'unique' => true, ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryRepositoryPHIDType::TYPECONST); } public static function getStatusMap() { return array( self::STATUS_ACTIVE => array( 'name' => pht('Active'), 'isTracked' => 1, ), self::STATUS_INACTIVE => array( 'name' => pht('Inactive'), 'isTracked' => 0, ), ); } public static function getStatusNameMap() { return ipull(self::getStatusMap(), 'name'); } public function getStatus() { if ($this->isTracked()) { return self::STATUS_ACTIVE; } else { return self::STATUS_INACTIVE; } } public function toDictionary() { return array( 'id' => $this->getID(), 'name' => $this->getName(), 'phid' => $this->getPHID(), 'callsign' => $this->getCallsign(), 'monogram' => $this->getMonogram(), 'vcs' => $this->getVersionControlSystem(), 'uri' => PhabricatorEnv::getProductionURI($this->getURI()), 'remoteURI' => (string)$this->getRemoteURI(), 'description' => $this->getDetail('description'), 'isActive' => $this->isTracked(), 'isHosted' => $this->isHosted(), 'isImporting' => $this->isImporting(), 'encoding' => $this->getDefaultTextEncoding(), 'staging' => array( 'supported' => $this->supportsStaging(), 'prefix' => 'phabricator', 'uri' => $this->getStagingURI(), ), ); } public function getDefaultTextEncoding() { return $this->getDetail('encoding', 'UTF-8'); } public function getMonogram() { $callsign = $this->getCallsign(); if (strlen($callsign)) { return "r{$callsign}"; } $id = $this->getID(); return "R{$id}"; } public function getDisplayName() { $slug = $this->getRepositorySlug(); if (strlen($slug)) { return $slug; } return $this->getMonogram(); } public function getAllMonograms() { $monograms = array(); $monograms[] = 'R'.$this->getID(); $callsign = $this->getCallsign(); if (strlen($callsign)) { $monograms[] = 'r'.$callsign; } return $monograms; } public function setLocalPath($path) { // Convert any extra slashes ("//") in the path to a single slash ("/"). $path = preg_replace('(//+)', '/', $path); return parent::setLocalPath($path); } public function getDetail($key, $default = null) { return idx($this->details, $key, $default); } public function getHumanReadableDetail($key, $default = null) { $value = $this->getDetail($key, $default); switch ($key) { case 'branch-filter': case 'close-commits-filter': $value = array_keys($value); $value = implode(', ', $value); break; } return $value; } public function setDetail($key, $value) { $this->details[$key] = $value; return $this; } public function attachCommitCount($count) { $this->commitCount = $count; return $this; } public function getCommitCount() { return $this->assertAttached($this->commitCount); } public function attachMostRecentCommit( PhabricatorRepositoryCommit $commit = null) { $this->mostRecentCommit = $commit; return $this; } public function getMostRecentCommit() { return $this->assertAttached($this->mostRecentCommit); } public function getDiffusionBrowseURIForPath( PhabricatorUser $user, $path, $line = null, $branch = null) { $drequest = DiffusionRequest::newFromDictionary( array( 'user' => $user, 'repository' => $this, 'path' => $path, 'branch' => $branch, )); return $drequest->generateURI( array( 'action' => 'browse', 'line' => $line, )); } public function getSubversionBaseURI($commit = null) { $subpath = $this->getDetail('svn-subpath'); if (!strlen($subpath)) { $subpath = null; } return $this->getSubversionPathURI($subpath, $commit); } public function getSubversionPathURI($path = null, $commit = null) { $vcs = $this->getVersionControlSystem(); if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) { throw new Exception(pht('Not a subversion repository!')); } if ($this->isHosted()) { $uri = 'file://'.$this->getLocalPath(); } else { $uri = $this->getDetail('remote-uri'); } $uri = rtrim($uri, '/'); if (strlen($path)) { $path = rawurlencode($path); $path = str_replace('%2F', '/', $path); $uri = $uri.'/'.ltrim($path, '/'); } if ($path !== null || $commit !== null) { $uri .= '@'; } if ($commit !== null) { $uri .= $commit; } return $uri; } public function attachProjectPHIDs(array $project_phids) { $this->projectPHIDs = $project_phids; return $this; } public function getProjectPHIDs() { return $this->assertAttached($this->projectPHIDs); } /** * Get the name of the directory this repository should clone or checkout * into. For example, if the repository name is "Example Repository", a * reasonable name might be "example-repository". This is used to help users * get reasonable results when cloning repositories, since they generally do * not want to clone into directories called "X/" or "Example Repository/". * * @return string */ public function getCloneName() { $name = $this->getRepositorySlug(); // Make some reasonable effort to produce reasonable default directory // names from repository names. if (!strlen($name)) { $name = $this->getName(); $name = phutil_utf8_strtolower($name); $name = preg_replace('@[/ -:<>]+@', '-', $name); $name = trim($name, '-'); if (!strlen($name)) { $name = $this->getCallsign(); } } return $name; } public static function isValidRepositorySlug($slug) { try { self::assertValidRepositorySlug($slug); return true; } catch (Exception $ex) { return false; } } public static function assertValidRepositorySlug($slug) { if (!strlen($slug)) { throw new Exception( pht( 'The empty string is not a valid repository short name. '. 'Repository short names must be at least one character long.')); } if (strlen($slug) > 64) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must not be longer than 64 characters.', $slug)); } if (preg_match('/[^a-zA-Z0-9._-]/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may only contain letters, numbers, periods, hyphens '. 'and underscores.', $slug)); } if (!preg_match('/^[a-zA-Z0-9]/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must begin with a letter or number.', $slug)); } if (!preg_match('/[a-zA-Z0-9]\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must end with a letter or number.', $slug)); } if (preg_match('/__|--|\\.\\./', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must not contain multiple consecutive underscores, '. 'hyphens, or periods.', $slug)); } if (preg_match('/^[A-Z]+\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may not contain only uppercase letters.', $slug)); } if (preg_match('/^\d+\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may not contain only numbers.', $slug)); } } public static function assertValidCallsign($callsign) { if (!strlen($callsign)) { throw new Exception( pht( 'A repository callsign must be at least one character long.')); } if (strlen($callsign) > 32) { throw new Exception( pht( 'The callsign "%s" is not a valid repository callsign. Callsigns '. 'must be no more than 32 bytes long.', $callsign)); } if (!preg_match('/^[A-Z]+\z/', $callsign)) { throw new Exception( pht( 'The callsign "%s" is not a valid repository callsign. Callsigns '. 'may only contain UPPERCASE letters.', $callsign)); } } /* -( Remote Command Execution )------------------------------------------- */ public function execRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args)->resolve(); } public function execxRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args)->resolvex(); } public function getRemoteCommandFuture($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args); } public function passthruRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandPassthru($args)->execute(); } private function newRemoteCommandFuture(array $argv) { return $this->newRemoteCommandEngine($argv) ->newFuture(); } private function newRemoteCommandPassthru(array $argv) { return $this->newRemoteCommandEngine($argv) ->setPassthru(true) ->newFuture(); } private function newRemoteCommandEngine(array $argv) { return DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->setCredentialPHID($this->getCredentialPHID()) ->setURI($this->getRemoteURIObject()); } /* -( Local Command Execution )-------------------------------------------- */ public function execLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args)->resolve(); } public function execxLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args)->resolvex(); } public function getLocalCommandFuture($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args); } public function passthruLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandPassthru($args)->execute(); } private function newLocalCommandFuture(array $argv) { $this->assertLocalExists(); $future = DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); } return $future; } private function newLocalCommandPassthru(array $argv) { $this->assertLocalExists(); $future = DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->setPassthru(true) ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); } return $future; } public function getURI() { $callsign = $this->getCallsign(); if (strlen($callsign)) { return "/diffusion/{$callsign}/"; } $id = $this->getID(); return "/diffusion/{$id}/"; } public function getPathURI($path) { return $this->getURI().$path; } public function getCommitURI($identifier) { $callsign = $this->getCallsign(); if (strlen($callsign)) { return "/r{$callsign}{$identifier}"; } $id = $this->getID(); return "/R{$id}:{$identifier}"; } public static function parseRepositoryServicePath($request_path) { // NOTE: In Mercurial over SSH, the path will begin without a leading "/", // so we're matching it optionally. $patterns = array( '(^'. '(?P/?diffusion/(?P[A-Z]+|[0-9]\d*))'. '(?P(?:/.*)?)'. '\z)', ); $identifier = null; foreach ($patterns as $pattern) { $matches = null; if (!preg_match($pattern, $request_path, $matches)) { continue; } $identifier = $matches['identifier']; $base = $matches['base']; $path = $matches['path']; break; } if ($identifier === null) { return null; } return array( 'identifier' => $identifier, 'base' => $base, 'path' => $path, ); } public function getCanonicalPath($request_path) { $standard_pattern = '(^'. '(?P/diffusion/)'. '(?P[^/]+)'. '(?P(?:/.*)?)'. '\z)'; $matches = null; if (preg_match($standard_pattern, $request_path, $matches)) { $prefix = $matches['prefix']; $callsign = $this->getCallsign(); if ($callsign) { $identifier = $callsign; } else { $identifier = $this->getID(); } $suffix = $matches['suffix']; if (!strlen($suffix)) { $suffix = '/'; } return $prefix.$identifier.$suffix; } $commit_pattern = '(^'. '(?P/)'. '(?P'. '(?:'. 'r(?P[A-Z]+)'. '|'. 'R(?P[1-9]\d*):'. ')'. '(?P[a-f0-9]+)'. ')'. '\z)'; $matches = null; if (preg_match($commit_pattern, $request_path, $matches)) { $commit = $matches['commit']; return $this->getCommitURI($commit); } return null; } public function generateURI(array $params) { $req_branch = false; $req_commit = false; $action = idx($params, 'action'); switch ($action) { case 'history': case 'browse': case 'change': case 'lastmodified': case 'tags': case 'branches': case 'lint': case 'pathtree': case 'refs': break; case 'branch': // NOTE: This does not actually require a branch, and won't have one // in Subversion. Possibly this should be more clear. break; case 'commit': case 'rendering-ref': $req_commit = true; break; default: throw new Exception( pht( 'Action "%s" is not a valid repository URI action.', $action)); } $path = idx($params, 'path'); $branch = idx($params, 'branch'); $commit = idx($params, 'commit'); $line = idx($params, 'line'); if ($req_commit && !strlen($commit)) { throw new Exception( pht( 'Diffusion URI action "%s" requires commit!', $action)); } if ($req_branch && !strlen($branch)) { throw new Exception( pht( 'Diffusion URI action "%s" requires branch!', $action)); } if ($action === 'commit') { return $this->getCommitURI($commit); } $identifier = $this->getID(); $callsign = $this->getCallsign(); if ($callsign !== null) { $identifier = $callsign; } if (strlen($identifier)) { $identifier = phutil_escape_uri_path_component($identifier); } if (strlen($path)) { $path = ltrim($path, '/'); $path = str_replace(array(';', '$'), array(';;', '$$'), $path); $path = phutil_escape_uri($path); } if (strlen($branch)) { $branch = phutil_escape_uri_path_component($branch); $path = "{$branch}/{$path}"; } if (strlen($commit)) { $commit = str_replace('$', '$$', $commit); $commit = ';'.phutil_escape_uri($commit); } if (strlen($line)) { $line = '$'.phutil_escape_uri($line); } switch ($action) { case 'change': case 'history': case 'browse': case 'lastmodified': case 'tags': case 'branches': case 'lint': case 'pathtree': case 'refs': $uri = "/diffusion/{$identifier}/{$action}/{$path}{$commit}{$line}"; break; case 'branch': if (strlen($path)) { $uri = "/diffusion/{$identifier}/repository/{$path}"; } else { $uri = "/diffusion/{$identifier}/"; } break; case 'external': $commit = ltrim($commit, ';'); $uri = "/diffusion/external/{$commit}/"; break; case 'rendering-ref': // This isn't a real URI per se, it's passed as a query parameter to // the ajax changeset stuff but then we parse it back out as though // it came from a URI. $uri = rawurldecode("{$path}{$commit}"); break; } if ($action == 'rendering-ref') { return $uri; } $uri = new PhutilURI($uri); if (isset($params['lint'])) { $params['params'] = idx($params, 'params', array()) + array( 'lint' => $params['lint'], ); } if (idx($params, 'params')) { $uri->setQueryParams($params['params']); } return $uri; } public function updateURIIndex() { $indexes = array(); $uris = $this->getURIs(); foreach ($uris as $uri) { if ($uri->getIsDisabled()) { continue; } $indexes[] = $uri->getNormalizedURI(); } PhabricatorRepositoryURIIndex::updateRepositoryURIs( $this->getPHID(), $indexes); return $this; } public function isTracked() { $status = $this->getDetail('tracking-enabled'); $map = self::getStatusMap(); $spec = idx($map, $status); if (!$spec) { if ($status) { $status = self::STATUS_ACTIVE; } else { $status = self::STATUS_INACTIVE; } $spec = idx($map, $status); } return (bool)idx($spec, 'isTracked', false); } public function getDefaultBranch() { $default = $this->getDetail('default-branch'); if (strlen($default)) { return $default; } $default_branches = array( PhabricatorRepositoryType::REPOSITORY_TYPE_GIT => 'master', PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL => 'default', ); return idx($default_branches, $this->getVersionControlSystem()); } public function getDefaultArcanistBranch() { return coalesce($this->getDefaultBranch(), 'svn'); } private function isBranchInFilter($branch, $filter_key) { $vcs = $this->getVersionControlSystem(); $is_git = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT); $use_filter = ($is_git); if (!$use_filter) { // If this VCS doesn't use filters, pass everything through. return true; } $filter = $this->getDetail($filter_key, array()); // If there's no filter set, let everything through. if (!$filter) { return true; } // If this branch isn't literally named `regexp(...)`, and it's in the // filter list, let it through. if (isset($filter[$branch])) { if (self::extractBranchRegexp($branch) === null) { return true; } } // If the branch matches a regexp, let it through. foreach ($filter as $pattern => $ignored) { $regexp = self::extractBranchRegexp($pattern); if ($regexp !== null) { if (preg_match($regexp, $branch)) { return true; } } } // Nothing matched, so filter this branch out. return false; } public static function extractBranchRegexp($pattern) { $matches = null; if (preg_match('/^regexp\\((.*)\\)\z/', $pattern, $matches)) { return $matches[1]; } return null; } + public function shouldTrackRef(DiffusionRepositoryRef $ref) { + if (!$ref->isBranch()) { + return true; + } + + return $this->shouldTrackBranch($ref->getShortName()); + } + public function shouldTrackBranch($branch) { return $this->isBranchInFilter($branch, 'branch-filter'); } public function formatCommitName($commit_identifier, $local = false) { $vcs = $this->getVersionControlSystem(); $type_git = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; $type_hg = PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL; $is_git = ($vcs == $type_git); $is_hg = ($vcs == $type_hg); if ($is_git || $is_hg) { $name = substr($commit_identifier, 0, 12); $need_scope = false; } else { $name = $commit_identifier; $need_scope = true; } if (!$local) { $need_scope = true; } if ($need_scope) { $callsign = $this->getCallsign(); if ($callsign) { $scope = "r{$callsign}"; } else { $id = $this->getID(); $scope = "R{$id}:"; } $name = $scope.$name; } return $name; } public function isImporting() { return (bool)$this->getDetail('importing', false); } public function isNewlyInitialized() { return (bool)$this->getDetail('newly-initialized', false); } public function loadImportProgress() { $progress = queryfx_all( $this->establishConnection('r'), 'SELECT importStatus, count(*) N FROM %T WHERE repositoryID = %d GROUP BY importStatus', id(new PhabricatorRepositoryCommit())->getTableName(), $this->getID()); $done = 0; $total = 0; foreach ($progress as $row) { $total += $row['N'] * 4; $status = $row['importStatus']; if ($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE) { $done += $row['N']; } if ($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE) { $done += $row['N']; } if ($status & PhabricatorRepositoryCommit::IMPORTED_OWNERS) { $done += $row['N']; } if ($status & PhabricatorRepositoryCommit::IMPORTED_HERALD) { $done += $row['N']; } } if ($total) { $ratio = ($done / $total); } else { $ratio = 0; } // Cap this at "99.99%", because it's confusing to users when the actual // fraction is "99.996%" and it rounds up to "100.00%". if ($ratio > 0.9999) { $ratio = 0.9999; } return $ratio; } /** * Should this repository publish feed, notifications, audits, and email? * * We do not publish information about repositories during initial import, * or if the repository has been set not to publish. */ public function shouldPublish() { if ($this->isImporting()) { return false; } if ($this->getDetail('herald-disabled')) { return false; } return true; } /* -( Autoclose )---------------------------------------------------------- */ + public function shouldAutocloseRef(DiffusionRepositoryRef $ref) { + if (!$ref->isBranch()) { + return false; + } + + return $this->shouldAutocloseBranch($ref->getShortName()); + } + /** * Determine if autoclose is active for a branch. * * For more details about why, use @{method:shouldSkipAutocloseBranch}. * * @param string Branch name to check. * @return bool True if autoclose is active for the branch. * @task autoclose */ public function shouldAutocloseBranch($branch) { return ($this->shouldSkipAutocloseBranch($branch) === null); } /** * Determine if autoclose is active for a commit. * * For more details about why, use @{method:shouldSkipAutocloseCommit}. * * @param PhabricatorRepositoryCommit Commit to check. * @return bool True if autoclose is active for the commit. * @task autoclose */ public function shouldAutocloseCommit(PhabricatorRepositoryCommit $commit) { return ($this->shouldSkipAutocloseCommit($commit) === null); } /** * Determine why autoclose should be skipped for a branch. * * This method gives a detailed reason why autoclose will be skipped. To * perform a simple test, use @{method:shouldAutocloseBranch}. * * @param string Branch name to check. * @return const|null Constant identifying reason to skip this branch, or null * if autoclose is active. * @task autoclose */ public function shouldSkipAutocloseBranch($branch) { $all_reason = $this->shouldSkipAllAutoclose(); if ($all_reason) { return $all_reason; } if (!$this->shouldTrackBranch($branch)) { return self::BECAUSE_BRANCH_UNTRACKED; } if (!$this->isBranchInFilter($branch, 'close-commits-filter')) { return self::BECAUSE_BRANCH_NOT_AUTOCLOSE; } return null; } /** * Determine why autoclose should be skipped for a commit. * * This method gives a detailed reason why autoclose will be skipped. To * perform a simple test, use @{method:shouldAutocloseCommit}. * * @param PhabricatorRepositoryCommit Commit to check. * @return const|null Constant identifying reason to skip this commit, or null * if autoclose is active. * @task autoclose */ public function shouldSkipAutocloseCommit( PhabricatorRepositoryCommit $commit) { $all_reason = $this->shouldSkipAllAutoclose(); if ($all_reason) { return $all_reason; } switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return null; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: break; default: throw new Exception(pht('Unrecognized version control system.')); } $closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE; if (!$commit->isPartiallyImported($closeable_flag)) { return self::BECAUSE_NOT_ON_AUTOCLOSE_BRANCH; } return null; } /** * Determine why all autoclose operations should be skipped for this * repository. * * @return const|null Constant identifying reason to skip all autoclose * operations, or null if autoclose operations are not blocked at the * repository level. * @task autoclose */ private function shouldSkipAllAutoclose() { if ($this->isImporting()) { return self::BECAUSE_REPOSITORY_IMPORTING; } if ($this->getDetail('disable-autoclose', false)) { return self::BECAUSE_AUTOCLOSE_DISABLED; } return null; } /* -( Repository URI Management )------------------------------------------ */ /** * Get the remote URI for this repository. * * @return string * @task uri */ public function getRemoteURI() { return (string)$this->getRemoteURIObject(); } /** * Get the remote URI for this repository, including credentials if they're * used by this repository. * * @return PhutilOpaqueEnvelope URI, possibly including credentials. * @task uri */ public function getRemoteURIEnvelope() { $uri = $this->getRemoteURIObject(); $remote_protocol = $this->getRemoteProtocol(); if ($remote_protocol == 'http' || $remote_protocol == 'https') { // For SVN, we use `--username` and `--password` flags separately, so // don't add any credentials here. if (!$this->isSVN()) { $credential_phid = $this->getCredentialPHID(); if ($credential_phid) { $key = PassphrasePasswordKey::loadFromPHID( $credential_phid, PhabricatorUser::getOmnipotentUser()); $uri->setUser($key->getUsernameEnvelope()->openEnvelope()); $uri->setPass($key->getPasswordEnvelope()->openEnvelope()); } } } return new PhutilOpaqueEnvelope((string)$uri); } /** * Get the clone (or checkout) URI for this repository, without authentication * information. * * @return string Repository URI. * @task uri */ public function getPublicCloneURI() { return (string)$this->getCloneURIObject(); } /** * Get the protocol for the repository's remote. * * @return string Protocol, like "ssh" or "git". * @task uri */ public function getRemoteProtocol() { $uri = $this->getRemoteURIObject(); return $uri->getProtocol(); } /** * Get a parsed object representation of the repository's remote URI.. * * @return wild A @{class@libphutil:PhutilURI}. * @task uri */ public function getRemoteURIObject() { $raw_uri = $this->getDetail('remote-uri'); if (!strlen($raw_uri)) { return new PhutilURI(''); } if (!strncmp($raw_uri, '/', 1)) { return new PhutilURI('file://'.$raw_uri); } return new PhutilURI($raw_uri); } /** * Get the "best" clone/checkout URI for this repository, on any protocol. */ public function getCloneURIObject() { if (!$this->isHosted()) { if ($this->isSVN()) { // Make sure we pick up the "Import Only" path for Subversion, so // the user clones the repository starting at the correct path, not // from the root. $base_uri = $this->getSubversionBaseURI(); $base_uri = new PhutilURI($base_uri); $path = $base_uri->getPath(); if (!$path) { $path = '/'; } // If the trailing "@" is not required to escape the URI, strip it for // readability. if (!preg_match('/@.*@/', $path)) { $path = rtrim($path, '@'); } $base_uri->setPath($path); return $base_uri; } else { return $this->getRemoteURIObject(); } } // TODO: This should be cleaned up to deal with all the new URI handling. $another_copy = id(new PhabricatorRepositoryQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($this->getPHID())) ->needURIs(true) ->executeOne(); $clone_uris = $another_copy->getCloneURIs(); if (!$clone_uris) { return null; } return head($clone_uris)->getEffectiveURI(); } private function getRawHTTPCloneURIObject() { $uri = PhabricatorEnv::getProductionURI($this->getURI()); $uri = new PhutilURI($uri); if ($this->isGit()) { $uri->setPath($uri->getPath().$this->getCloneName().'.git'); } else if ($this->isHg()) { $uri->setPath($uri->getPath().$this->getCloneName().'/'); } return $uri; } /** * Determine if we should connect to the remote using SSH flags and * credentials. * * @return bool True to use the SSH protocol. * @task uri */ private function shouldUseSSH() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); if ($this->isSSHProtocol($protocol)) { return true; } return false; } /** * Determine if we should connect to the remote using HTTP flags and * credentials. * * @return bool True to use the HTTP protocol. * @task uri */ private function shouldUseHTTP() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); return ($protocol == 'http' || $protocol == 'https'); } /** * Determine if we should connect to the remote using SVN flags and * credentials. * * @return bool True to use the SVN protocol. * @task uri */ private function shouldUseSVNProtocol() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); return ($protocol == 'svn'); } /** * Determine if a protocol is SSH or SSH-like. * * @param string A protocol string, like "http" or "ssh". * @return bool True if the protocol is SSH-like. * @task uri */ private function isSSHProtocol($protocol) { return ($protocol == 'ssh' || $protocol == 'svn+ssh'); } public function delete() { $this->openTransaction(); $paths = id(new PhabricatorOwnersPath()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($paths as $path) { $path->delete(); } queryfx( $this->establishConnection('w'), 'DELETE FROM %T WHERE repositoryPHID = %s', id(new PhabricatorRepositorySymbol())->getTableName(), $this->getPHID()); $commits = id(new PhabricatorRepositoryCommit()) ->loadAllWhere('repositoryID = %d', $this->getID()); foreach ($commits as $commit) { // note PhabricatorRepositoryAuditRequests and // PhabricatorRepositoryCommitData are deleted here too. $commit->delete(); } $uris = id(new PhabricatorRepositoryURI()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($uris as $uri) { $uri->delete(); } $ref_cursors = id(new PhabricatorRepositoryRefCursor()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($ref_cursors as $cursor) { $cursor->delete(); } $conn_w = $this->establishConnection('w'); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_FILESYSTEM, $this->getID()); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_PATHCHANGE, $this->getID()); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_SUMMARY, $this->getID()); $result = parent::delete(); $this->saveTransaction(); return $result; } public function isGit() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT); } public function isSVN() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN); } public function isHg() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL); } public function isHosted() { return (bool)$this->getDetail('hosting-enabled', false); } public function setHosted($enabled) { return $this->setDetail('hosting-enabled', $enabled); } public function canServeProtocol($protocol, $write) { if (!$this->isTracked()) { return false; } $clone_uris = $this->getCloneURIs(); foreach ($clone_uris as $uri) { if ($uri->getBuiltinProtocol() !== $protocol) { continue; } $io_type = $uri->getEffectiveIoType(); if ($io_type == PhabricatorRepositoryURI::IO_READWRITE) { return true; } if (!$write) { if ($io_type == PhabricatorRepositoryURI::IO_READ) { return true; } } } return false; } public function hasLocalWorkingCopy() { try { self::assertLocalExists(); return true; } catch (Exception $ex) { return false; } } /** * Raise more useful errors when there are basic filesystem problems. */ private function assertLocalExists() { if (!$this->usesLocalWorkingCopy()) { return; } $local = $this->getLocalPath(); Filesystem::assertExists($local); Filesystem::assertIsDirectory($local); Filesystem::assertReadable($local); } /** * Determine if the working copy is bare or not. In Git, this corresponds * to `--bare`. In Mercurial, `--noupdate`. */ public function isWorkingCopyBare() { switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return false; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $local = $this->getLocalPath(); if (Filesystem::pathExists($local.'/.git')) { return false; } else { return true; } } } public function usesLocalWorkingCopy() { switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->isHosted(); case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return true; } } public function getHookDirectories() { $directories = array(); if (!$this->isHosted()) { return $directories; } $root = $this->getLocalPath(); switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: if ($this->isWorkingCopyBare()) { $directories[] = $root.'/hooks/pre-receive-phabricator.d/'; } else { $directories[] = $root.'/.git/hooks/pre-receive-phabricator.d/'; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $directories[] = $root.'/hooks/pre-commit-phabricator.d/'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // NOTE: We don't support custom Mercurial hooks for now because they're // messy and we can't easily just drop a `hooks.d/` directory next to // the hooks. break; } return $directories; } public function canDestroyWorkingCopy() { if ($this->isHosted()) { // Never destroy hosted working copies. return false; } $default_path = PhabricatorEnv::getEnvConfig( 'repository.default-local-path'); return Filesystem::isDescendant($this->getLocalPath(), $default_path); } public function canUsePathTree() { return !$this->isSVN(); } public function canUseGitLFS() { if (!$this->isGit()) { return false; } if (!$this->isHosted()) { return false; } // TODO: Unprototype this feature. if (!PhabricatorEnv::getEnvConfig('phabricator.show-prototypes')) { return false; } return true; } public function getGitLFSURI($path = null) { if (!$this->canUseGitLFS()) { throw new Exception( pht( 'This repository does not support Git LFS, so Git LFS URIs can '. 'not be generated for it.')); } $uri = $this->getRawHTTPCloneURIObject(); $uri = (string)$uri; $uri = $uri.'/'.$path; return $uri; } public function canMirror() { if ($this->isGit() || $this->isHg()) { return true; } return false; } public function canAllowDangerousChanges() { if (!$this->isHosted()) { return false; } if ($this->isGit() || $this->isHg()) { return true; } return false; } public function shouldAllowDangerousChanges() { return (bool)$this->getDetail('allow-dangerous-changes'); } public function writeStatusMessage( $status_type, $status_code, array $parameters = array()) { $table = new PhabricatorRepositoryStatusMessage(); $conn_w = $table->establishConnection('w'); $table_name = $table->getTableName(); if ($status_code === null) { queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d AND statusType = %s', $table_name, $this->getID(), $status_type); } else { queryfx( $conn_w, 'INSERT INTO %T (repositoryID, statusType, statusCode, parameters, epoch) VALUES (%d, %s, %s, %s, %d) ON DUPLICATE KEY UPDATE statusCode = VALUES(statusCode), parameters = VALUES(parameters), epoch = VALUES(epoch)', $table_name, $this->getID(), $status_type, $status_code, json_encode($parameters), time()); } return $this; } public static function assertValidRemoteURI($uri) { if (trim($uri) != $uri) { throw new Exception( pht('The remote URI has leading or trailing whitespace.')); } $uri_object = new PhutilURI($uri); $protocol = $uri_object->getProtocol(); // Catch confusion between Git/SCP-style URIs and normal URIs. See T3619 // for discussion. This is usually a user adding "ssh://" to an implicit // SSH Git URI. if ($protocol == 'ssh') { if (preg_match('(^[^:@]+://[^/:]+:[^\d])', $uri)) { throw new Exception( pht( "The remote URI is not formatted correctly. Remote URIs ". "with an explicit protocol should be in the form ". "'%s', not '%s'. The '%s' syntax is only valid in SCP-style URIs.", 'proto://domain/path', 'proto://domain:/path', ':/path')); } } switch ($protocol) { case 'ssh': case 'http': case 'https': case 'git': case 'svn': case 'svn+ssh': break; default: // NOTE: We're explicitly rejecting 'file://' because it can be // used to clone from the working copy of another repository on disk // that you don't normally have permission to access. throw new Exception( pht( 'The URI protocol is unrecognized. It should begin with '. '"%s", "%s", "%s", "%s", "%s", "%s", or be in the form "%s".', 'ssh://', 'http://', 'https://', 'git://', 'svn://', 'svn+ssh://', 'git@domain.com:path')); } return true; } /** * Load the pull frequency for this repository, based on the time since the * last activity. * * We pull rarely used repositories less frequently. This finds the most * recent commit which is older than the current time (which prevents us from * spinning on repositories with a silly commit post-dated to some time in * 2037). We adjust the pull frequency based on when the most recent commit * occurred. * * @param int The minimum update interval to use, in seconds. * @return int Repository update interval, in seconds. */ public function loadUpdateInterval($minimum = 15) { // If a repository is still importing, always pull it as frequently as // possible. This prevents us from hanging for a long time at 99.9% when // importing an inactive repository. if ($this->isImporting()) { return $minimum; } $window_start = (PhabricatorTime::getNow() + $minimum); $table = id(new PhabricatorRepositoryCommit()); $last_commit = queryfx_one( $table->establishConnection('r'), 'SELECT epoch FROM %T WHERE repositoryID = %d AND epoch <= %d ORDER BY epoch DESC LIMIT 1', $table->getTableName(), $this->getID(), $window_start); if ($last_commit) { $time_since_commit = ($window_start - $last_commit['epoch']); $last_few_days = phutil_units('3 days in seconds'); if ($time_since_commit <= $last_few_days) { // For repositories with activity in the recent past, we wait one // extra second for every 10 minutes since the last commit. This // shorter backoff is intended to handle weekends and other short // breaks from development. $smart_wait = ($time_since_commit / 600); } else { // For repositories without recent activity, we wait one extra second // for every 4 minutes since the last commit. This longer backoff // handles rarely used repositories, up to the maximum. $smart_wait = ($time_since_commit / 240); } // We'll never wait more than 6 hours to pull a repository. $longest_wait = phutil_units('6 hours in seconds'); $smart_wait = min($smart_wait, $longest_wait); $smart_wait = max($minimum, $smart_wait); } else { $smart_wait = $minimum; } return $smart_wait; } /** * Retrieve the sevice URI for the device hosting this repository. * * See @{method:newConduitClient} for a general discussion of interacting * with repository services. This method provides lower-level resolution of * services, returning raw URIs. * * @param PhabricatorUser Viewing user. * @param bool `true` to throw if a remote URI would be returned. * @param list List of allowable protocols. * @return string|null URI, or `null` for local repositories. */ public function getAlmanacServiceURI( PhabricatorUser $viewer, $never_proxy, array $protocols) { $service = $this->loadAlmanacService(); if (!$service) { return null; } $bindings = $service->getActiveBindings(); if (!$bindings) { throw new Exception( pht( 'The Almanac service for this repository is not bound to any '. 'interfaces.')); } $local_device = AlmanacKeys::getDeviceID(); if ($never_proxy && !$local_device) { throw new Exception( pht( 'Unable to handle proxied service request. This device is not '. 'registered, so it can not identify local services. Register '. 'this device before sending requests here.')); } $protocol_map = array_fuse($protocols); $uris = array(); foreach ($bindings as $binding) { $iface = $binding->getInterface(); // If we're never proxying this and it's locally satisfiable, return // `null` to tell the caller to handle it locally. If we're allowed to // proxy, we skip this check and may proxy the request to ourselves. // (That proxied request will end up here with proxying forbidden, // return `null`, and then the request will actually run.) if ($local_device && $never_proxy) { if ($iface->getDevice()->getName() == $local_device) { return null; } } $uri = $this->getClusterRepositoryURIFromBinding($binding); $protocol = $uri->getProtocol(); if (empty($protocol_map[$protocol])) { continue; } $uris[] = $uri; } if (!$uris) { throw new Exception( pht( 'The Almanac service for this repository is not bound to any '. 'interfaces which support the required protocols (%s).', implode(', ', $protocols))); } if ($never_proxy) { throw new Exception( pht( 'Refusing to proxy a repository request from a cluster host. '. 'Cluster hosts must correctly route their intracluster requests.')); } shuffle($uris); return head($uris); } /** * Build a new Conduit client in order to make a service call to this * repository. * * If the repository is hosted locally, this method may return `null`. The * caller should use `ConduitCall` or other local logic to complete the * request. * * By default, we will return a @{class:ConduitClient} for any repository with * a service, even if that service is on the current device. * * We do this because this configuration does not make very much sense in a * production context, but is very common in a test/development context * (where the developer's machine is both the web host and the repository * service). By proxying in development, we get more consistent behavior * between development and production, and don't have a major untested * codepath. * * The `$never_proxy` parameter can be used to prevent this local proxying. * If the flag is passed: * * - The method will return `null` (implying a local service call) * if the repository service is hosted on the current device. * - The method will throw if it would need to return a client. * * This is used to prevent loops in Conduit: the first request will proxy, * even in development, but the second request will be identified as a * cluster request and forced not to proxy. * * For lower-level service resolution, see @{method:getAlmanacServiceURI}. * * @param PhabricatorUser Viewing user. * @param bool `true` to throw if a client would be returned. * @return ConduitClient|null Client, or `null` for local repositories. */ public function newConduitClient( PhabricatorUser $viewer, $never_proxy = false) { $uri = $this->getAlmanacServiceURI( $viewer, $never_proxy, array( 'http', 'https', )); if ($uri === null) { return null; } $domain = id(new PhutilURI(PhabricatorEnv::getURI('/')))->getDomain(); $client = id(new ConduitClient($uri)) ->setHost($domain); if ($viewer->isOmnipotent()) { // If the caller is the omnipotent user (normally, a daemon), we will // sign the request with this host's asymmetric keypair. $public_path = AlmanacKeys::getKeyPath('device.pub'); try { $public_key = Filesystem::readFile($public_path); } catch (Exception $ex) { throw new PhutilAggregateException( pht( 'Unable to read device public key while attempting to make '. 'authenticated method call within the Phabricator cluster. '. 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex)); } $private_path = AlmanacKeys::getKeyPath('device.key'); try { $private_key = Filesystem::readFile($private_path); $private_key = new PhutilOpaqueEnvelope($private_key); } catch (Exception $ex) { throw new PhutilAggregateException( pht( 'Unable to read device private key while attempting to make '. 'authenticated method call within the Phabricator cluster. '. 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex)); } $client->setSigningKeys($public_key, $private_key); } else { // If the caller is a normal user, we generate or retrieve a cluster // API token. $token = PhabricatorConduitToken::loadClusterTokenForUser($viewer); if ($token) { $client->setConduitToken($token->getToken()); } } return $client; } /* -( Repository URIs )---------------------------------------------------- */ public function attachURIs(array $uris) { $custom_map = array(); foreach ($uris as $key => $uri) { $builtin_key = $uri->getRepositoryURIBuiltinKey(); if ($builtin_key !== null) { $custom_map[$builtin_key] = $key; } } $builtin_uris = $this->newBuiltinURIs(); $seen_builtins = array(); foreach ($builtin_uris as $builtin_uri) { $builtin_key = $builtin_uri->getRepositoryURIBuiltinKey(); $seen_builtins[$builtin_key] = true; // If this builtin URI is disabled, don't attach it and remove the // persisted version if it exists. if ($builtin_uri->getIsDisabled()) { if (isset($custom_map[$builtin_key])) { unset($uris[$custom_map[$builtin_key]]); } continue; } // If the URI exists, make sure it's marked as not being disabled. if (isset($custom_map[$builtin_key])) { $uris[$custom_map[$builtin_key]]->setIsDisabled(false); } } // Remove any builtins which no longer exist. foreach ($custom_map as $builtin_key => $key) { if (empty($seen_builtins[$builtin_key])) { unset($uris[$key]); } } $this->uris = $uris; return $this; } public function getURIs() { return $this->assertAttached($this->uris); } public function getCloneURIs() { $uris = $this->getURIs(); $clone = array(); foreach ($uris as $uri) { if (!$uri->isBuiltin()) { continue; } if ($uri->getIsDisabled()) { continue; } $io_type = $uri->getEffectiveIoType(); $is_clone = ($io_type == PhabricatorRepositoryURI::IO_READ) || ($io_type == PhabricatorRepositoryURI::IO_READWRITE); if (!$is_clone) { continue; } $clone[] = $uri; } $clone = msort($clone, 'getURIScore'); $clone = array_reverse($clone); return $clone; } public function newBuiltinURIs() { $has_callsign = ($this->getCallsign() !== null); $has_shortname = ($this->getRepositorySlug() !== null); // TODO: For now, never enable these because they don't work yet. $has_shortname = false; $identifier_map = array( PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_CALLSIGN => $has_callsign, PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_SHORTNAME => $has_shortname, PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_ID => true, ); // If the view policy of the repository is public, support anonymous HTTP // even if authenticated HTTP is not supported. if ($this->getViewPolicy() === PhabricatorPolicies::POLICY_PUBLIC) { $allow_http = true; } else { $allow_http = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth'); } $base_uri = PhabricatorEnv::getURI('/'); $base_uri = new PhutilURI($base_uri); $has_https = ($base_uri->getProtocol() == 'https'); $has_https = ($has_https && $allow_http); $has_http = !PhabricatorEnv::getEnvConfig('security.require-https'); $has_http = ($has_http && $allow_http); // HTTP is not supported for Subversion. if ($this->isSVN()) { $has_http = false; $has_https = false; } $has_ssh = (bool)strlen(PhabricatorEnv::getEnvConfig('phd.user')); $protocol_map = array( PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH => $has_ssh, PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTPS => $has_https, PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTP => $has_http, ); $uris = array(); foreach ($protocol_map as $protocol => $proto_supported) { foreach ($identifier_map as $identifier => $id_supported) { // This is just a dummy value because it can't be empty; we'll force // it to a proper value when using it in the UI. $builtin_uri = "{$protocol}://{$identifier}"; $uris[] = PhabricatorRepositoryURI::initializeNewURI() ->setRepositoryPHID($this->getPHID()) ->attachRepository($this) ->setBuiltinProtocol($protocol) ->setBuiltinIdentifier($identifier) ->setURI($builtin_uri) ->setIsDisabled((int)(!$proto_supported || !$id_supported)); } } return $uris; } public function getClusterRepositoryURIFromBinding( AlmanacBinding $binding) { $protocol = $binding->getAlmanacPropertyValue('protocol'); if ($protocol === null) { $protocol = 'https'; } $iface = $binding->getInterface(); $address = $iface->renderDisplayAddress(); $path = $this->getURI(); return id(new PhutilURI("{$protocol}://{$address}")) ->setPath($path); } public function loadAlmanacService() { $service_phid = $this->getAlmanacServicePHID(); if (!$service_phid) { // No service, so this is a local repository. return null; } $service = id(new AlmanacServiceQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($service_phid)) ->needBindings(true) ->needProperties(true) ->executeOne(); if (!$service) { throw new Exception( pht( 'The Almanac service for this repository is invalid or could not '. 'be loaded.')); } $service_type = $service->getServiceImplementation(); if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) { throw new Exception( pht( 'The Almanac service for this repository does not have the correct '. 'service type.')); } return $service; } public function markImporting() { $this->openTransaction(); $this->beginReadLocking(); $repository = $this->reload(); $repository->setDetail('importing', true); $repository->save(); $this->endReadLocking(); $this->saveTransaction(); return $repository; } /* -( Symbols )-------------------------------------------------------------*/ public function getSymbolSources() { return $this->getDetail('symbol-sources', array()); } public function getSymbolLanguages() { return $this->getDetail('symbol-languages', array()); } /* -( Staging )------------------------------------------------------------ */ public function supportsStaging() { return $this->isGit(); } public function getStagingURI() { if (!$this->supportsStaging()) { return null; } return $this->getDetail('staging-uri', null); } /* -( Automation )--------------------------------------------------------- */ public function supportsAutomation() { return $this->isGit(); } public function canPerformAutomation() { if (!$this->supportsAutomation()) { return false; } if (!$this->getAutomationBlueprintPHIDs()) { return false; } return true; } public function getAutomationBlueprintPHIDs() { if (!$this->supportsAutomation()) { return array(); } return $this->getDetail('automation.blueprintPHIDs', array()); } /* -( PhabricatorApplicationTransactionInterface )------------------------- */ public function getApplicationTransactionEditor() { return new PhabricatorRepositoryEditor(); } public function getApplicationTransactionObject() { return $this; } public function getApplicationTransactionTemplate() { return new PhabricatorRepositoryTransaction(); } public function willRenderTimeline( PhabricatorApplicationTransactionView $timeline, AphrontRequest $request) { return $timeline; } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, DiffusionPushCapability::CAPABILITY, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: return $this->getViewPolicy(); case PhabricatorPolicyCapability::CAN_EDIT: return $this->getEditPolicy(); case DiffusionPushCapability::CAPABILITY: return $this->getPushPolicy(); } } public function hasAutomaticCapability($capability, PhabricatorUser $user) { return false; } public function describeAutomaticCapability($capability) { return null; } /* -( PhabricatorMarkupInterface )----------------------------------------- */ public function getMarkupFieldKey($field) { $hash = PhabricatorHash::digestForIndex($this->getMarkupText($field)); return "repo:{$hash}"; } public function newMarkupEngine($field) { return PhabricatorMarkupEngine::newMarkupEngine(array()); } public function getMarkupText($field) { return $this->getDetail('description'); } public function didMarkupText( $field, $output, PhutilMarkupEngine $engine) { require_celerity_resource('phabricator-remarkup-css'); return phutil_tag( 'div', array( 'class' => 'phabricator-remarkup', ), $output); } public function shouldUseMarkupCache($field) { return true; } /* -( PhabricatorDestructibleInterface )----------------------------------- */ public function destroyObjectPermanently( PhabricatorDestructionEngine $engine) { $phid = $this->getPHID(); $this->openTransaction(); $this->delete(); PhabricatorRepositoryURIIndex::updateRepositoryURIs($phid, array()); $books = id(new DivinerBookQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($books as $book) { $engine->destroyObject($book); } $atoms = id(new DivinerAtomQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($atoms as $atom) { $engine->destroyObject($atom); } $lfs_refs = id(new PhabricatorRepositoryGitLFSRefQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($lfs_refs as $ref) { $engine->destroyObject($ref); } $this->saveTransaction(); } /* -( PhabricatorSpacesInterface )----------------------------------------- */ public function getSpacePHID() { return $this->spacePHID; } /* -( PhabricatorConduitResultInterface )---------------------------------- */ public function getFieldSpecificationsForConduit() { return array( id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('name') ->setType('string') ->setDescription(pht('The repository name.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('vcs') ->setType('string') ->setDescription( pht('The VCS this repository uses ("git", "hg" or "svn").')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('callsign') ->setType('string') ->setDescription(pht('The repository callsign, if it has one.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('shortName') ->setType('string') ->setDescription(pht('Unique short name, if the repository has one.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('status') ->setType('string') ->setDescription(pht('Active or inactive status.')), ); } public function getFieldValuesForConduit() { return array( 'name' => $this->getName(), 'vcs' => $this->getVersionControlSystem(), 'callsign' => $this->getCallsign(), 'shortName' => $this->getRepositorySlug(), 'status' => $this->getStatus(), ); } public function getConduitSearchAttachments() { return array( id(new DiffusionRepositoryURIsSearchEngineAttachment()) ->setAttachmentKey('uris'), ); } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryRefCursor.php b/src/applications/repository/storage/PhabricatorRepositoryRefCursor.php index d79aa9be70..febbed3ee6 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryRefCursor.php +++ b/src/applications/repository/storage/PhabricatorRepositoryRefCursor.php @@ -1,98 +1,99 @@ false, self::CONFIG_AUX_PHID => true, self::CONFIG_BINARY => array( 'refNameRaw' => true, ), self::CONFIG_COLUMN_SCHEMA => array( 'refType' => 'text32', 'refNameHash' => 'bytes12', 'commitIdentifier' => 'text40', 'refNameEncoding' => 'text16?', 'isClosed' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_cursor' => array( 'columns' => array('repositoryPHID', 'refType', 'refNameHash'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryRefCursorPHIDType::TYPECONST); } public function getRefName() { return $this->getUTF8StringFromStorage( $this->getRefNameRaw(), $this->getRefNameEncoding()); } public function setRefName($ref_raw) { $this->setRefNameRaw($ref_raw); $this->setRefNameHash(PhabricatorHash::digestForIndex($ref_raw)); $this->setRefNameEncoding($this->detectEncodingForStorage($ref_raw)); return $this; } public function attachRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->assertAttached($this->repository); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getRepository()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getRepository()->hasAutomaticCapability($capability, $viewer); } public function describeAutomaticCapability($capability) { return pht('Repository refs have the same policies as their repository.'); } }