diff --git a/src/applications/diffusion/query/blame/DiffusionBlameQuery.php b/src/applications/diffusion/query/blame/DiffusionBlameQuery.php index 6d83020dd2..4d03b4a143 100644 --- a/src/applications/diffusion/query/blame/DiffusionBlameQuery.php +++ b/src/applications/diffusion/query/blame/DiffusionBlameQuery.php @@ -1,69 +1,180 @@ timeout = $timeout; return $this; } public function getTimeout() { return $this->timeout; } public function setPaths(array $paths) { $this->paths = $paths; return $this; } public function getPaths() { return $this->paths; } abstract protected function newBlameFuture(DiffusionRequest $request, $path); abstract protected function resolveBlameFuture(ExecFuture $future); final public static function newFromDiffusionRequest( DiffusionRequest $request) { return parent::newQueryObject(__CLASS__, $request); } final protected function executeQuery() { $paths = $this->getPaths(); + + $blame = array(); + + // Load cache keys: these are the commits at which each path was last + // touched. + $keys = $this->loadCacheKeys($paths); + + // Try to read blame data from cache. + $cache = $this->readCacheData($keys); + foreach ($paths as $key => $path) { + if (!isset($cache[$path])) { + continue; + } + + $blame[$path] = $cache[$path]; + unset($paths[$key]); + } + + // If we have no paths left, we filled everything from cache and can + // bail out early. + if (!$paths) { + return $blame; + } + $request = $this->getRequest(); $timeout = $this->getTimeout(); + // We're still missing at least some data, so we need to run VCS commands + // to pull it. $futures = array(); foreach ($paths as $path) { $future = $this->newBlameFuture($request, $path); if ($timeout) { $future->setTimeout($timeout); } $futures[$path] = $future; } + $futures = id(new FutureIterator($futures)) + ->limit(4); - $blame = array(); + foreach ($futures as $path => $future) { + $path_blame = $this->resolveBlameFuture($future); + if ($path_blame !== null) { + $blame[$path] = $path_blame; + } + } + + // Fill the cache with anything we generated. + $this->writeCacheData( + array_select_keys($keys, $paths), + $blame); - if ($futures) { - $futures = id(new FutureIterator($futures)) - ->limit(4); + return $blame; + } - foreach ($futures as $path => $future) { - $path_blame = $this->resolveBlameFuture($future); - if ($path_blame !== null) { - $blame[$path] = $path_blame; - } + private function loadCacheKeys(array $paths) { + $request = $this->getRequest(); + $viewer = $request->getUser(); + + $repository = $request->getRepository(); + $repository_id = $repository->getID(); + + $last_modified = parent::callConduitWithDiffusionRequest( + $viewer, + $request, + 'diffusion.lastmodifiedquery', + array( + 'paths' => array_fill_keys($paths, $request->getCommit()), + )); + + $map = array(); + foreach ($paths as $path) { + $identifier = idx($last_modified, $path); + if ($identifier === null) { + continue; } + + $map[$path] = "blame({$repository_id}, {$identifier}, {$path}, raw)"; } - return $blame; + return $map; + } + + private function readCacheData(array $keys) { + $cache = PhabricatorCaches::getImmutableCache(); + $data = $cache->getKeys($keys); + + $results = array(); + foreach ($keys as $path => $key) { + if (!isset($data[$key])) { + continue; + } + $results[$path] = $data[$key]; + } + + // Decode the cache storage format. + foreach ($results as $path => $cache) { + list($head, $body) = explode("\n", $cache, 2); + switch ($head) { + case 'raw': + $body = explode("\n", $body); + break; + default: + $body = null; + break; + } + + if ($body === null) { + unset($results[$path]); + } else { + $results[$path] = $body; + } + } + + return $results; + } + + private function writeCacheData(array $keys, array $blame) { + $writes = array(); + foreach ($keys as $path => $key) { + $value = idx($blame, $path); + if ($value === null) { + continue; + } + + // For now, just store the entire value with a "raw" header. In the + // future, we could compress this or use IDs instead. + $value = "raw\n".implode("\n", $value); + + $writes[$key] = $value; + } + + if (!$writes) { + return; + } + + $cache = PhabricatorCaches::getImmutableCache(); + $data = $cache->setKeys($writes, phutil_units('14 days in seconds')); } } diff --git a/src/applications/repository/query/PhabricatorRepositoryQuery.php b/src/applications/repository/query/PhabricatorRepositoryQuery.php index 38fa5bf56c..bd9765130a 100644 --- a/src/applications/repository/query/PhabricatorRepositoryQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryQuery.php @@ -1,596 +1,598 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withCallsigns(array $callsigns) { $this->callsigns = $callsigns; return $this; } public function withIdentifiers(array $identifiers) { + $identifiers = array_fuse($identifiers); + $ids = array(); $callsigns = array(); $phids = array(); $monograms = array(); foreach ($identifiers as $identifier) { if (ctype_digit((string)$identifier)) { $ids[$identifier] = $identifier; } else if (preg_match('/^(r[A-Z]+)|(R[1-9]\d*)\z/', $identifier)) { $monograms[$identifier] = $identifier; } else { $repository_type = PhabricatorRepositoryRepositoryPHIDType::TYPECONST; if (phid_get_type($identifier) === $repository_type) { $phids[$identifier] = $identifier; } else { $callsigns[$identifier] = $identifier; } } } $this->numericIdentifiers = $ids; $this->callsignIdentifiers = $callsigns; $this->phidIdentifiers = $phids; $this->monogramIdentifiers = $monograms; return $this; } public function withStatus($status) { $this->status = $status; return $this; } public function withHosted($hosted) { $this->hosted = $hosted; return $this; } public function withTypes(array $types) { $this->types = $types; return $this; } public function withUUIDs(array $uuids) { $this->uuids = $uuids; return $this; } public function withNameContains($contains) { $this->nameContains = $contains; return $this; } public function withRemoteURIs(array $uris) { $this->remoteURIs = $uris; return $this; } public function withDatasourceQuery($query) { $this->datasourceQuery = $query; return $this; } public function needCommitCounts($need_counts) { $this->needCommitCounts = $need_counts; return $this; } public function needMostRecentCommits($need_commits) { $this->needMostRecentCommits = $need_commits; return $this; } public function needProjectPHIDs($need_phids) { $this->needProjectPHIDs = $need_phids; return $this; } public function getBuiltinOrders() { return array( 'committed' => array( 'vector' => array('committed', 'id'), 'name' => pht('Most Recent Commit'), ), 'name' => array( 'vector' => array('name', 'id'), 'name' => pht('Name'), ), 'callsign' => array( 'vector' => array('callsign'), 'name' => pht('Callsign'), ), 'size' => array( 'vector' => array('size', 'id'), 'name' => pht('Size'), ), ) + parent::getBuiltinOrders(); } public function getIdentifierMap() { if ($this->identifierMap === null) { throw new PhutilInvalidStateException('execute'); } return $this->identifierMap; } protected function willExecute() { $this->identifierMap = array(); } public function newResultObject() { return new PhabricatorRepository(); } protected function loadPage() { $table = $this->newResultObject(); $data = $this->loadStandardPageRows($table); $repositories = $table->loadAllFromArray($data); if ($this->needCommitCounts) { $sizes = ipull($data, 'size', 'id'); foreach ($repositories as $id => $repository) { $repository->attachCommitCount(nonempty($sizes[$id], 0)); } } if ($this->needMostRecentCommits) { $commit_ids = ipull($data, 'lastCommitID', 'id'); $commit_ids = array_filter($commit_ids); if ($commit_ids) { $commits = id(new DiffusionCommitQuery()) ->setViewer($this->getViewer()) ->withIDs($commit_ids) ->execute(); } else { $commits = array(); } foreach ($repositories as $id => $repository) { $commit = null; if (idx($commit_ids, $id)) { $commit = idx($commits, $commit_ids[$id]); } $repository->attachMostRecentCommit($commit); } } return $repositories; } protected function willFilterPage(array $repositories) { assert_instances_of($repositories, 'PhabricatorRepository'); // TODO: Denormalize repository status into the PhabricatorRepository // table so we can do this filtering in the database. foreach ($repositories as $key => $repo) { $status = $this->status; switch ($status) { case self::STATUS_OPEN: if (!$repo->isTracked()) { unset($repositories[$key]); } break; case self::STATUS_CLOSED: if ($repo->isTracked()) { unset($repositories[$key]); } break; case self::STATUS_ALL: break; default: throw new Exception("Unknown status '{$status}'!"); } // TODO: This should also be denormalized. $hosted = $this->hosted; switch ($hosted) { case self::HOSTED_PHABRICATOR: if (!$repo->isHosted()) { unset($repositories[$key]); } break; case self::HOSTED_REMOTE: if ($repo->isHosted()) { unset($repositories[$key]); } break; case self::HOSTED_ALL: break; default: throw new Exception(pht("Unknown hosted failed '%s'!", $hosted)); } } // TODO: Denormalize this, too. if ($this->remoteURIs) { $try_uris = $this->getNormalizedPaths(); $try_uris = array_fuse($try_uris); foreach ($repositories as $key => $repository) { if (!isset($try_uris[$repository->getNormalizedPath()])) { unset($repositories[$key]); } } } // Build the identifierMap if ($this->numericIdentifiers) { foreach ($this->numericIdentifiers as $id) { if (isset($repositories[$id])) { $this->identifierMap[$id] = $repositories[$id]; } } } if ($this->callsignIdentifiers) { $repository_callsigns = mpull($repositories, null, 'getCallsign'); foreach ($this->callsignIdentifiers as $callsign) { if (isset($repository_callsigns[$callsign])) { $this->identifierMap[$callsign] = $repository_callsigns[$callsign]; } } } if ($this->phidIdentifiers) { $repository_phids = mpull($repositories, null, 'getPHID'); foreach ($this->phidIdentifiers as $phid) { if (isset($repository_phids[$phid])) { $this->identifierMap[$phid] = $repository_phids[$phid]; } } } if ($this->monogramIdentifiers) { $monogram_map = array(); foreach ($repositories as $repository) { foreach ($repository->getAllMonograms() as $monogram) { $monogram_map[$monogram] = $repository; } } foreach ($this->monogramIdentifiers as $monogram) { if (isset($monogram_map[$monogram])) { $this->identifierMap[$monogram] = $monogram_map[$monogram]; } } } return $repositories; } protected function didFilterPage(array $repositories) { if ($this->needProjectPHIDs) { $type_project = PhabricatorProjectObjectHasProjectEdgeType::EDGECONST; $edge_query = id(new PhabricatorEdgeQuery()) ->withSourcePHIDs(mpull($repositories, 'getPHID')) ->withEdgeTypes(array($type_project)); $edge_query->execute(); foreach ($repositories as $repository) { $project_phids = $edge_query->getDestinationPHIDs( array( $repository->getPHID(), )); $repository->attachProjectPHIDs($project_phids); } } return $repositories; } protected function getPrimaryTableAlias() { return 'r'; } public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'committed' => array( 'table' => 's', 'column' => 'epoch', 'type' => 'int', 'null' => 'tail', ), 'callsign' => array( 'table' => 'r', 'column' => 'callsign', 'type' => 'string', 'unique' => true, 'reverse' => true, ), 'name' => array( 'table' => 'r', 'column' => 'name', 'type' => 'string', 'reverse' => true, ), 'size' => array( 'table' => 's', 'column' => 'size', 'type' => 'int', 'null' => 'tail', ), ); } protected function willExecuteCursorQuery( PhabricatorCursorPagedPolicyAwareQuery $query) { $vector = $this->getOrderVector(); if ($vector->containsKey('committed')) { $query->needMostRecentCommits(true); } if ($vector->containsKey('size')) { $query->needCommitCounts(true); } } protected function getPagingValueMap($cursor, array $keys) { $repository = $this->loadCursorObject($cursor); $map = array( 'id' => $repository->getID(), 'callsign' => $repository->getCallsign(), 'name' => $repository->getName(), ); foreach ($keys as $key) { switch ($key) { case 'committed': $commit = $repository->getMostRecentCommit(); if ($commit) { $map[$key] = $commit->getEpoch(); } else { $map[$key] = null; } break; case 'size': $count = $repository->getCommitCount(); if ($count) { $map[$key] = $count; } else { $map[$key] = null; } break; } } return $map; } protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { $parts = parent::buildSelectClauseParts($conn); if ($this->shouldJoinSummaryTable()) { $parts[] = 's.*'; } return $parts; } protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = parent::buildJoinClauseParts($conn); if ($this->shouldJoinSummaryTable()) { $joins[] = qsprintf( $conn, 'LEFT JOIN %T s ON r.id = s.repositoryID', PhabricatorRepository::TABLE_SUMMARY); } return $joins; } private function shouldJoinSummaryTable() { if ($this->needCommitCounts) { return true; } if ($this->needMostRecentCommits) { return true; } $vector = $this->getOrderVector(); if ($vector->containsKey('committed')) { return true; } if ($vector->containsKey('size')) { return true; } return false; } protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( $conn, 'r.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, 'r.phid IN (%Ls)', $this->phids); } if ($this->callsigns !== null) { $where[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $this->callsigns); } if ($this->numericIdentifiers || $this->callsignIdentifiers || $this->phidIdentifiers || $this->monogramIdentifiers) { $identifier_clause = array(); if ($this->numericIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.id IN (%Ld)', $this->numericIdentifiers); } if ($this->callsignIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $this->callsignIdentifiers); } if ($this->phidIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.phid IN (%Ls)', $this->phidIdentifiers); } if ($this->monogramIdentifiers) { $monogram_callsigns = array(); $monogram_ids = array(); foreach ($this->monogramIdentifiers as $identifier) { if ($identifier[0] == 'r') { $monogram_callsigns[] = substr($identifier, 1); } else { $monogram_ids[] = substr($identifier, 1); } } if ($monogram_ids) { $identifier_clause[] = qsprintf( $conn, 'r.id IN (%Ld)', $monogram_ids); } if ($monogram_callsigns) { $identifier_clause[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $monogram_callsigns); } } $where = array('('.implode(' OR ', $identifier_clause).')'); } if ($this->types) { $where[] = qsprintf( $conn, 'r.versionControlSystem IN (%Ls)', $this->types); } if ($this->uuids) { $where[] = qsprintf( $conn, 'r.uuid IN (%Ls)', $this->uuids); } if (strlen($this->nameContains)) { $where[] = qsprintf( $conn, 'name LIKE %~', $this->nameContains); } if (strlen($this->datasourceQuery)) { // This handles having "rP" match callsigns starting with "P...". $query = trim($this->datasourceQuery); if (preg_match('/^r/', $query)) { $callsign = substr($query, 1); } else { $callsign = $query; } $where[] = qsprintf( $conn, 'r.name LIKE %> OR r.callsign LIKE %>', $query, $callsign); } return $where; } public function getQueryApplicationClass() { return 'PhabricatorDiffusionApplication'; } private function getNormalizedPaths() { $normalized_uris = array(); // Since we don't know which type of repository this URI is in the general // case, just generate all the normalizations. We could refine this in some // cases: if the query specifies VCS types, or the URI is a git-style URI // or an `svn+ssh` URI, we could deduce how to normalize it. However, this // would be more complicated and it's not clear if it matters in practice. foreach ($this->remoteURIs as $uri) { $normalized_uris[] = new PhabricatorRepositoryURINormalizer( PhabricatorRepositoryURINormalizer::TYPE_GIT, $uri); $normalized_uris[] = new PhabricatorRepositoryURINormalizer( PhabricatorRepositoryURINormalizer::TYPE_SVN, $uri); $normalized_uris[] = new PhabricatorRepositoryURINormalizer( PhabricatorRepositoryURINormalizer::TYPE_MERCURIAL, $uri); } return array_unique(mpull($normalized_uris, 'getNormalizedPath')); } }