diff --git a/resources/sql/autopatches/20140928.schema.blamecache.sql b/resources/sql/autopatches/20140928.schema.blamecache.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20140928.schema.blamecache.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_blamecache ( + pathId int(10) unsigned NOT NULL, + commitId int(10) unsigned NOT NULL, + blameDict mediumblob, + PRIMARY KEY (pathId,commitId) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2085,6 +2085,7 @@ 'PhabricatorRepositoryArcanistProjectPHIDType' => 'applications/repository/phid/PhabricatorRepositoryArcanistProjectPHIDType.php', 'PhabricatorRepositoryArcanistProjectQuery' => 'applications/repository/query/PhabricatorRepositoryArcanistProjectQuery.php', 'PhabricatorRepositoryAuditRequest' => 'applications/repository/storage/PhabricatorRepositoryAuditRequest.php', + 'PhabricatorRepositoryBlameCache' => 'applications/repository/storage/PhabricatorRepositoryBlameCache.php', 'PhabricatorRepositoryBranch' => 'applications/repository/storage/PhabricatorRepositoryBranch.php', 'PhabricatorRepositoryCommit' => 'applications/repository/storage/PhabricatorRepositoryCommit.php', 'PhabricatorRepositoryCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php', @@ -5062,6 +5063,7 @@ 'PhabricatorRepositoryDAO', 'PhabricatorPolicyInterface', ), + 'PhabricatorRepositoryBlameCache' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryBranch' => 'PhabricatorRepositoryDAO', 'PhabricatorRepositoryCommit' => array( 'PhabricatorRepositoryDAO', diff --git a/src/applications/diffusion/query/DiffusionCommitQuery.php b/src/applications/diffusion/query/DiffusionCommitQuery.php --- a/src/applications/diffusion/query/DiffusionCommitQuery.php +++ b/src/applications/diffusion/query/DiffusionCommitQuery.php @@ -361,7 +361,7 @@ } if ($refs) { - $callsigns = ipull($refs, 'callsign'); + $callsigns = array_unique(ipull($refs, 'callsign')); $repos = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()) ->withCallsigns($callsigns) diff --git a/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php b/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php --- a/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php +++ b/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php @@ -11,6 +11,7 @@ private $needsBlame; private $fileContent; private $viewer; + private $cachedBlameInfo; final public static function newFromDiffusionRequest( DiffusionRequest $request) { @@ -39,13 +40,62 @@ } final public function loadFileContent() { - return $this->executeQuery(); + if ($this->needsBlame) { + $this->loadBlameCache(); + } + $content = $this->executeQuery(); + return $content; } final public function getRawData() { return $this->fileContent->getCorpus(); } + + /** Blame Cache Functions */ + + // Returns array(pathid, lastmodified commitId) for the queried file + private function getBlameCacheKey() { + $path_id = id( + new DiffusionPathIDQuery( + array($this->getRequest()->getPath()) + ))->loadPathIDs(); + $path_id = array_shift($path_id); + + // FIXME: This one is git specific! Make it work for SVN & Mercurial too + $latest_commit = $this->getRequest()->getRepository() + ->execxLocalCommand('log -n1 --format="%%H" %s -- %s', + $this->getRequest()->getCommit(), + $this->getRequest()->getPath()); + $latest_commit = trim(array_shift($latest_commit)); + + // Get the commitId from the Hash + $commit_id = id(id(new PhabricatorRepositoryCommit()) + ->attachRepository($this->getRequest()->getRepository()) + ->loadOneWhere('commitIdentifier = %s', $latest_commit))->getId(); + + return array($path_id, $commit_id); + } + + private function loadBlameCache() { + $key = $this->getBlameCacheKey(); + $cache_data = + PhabricatorRepositoryBlameCache::loadFromCache($key[0], $key[1]); + if ($cache_data) { + $this->cachedBlameInfo = $cache_data->getBlameDict(); + } + } + + private function writeBlameCache($blame_dict) { + $key = $this->getBlameCacheKey(); + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + PhabricatorRepositoryBlameCache::saveToCache($key[0], $key[1], $blame_dict); + unset($unguarded); + + return true; + } + /** * Pretty hairy function. If getNeedsBlame is false, this returns * @@ -72,58 +122,66 @@ $line_rev_dict = array(); $blame_dict = array(); - if (!$this->getNeedsBlame()) { + if (!$this->getNeedsBlame(true)) { $text_list = explode("\n", $raw_data); } else if ($raw_data != '') { - $lines = array(); - foreach (explode("\n", $raw_data) as $k => $line) { - $lines[$k] = $this->tokenizeLine($line); - - list($rev_id, $author, $text) = $lines[$k]; - $text_list[$k] = $text; - $line_rev_dict[$k] = $rev_id; - } + // Check if we have a cached blame + if ($this->cachedBlameInfo) { + $text_list = explode("\n", $raw_data); + $cache_data = $this->cachedBlameInfo; + + $commits = id(new DiffusionCommitQuery()) + ->setViewer($this->getViewer()) + ->withDefaultRepository($this->getRequest()->getRepository()) + ->withIDs(array_unique($cache_data)) + ->needCommitData(true) + ->execute(); + + // Rebuild line_rev_dict from cache + foreach ($cache_data as $k => $commit_id) { + $line_rev_dict[$k] = $commits[$commit_id]->getCommitIdentifier(); + } + } else { + $lines = array(); + foreach (explode("\n", $raw_data) as $k => $line) { + $lines[$k] = $this->tokenizeLine($line); + + list($rev_id, $author, $text) = $lines[$k]; + $text_list[$k] = $text; + $line_rev_dict[$k] = $rev_id; + } - $line_rev_dict = $this->processRevList($line_rev_dict); + $line_rev_dict = $this->processRevList($line_rev_dict); - foreach ($lines as $k => $line) { - list($rev_id, $author, $text) = $line; - $rev_id = $line_rev_dict[$k]; + $commits = id(new DiffusionCommitQuery()) + ->setViewer($this->getViewer()) + ->withDefaultRepository($this->getRequest()->getRepository()) + ->withIdentifiers(array_unique($line_rev_dict)) + ->needCommitData(true) + ->execute(); - if (!isset($blame_dict[$rev_id])) { - $blame_dict[$rev_id]['author'] = $author; + // Write results to cache + $cache_data = array(); + $commit_identifiers = mpull($commits, 'getID', 'getCommitIdentifier'); + foreach ($line_rev_dict as $k => $line) { + $cache_data[$k] = $commit_identifiers[$line]; } - } - - $repository = $this->getRequest()->getRepository(); - $commits = id(new DiffusionCommitQuery()) - ->setViewer($this->getViewer()) - ->withDefaultRepository($repository) - ->withIdentifiers(array_unique($line_rev_dict)) - ->execute(); + $this->writeBlameCache($cache_data); + } foreach ($commits as $commit) { $blame_dict[$commit->getCommitIdentifier()]['epoch'] = $commit->getEpoch(); - } - if ($commits) { - $commits_data = id(new PhabricatorRepositoryCommitData())->loadAllWhere( - 'commitID IN (%Ls)', - mpull($commits, 'getID')); - - foreach ($commits_data as $data) { - $author_phid = $data->getCommitDetail('authorPHID'); - if (!$author_phid) { - continue; - } - $commit = $commits[$data->getCommitID()]; - $commit_identifier = $commit->getCommitIdentifier(); - $blame_dict[$commit_identifier]['authorPHID'] = $author_phid; + $author_phid = $commit->getCommitData()->getCommitDetail('authorPHID'); + if ($author_phid) { + $blame_dict[$commit->getCommitIdentifier()]['author'] = $author_phid; + } else { + $blame_dict[$commit->getCommitIdentifier()]['author'] = + $commit->getCommitData()->getCommitDetail('authorName'); } } - } return array($text_list, $line_rev_dict, $blame_dict); @@ -136,8 +194,16 @@ return $this; } - public function getNeedsBlame() { - return $this->needsBlame; + public function getNeedsBlame($ignore_cache = false) { + if ($ignore_cache) { + return $this->needsBlame; + } else { + if (!$this->cachedBlameInfo) { + return $this->needsBlame; + } + } + + return false; } public function setViewer(PhabricatorUser $user) { diff --git a/src/applications/repository/storage/PhabricatorRepositoryBlameCache.php b/src/applications/repository/storage/PhabricatorRepositoryBlameCache.php new file mode 100644 --- /dev/null +++ b/src/applications/repository/storage/PhabricatorRepositoryBlameCache.php @@ -0,0 +1,41 @@ + false, + self::CONFIG_TIMESTAMPS => false, + self::CONFIG_BINARY => array( + 'blameDict' => true, + ), + ) + parent::getConfiguration(); + } + + public function willWriteData(array &$data) { + $data['blameDict'] = gzcompress(implode(' ', $data['blameDict'])); + parent::willWriteData($data); + } + + public function willReadData(array &$data) { + $data['blameDict'] = explode(' ', gzuncompress($data['blameDict'])); + parent::willReadData($data); + } + + public static function saveToCache($path_id, $commit_id, $blame_dict) { + return id(new PhabricatorRepositoryBlameCache()) + ->setPathId($path_id) + ->setCommitId($commit_id) + ->setBlameDict($blame_dict) + ->replace(); + } + + public static function loadFromCache($path_id, $commit_id) { + return id(new PhabricatorRepositoryBlameCache()) + ->loadOneWhere('pathId = %d AND commitId = %d', $path_id, $commit_id); + } +}