Index: src/__phutil_library_map__.php =================================================================== --- src/__phutil_library_map__.php +++ src/__phutil_library_map__.php @@ -1863,6 +1863,7 @@ 'PhabricatorRepositoryPHIDTypeMirror' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeMirror.php', 'PhabricatorRepositoryPHIDTypePushLog' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypePushLog.php', 'PhabricatorRepositoryPHIDTypeRepository' => 'applications/repository/phid/PhabricatorRepositoryPHIDTypeRepository.php', + 'PhabricatorRepositoryParsedChange' => 'applications/repository/data/PhabricatorRepositoryParsedChange.php', 'PhabricatorRepositoryPullEngine' => 'applications/repository/engine/PhabricatorRepositoryPullEngine.php', 'PhabricatorRepositoryPullLocalDaemon' => 'applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php', 'PhabricatorRepositoryPushLog' => 'applications/repository/storage/PhabricatorRepositoryPushLog.php', @@ -4523,6 +4524,7 @@ 'PhabricatorRepositoryPHIDTypeMirror' => 'PhabricatorPHIDType', 'PhabricatorRepositoryPHIDTypePushLog' => 'PhabricatorPHIDType', 'PhabricatorRepositoryPHIDTypeRepository' => 'PhabricatorPHIDType', + 'PhabricatorRepositoryParsedChange' => 'Phobject', 'PhabricatorRepositoryPullEngine' => 'PhabricatorRepositoryEngine', 'PhabricatorRepositoryPullLocalDaemon' => 'PhabricatorDaemon', 'PhabricatorRepositoryPushLog' => Index: src/applications/repository/data/PhabricatorRepositoryParsedChange.php =================================================================== --- /dev/null +++ src/applications/repository/data/PhabricatorRepositoryParsedChange.php @@ -0,0 +1,77 @@ +<?php + +final class PhabricatorRepositoryParsedChange extends Phobject { + + private $pathID; + private $targetPathID; + private $targetCommitID; + private $changeType; + private $fileType; + private $isDirect; + private $commitSequence; + + public function setPathID($path_id) { + $this->pathID = $path_id; + return $this; + } + + public function getPathID() { + return $this->pathID; + } + + public function setCommitSequence($commit_sequence) { + $this->commitSequence = $commit_sequence; + return $this; + } + + public function getCommitSequence() { + return $this->commitSequence; + } + + public function setIsDirect($is_direct) { + $this->isDirect = $is_direct; + return $this; + } + + public function getIsDirect() { + return $this->isDirect; + } + + public function setFileType($file_type) { + $this->fileType = $file_type; + return $this; + } + + public function getFileType() { + return $this->fileType; + } + + public function setChangeType($change_type) { + $this->changeType = $change_type; + return $this; + } + + public function getChangeType() { + return $this->changeType; + } + + public function setTargetCommitID($target_commit_id) { + $this->targetCommitID = $target_commit_id; + return $this; + } + + public function getTargetCommitID() { + return $this->targetCommitID; + } + + + public function setTargetPathID($target_path_id) { + $this->targetPathID = $target_path_id; + return $this; + } + + public function getTargetPathID() { + return $this->targetPathID; + } + +} Index: src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php =================================================================== --- src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php +++ src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php @@ -40,8 +40,7 @@ } $this->repository = $repository; - - return $this->parseCommit($repository, $this->commit); + $this->parseCommit($repository, $this->commit); } final protected function shouldQueueFollowupTasks() { Index: src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php =================================================================== --- src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php +++ src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php @@ -24,14 +24,15 @@ $this->log("Parsing %s...\n", $full_name); if ($this->isBadCommit($full_name)) { $this->log("This commit is marked bad!"); - $result = null; - } else { - $result = $this->parseCommitChanges($repository, $commit); + return; } - $this->finishParse(); + $results = $this->parseCommitChanges($repository, $commit); + if ($results) { + $this->writeCommitChanges($repository, $commit, $results); + } - return $result; + $this->finishParse(); } public static function lookupOrCreatePaths(array $paths) { @@ -99,4 +100,52 @@ } } + private function writeCommitChanges( + PhabricatorRepository $repository, + PhabricatorRepositoryCommit $commit, + array $changes) { + + $repository_id = (int)$repository->getID(); + $commit_id = (int)$commit->getID(); + + // NOTE: This SQL is being built manually instead of with qsprintf() + // because some SVN changes affect an enormous number of paths (millions) + // and this showed up as significantly slow on a profile at some point. + + $changes_sql = array(); + foreach ($changes as $change) { + $values = array( + $repository_id, + (int)$change->getPathID(), + $commit_id, + nonempty((int)$change->getTargetPathID(), 'null'), + nonempty((int)$change->getTargetCommitID(), 'null'), + (int)$change->getChangeType(), + (int)$change->getFileType(), + (int)$change->getIsDirect(), + (int)$change->getCommitSequence(), + ); + $changes_sql[] = '('.implode(', ', $values).')'; + } + + $conn_w = $repository->establishConnection('w'); + + queryfx( + $conn_w, + 'DELETE FROM %T WHERE commitID = %d', + PhabricatorRepository::TABLE_PATHCHANGE, + $commit_id); + + foreach (PhabricatorLiskDAO::chunkSQL($changes_sql) as $chunk) { + queryfx( + $conn_w, + 'INSERT INTO %T + (repositoryID, pathID, commitID, targetPathID, targetCommitID, + changeType, fileType, isDirect, commitSequence) + VALUES %Q', + PhabricatorRepository::TABLE_PATHCHANGE, + $chunk); + } + } + } Index: src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php =================================================================== --- src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php +++ src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php @@ -228,43 +228,29 @@ } } - $conn_w = $repository->establishConnection('w'); - $changes_sql = array(); + $results = array(); foreach ($changes as $change) { - $values = array( - (int)$change['repositoryID'], - (int)$change['pathID'], - (int)$change['commitID'], - $change['targetPathID'] - ? (int)$change['targetPathID'] - : 'null', - $change['targetCommitID'] - ? (int)$change['targetCommitID'] - : 'null', - (int)$change['changeType'], - (int)$change['fileType'], - (int)$change['isDirect'], - (int)$change['commitSequence'], - ); - $changes_sql[] = '('.implode(', ', $values).')'; + $target_path_id = $change['targetPathID'] + ? (int)$change['targetPathID'] + : null; + + $target_commit_id = $change['targetCommitID'] + ? (int)$change['targetCommitID'] + : null; + + $result = id(new PhabricatorRepositoryParsedChange()) + ->setPathID((int)$change['pathID']) + ->setTargetPathID($target_path_id) + ->setTargetCommitID($target_commit_id) + ->setChangeType((int)$change['changeType']) + ->setIsDirect((int)$change['isDirect']) + ->setCommitSequence((int)$change['commitSequence']); + + $results[] = $result; } - queryfx( - $conn_w, - 'DELETE FROM %T WHERE commitID = %d', - PhabricatorRepository::TABLE_PATHCHANGE, - $commit->getID()); - foreach (array_chunk($changes_sql, 256) as $sql_chunk) { - queryfx( - $conn_w, - 'INSERT INTO %T - (repositoryID, pathID, commitID, targetPathID, targetCommitID, - changeType, fileType, isDirect, commitSequence) - VALUES %Q', - PhabricatorRepository::TABLE_PATHCHANGE, - implode(', ', $sql_chunk)); - } + return $results; } } Index: src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php =================================================================== --- src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php +++ src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php @@ -300,6 +300,8 @@ PhabricatorRepository::TABLE_PATHCHANGE, implode(', ', $sql_chunk)); } + + return array(); } private function mercurialPathExists( Index: src/applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php =================================================================== --- src/applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php +++ src/applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php @@ -359,6 +359,8 @@ $this->writeChanges($repository, $commit, $effects, $path_map, $commit_map); $this->writeBrowse($repository, $commit, $effects, $path_map); + + return array(); } private function writeChanges(