Differential D21516 Diff 51210 src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
Changeset View
Changeset View
Standalone View
Standalone View
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
| Show First 20 Lines • Show All 649 Lines • ▼ Show 20 Lines | /* -( Internals )---------------------------------------------------------- */ | ||||
| } | } | ||||
| private function didDiscoverCommit( | private function didDiscoverCommit( | ||||
| PhabricatorRepository $repository, | PhabricatorRepository $repository, | ||||
| PhabricatorRepositoryCommit $commit, | PhabricatorRepositoryCommit $commit, | ||||
| $epoch, | $epoch, | ||||
| $task_priority) { | $task_priority) { | ||||
| $this->insertTask($repository, $commit, $task_priority); | $this->queueCommitImportTask( | ||||
| $repository, | |||||
| $commit->getID(), | |||||
| $commit->getPHID(), | |||||
| $task_priority, | |||||
| $via = 'discovery'); | |||||
| // Update the repository summary table. | // Update the repository summary table. | ||||
| queryfx( | queryfx( | ||||
| $commit->establishConnection('w'), | $commit->establishConnection('w'), | ||||
| 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) | 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch) | ||||
| VALUES (%d, 1, %d, %d) | VALUES (%d, 1, %d, %d) | ||||
| ON DUPLICATE KEY UPDATE | ON DUPLICATE KEY UPDATE | ||||
| size = size + 1, | size = size + 1, | ||||
| lastCommitID = | lastCommitID = | ||||
| IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), | IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID), | ||||
| epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', | epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)', | ||||
| PhabricatorRepository::TABLE_SUMMARY, | PhabricatorRepository::TABLE_SUMMARY, | ||||
| $repository->getID(), | $repository->getID(), | ||||
| $commit->getID(), | $commit->getID(), | ||||
| $epoch); | $epoch); | ||||
| } | } | ||||
| private function didDiscoverRefs(array $refs) { | private function didDiscoverRefs(array $refs) { | ||||
| foreach ($refs as $ref) { | foreach ($refs as $ref) { | ||||
| $this->workingSet[$ref->getIdentifier()] = true; | $this->workingSet[$ref->getIdentifier()] = true; | ||||
| } | } | ||||
| } | } | ||||
| private function insertTask( | |||||
| PhabricatorRepository $repository, | |||||
| PhabricatorRepositoryCommit $commit, | |||||
| $task_priority, | |||||
| $data = array()) { | |||||
| $vcs = $repository->getVersionControlSystem(); | |||||
| switch ($vcs) { | |||||
| case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: | |||||
| $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; | |||||
| break; | |||||
| case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: | |||||
| $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; | |||||
| break; | |||||
| case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: | |||||
| $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; | |||||
| break; | |||||
| default: | |||||
| throw new Exception(pht("Unknown repository type '%s'!", $vcs)); | |||||
| } | |||||
| $data['commitID'] = $commit->getID(); | |||||
| $options = array( | |||||
| 'priority' => $task_priority, | |||||
| ); | |||||
| PhabricatorWorker::scheduleTask($class, $data, $options); | |||||
| } | |||||
| private function isInitialImport(array $refs) { | private function isInitialImport(array $refs) { | ||||
| $commit_count = count($refs); | $commit_count = count($refs); | ||||
| if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) { | if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) { | ||||
| // If we fetched a small number of commits, assume it's an initial | // If we fetched a small number of commits, assume it's an initial | ||||
| // commit or a stack of a few initial commits. | // commit or a stack of a few initial commits. | ||||
| return false; | return false; | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 201 Lines • ▼ Show 20 Lines | queryfx( | ||||
| epoch = VALUES(epoch)', | epoch = VALUES(epoch)', | ||||
| PhabricatorRepository::TABLE_SUMMARY, | PhabricatorRepository::TABLE_SUMMARY, | ||||
| $repository->getID(), | $repository->getID(), | ||||
| $data['N'], | $data['N'], | ||||
| $data['id'], | $data['id'], | ||||
| $data['epoch']); | $data['epoch']); | ||||
| } | } | ||||
| private function getImportTaskPriority( | |||||
| PhabricatorRepository $repository, | |||||
| array $refs) { | |||||
| // If the repository is importing for the first time, we schedule tasks | |||||
| // at IMPORT priority, which is very low. Making progress on importing a | |||||
| // new repository for the first time is less important than any other | |||||
| // daemon task. | |||||
| // If the repository has finished importing and we're just catching up | |||||
| // on recent commits, we usually schedule discovery at COMMIT priority, | |||||
| // which is slightly below the default priority. | |||||
| // Note that followup tasks and triggered tasks (like those generated by | |||||
| // Herald or Harbormaster) will queue at DEFAULT priority, so that each | |||||
| // commit tends to fully import before we start the next one. This tends | |||||
| // to give imports fairly predictable progress. See T11677 for some | |||||
| // discussion. | |||||
| if ($repository->isImporting()) { | |||||
| $this->log( | |||||
| pht( | |||||
| 'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '. | |||||
| 'because this repository is still importing.', | |||||
| phutil_count($refs))); | |||||
| return PhabricatorWorker::PRIORITY_IMPORT; | |||||
| } | |||||
| // See T13369. If we've discovered a lot of commits at once, import them | |||||
| // at lower priority. | |||||
| // This is mostly aimed at reducing the impact that synchronizing thousands | |||||
| // of commits from a remote upstream has on other repositories. The queue | |||||
| // is "mostly FIFO", so queueing a thousand commit imports can stall other | |||||
| // repositories. | |||||
| // In a perfect world we'd probably give repositories round-robin queue | |||||
| // priority, but we don't currently have the primitives for this and there | |||||
| // isn't a strong case for building them. | |||||
| // Use "a whole lot of commits showed up at once" as a heuristic for | |||||
| // detecting "someone synchronized an upstream", and import them at a lower | |||||
| // priority to more closely approximate fair scheduling. | |||||
| if (count($refs) >= PhabricatorRepository::LOWPRI_THRESHOLD) { | |||||
| $this->log( | |||||
| pht( | |||||
| 'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '. | |||||
| 'because many commits were discovered at once.', | |||||
| phutil_count($refs))); | |||||
| return PhabricatorWorker::PRIORITY_IMPORT; | |||||
| } | |||||
| // Otherwise, import at normal priority. | |||||
| if ($refs) { | |||||
| $this->log( | |||||
| pht( | |||||
| 'Importing %s commit(s) at normal priority ("PRIORITY_COMMIT").', | |||||
| phutil_count($refs))); | |||||
| } | |||||
| return PhabricatorWorker::PRIORITY_COMMIT; | |||||
| } | |||||
| } | } | ||||