Page MenuHomePhabricator
Files F14066107

D10323.diff
No OneTemporary
Actions

D10323.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -475,6 +475,7 @@
'DiffusionRepositoryEditLocalController' => 'applications/diffusion/controller/DiffusionRepositoryEditLocalController.php',
'DiffusionRepositoryEditMainController' => 'applications/diffusion/controller/DiffusionRepositoryEditMainController.php',
'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php',
+ 'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php',
'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php',
'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php',
'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php',
@@ -3223,6 +3224,7 @@
'DiffusionRepositoryEditLocalController' => 'DiffusionRepositoryEditController',
'DiffusionRepositoryEditMainController' => 'DiffusionRepositoryEditController',
'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController',
+ 'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController',
'DiffusionRepositoryListController' => 'DiffusionController',
'DiffusionRepositoryNewController' => 'DiffusionController',
'DiffusionRepositoryRef' => 'Phobject',
diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php
--- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php
+++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php
@@ -92,6 +92,7 @@
'delete/' => 'DiffusionRepositoryEditDeleteController',
'hosting/' => 'DiffusionRepositoryEditHostingController',
'(?P<serve>serve)/' => 'DiffusionRepositoryEditHostingController',
+ 'update/' => 'DiffusionRepositoryEditUpdateController',
),
'pathtree/(?P<dblob>.*)' => 'DiffusionPathTreeController',
'mirror/' => array(
diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php
--- a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php
+++ b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php
@@ -207,6 +207,14 @@
->setHref($this->getRepositoryControllerURI($repository, 'edit/basic/'));
$view->addAction($edit);
+ $edit = id(new PhabricatorActionView())
+ ->setIcon('fa-refresh')
+ ->setName(pht('Update Now'))
+ ->setWorkflow(true)
+ ->setHref(
+ $this->getRepositoryControllerURI($repository, 'edit/update/'));
+ $view->addAction($edit);
+
$activate = id(new PhabricatorActionView())
->setHref(
$this->getRepositoryControllerURI($repository, 'edit/activate/'))
@@ -280,6 +288,10 @@
pht('Status'),
$this->buildRepositoryStatus($repository));
+ $view->addProperty(
+ pht('Update Frequency'),
+ $this->buildRepositoryUpdateInterval($repository));
+
$description = $repository->getDetail('description');
$view->addSectionHeader(pht('Description'));
if (!strlen($description)) {
@@ -942,14 +954,16 @@
->setNote($message->getParameter('message')));
return $view;
case PhabricatorRepositoryStatusMessage::CODE_OKAY:
+ $ago = (PhabricatorTime::getNow() - $message->getEpoch());
$view->addItem(
id(new PHUIStatusItemView())
->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green')
->setTarget(pht('Updates OK'))
->setNote(
pht(
- 'Last updated %s.',
- phabricator_datetime($message->getEpoch(), $viewer))));
+ 'Last updated %s (%s ago).',
+ phabricator_datetime($message->getEpoch(), $viewer),
+ phutil_format_relative_time_detailed($ago))));
break;
}
} else {
@@ -1020,12 +1034,34 @@
id(new PHUIStatusItemView())
->setIcon(PHUIStatusItemView::ICON_UP, 'indigo')
->setTarget(pht('Prioritized'))
- ->setNote(pht('This repository will be updated soon.')));
+ ->setNote(pht('This repository will be updated soon!')));
}
return $view;
}
+ private function buildRepositoryUpdateInterval(
+ PhabricatorRepository $repository) {
+
+ $smart_wait = $repository->loadUpdateInterval();
+
+ $doc_href = PhabricatorEnv::getDoclink(
+ 'Diffusion User Guide: Repository Updates');
+
+ return array(
+ phutil_format_relative_time_detailed($smart_wait),
+ " \xC2\xB7 ",
+ phutil_tag(
+ 'a',
+ array(
+ 'href' => $doc_href,
+ 'target' => '_blank',
+ ),
+ pht('Learn More')),
+ );
+ }
+
+
private function buildMirrorActions(
PhabricatorRepository $repository) {
diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php
new file mode 100644
--- /dev/null
+++ b/src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php
@@ -0,0 +1,75 @@
+<?php
+
+final class DiffusionRepositoryEditUpdateController
+ extends DiffusionRepositoryEditController {
+
+ public function processRequest() {
+ $request = $this->getRequest();
+ $viewer = $request->getUser();
+ $drequest = $this->diffusionRequest;
+ $repository = $drequest->getRepository();
+
+ $repository = id(new PhabricatorRepositoryQuery())
+ ->setViewer($viewer)
+ ->requireCapabilities(
+ array(
+ PhabricatorPolicyCapability::CAN_VIEW,
+ PhabricatorPolicyCapability::CAN_EDIT,
+ ))
+ ->withIDs(array($repository->getID()))
+ ->executeOne();
+ if (!$repository) {
+ return new Aphront404Response();
+ }
+
+ $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/');
+
+ if ($request->isFormPost()) {
+ $params = array(
+ 'callsigns' => array(
+ $repository->getCallsign(),
+ ),
+ );
+
+ id(new ConduitCall('diffusion.looksoon', $params))
+ ->setUser($viewer)
+ ->execute();
+
+ return id(new AphrontRedirectResponse())->setURI($edit_uri);
+ }
+
+ $doc_name = 'Diffusion User Guide: Repository Updates';
+ $doc_href = PhabricatorEnv::getDoclink($doc_name);
+ $doc_link = phutil_tag(
+ 'a',
+ array(
+ 'href' => $doc_href,
+ 'target' => '_blank',
+ ),
+ $doc_name);
+
+ return $this->newDialog()
+ ->setTitle(pht('Update Repository Now'))
+ ->appendParagraph(
+ pht(
+ 'Normally, Phabricator automatically updates repositories '.
+ 'based on how much time has elapsed since the last commit. '.
+ 'This helps reduce load if you have a large number of mostly '.
+ 'inactive repositories, which is common.'))
+ ->appendParagraph(
+ pht(
+ 'You can manually schedule an update for this repository. The '.
+ 'daemons will perform the update as soon as possible. This may '.
+ 'be helpful if you have just made a commit to a rarely used '.
+ 'repository.'))
+ ->appendParagraph(
+ pht(
+ 'To learn more about how Phabricator updates repositories, '.
+ 'read %s in the documentation.',
+ $doc_link))
+ ->addCancelButton($edit_uri)
+ ->addSubmitButton(pht('Schedule Update'));
+ }
+
+
+}
diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
--- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
@@ -345,55 +345,23 @@
phlog($stderr_msg);
}
- $sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep);
-
- // Smart wait: pull rarely used repositories less frequently. Find the
- // most recent commit which is older than the current time (this keeps us
- // from spinning on repositories with a silly commit post-dated to some time
- // in 2037), and adjust how frequently we pull based on how frequently this
- // repository updates.
-
- $table = id(new PhabricatorRepositoryCommit());
- $last_commit = queryfx_one(
- $table->establishConnection('w'),
- 'SELECT epoch FROM %T
- WHERE repositoryID = %d AND epoch <= %d
- ORDER BY epoch DESC LIMIT 1',
- $table->getTableName(),
- $repository->getID(),
- time() + $min_sleep);
- if ($last_commit) {
- $time_since_commit = (time() + $min_sleep) - $last_commit['epoch'];
-
- // Wait 0.5% of the time since the last commit before we pull. This gives
- // us these wait times:
- //
- // 50 minutes or less: 15 seconds
- // about 3 hours: 1 minute
- // about 16 hours: 5 minutes
- // about 2 days: 15 minutes
- // 50 days or more: 6 hours
-
- $smart_wait = ($time_since_commit / 200);
- $smart_wait = min($smart_wait, phutil_units('6 hours in seconds'));
+ // For now, continue respecting this deprecated setting for raising the
+ // minimum pull frequency.
+ // TODO: Remove this some day once this code has been completely stable
+ // for a while.
+ $sleep_for = (int)$repository->getDetail('pull-frequency');
+ $min_sleep = max($sleep_for, $min_sleep);
- $this->log(
- pht(
- 'Last commit to repository "%s" was %s seconds ago; considering '.
- 'a wait of %s seconds before update.',
- $repository->getMonogram(),
- new PhutilNumber($time_since_commit),
- new PhutilNumber($smart_wait)));
-
- $smart_wait = max(15, $smart_wait);
- $sleep_for = max($smart_wait, $sleep_for);
- }
+ $smart_wait = $repository->loadUpdateInterval($min_sleep);
- if ($sleep_for < $min_sleep) {
- $sleep_for = $min_sleep;
- }
+ $this->log(
+ pht(
+ 'Based on activity in repository "%s", considering a wait of %s '.
+ 'seconds before update.',
+ $repository->getMonogram(),
+ new PhutilNumber($smart_wait)));
- return time() + $sleep_for;
+ return time() + $smart_wait;
}
diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php
--- a/src/applications/repository/storage/PhabricatorRepository.php
+++ b/src/applications/repository/storage/PhabricatorRepository.php
@@ -1319,6 +1319,69 @@
}
+ /**
+ * Load the pull frequency for this repository, based on the time since the
+ * last activity.
+ *
+ * We pull rarely used repositories less frequently. This finds the most
+ * recent commit which is older than the current time (which prevents us from
+ * spinning on repositories with a silly commit post-dated to some time in
+ * 2037). We adjust the pull frequency based on when the most recent commit
+ * occurred.
+ *
+ * @param int The minimum update interval to use, in seconds.
+ * @return int Repository update interval, in seconds.
+ */
+ public function loadUpdateInterval($minimum = 15) {
+ // If a repository is still importing, always pull it as frequently as
+ // possible. This prevents us from hanging for a long time at 99.9% when
+ // importing an inactive repository.
+ if ($this->isImporting()) {
+ return $minimum;
+ }
+
+ $window_start = (PhabricatorTime::getNow() + $minimum);
+
+ $table = id(new PhabricatorRepositoryCommit());
+ $last_commit = queryfx_one(
+ $table->establishConnection('r'),
+ 'SELECT epoch FROM %T
+ WHERE repositoryID = %d AND epoch <= %d
+ ORDER BY epoch DESC LIMIT 1',
+ $table->getTableName(),
+ $this->getID(),
+ $window_start);
+ if ($last_commit) {
+ $time_since_commit = ($window_start - $last_commit['epoch']);
+
+ $last_few_days = phutil_units('3 days in seconds');
+
+ if ($time_since_commit <= $last_few_days) {
+ // For repositories with activity in the recent past, we wait one
+ // extra second for every 10 minutes since the last commit. This
+ // shorter backoff is intended to handle weekends and other short
+ // breaks from development.
+ $smart_wait = ($time_since_commit / 600);
+ } else {
+ // For repositories without recent activity, we wait one extra second
+ // for every 4 minutes since the last commit. This longer backoff
+ // handles rarely used repositories, up to the maximum.
+ $smart_wait = ($time_since_commit / 240);
+ }
+
+ // We'll never wait more than 6 hours to pull a repository.
+ $longest_wait = phutil_units('6 hours in seconds');
+ $smart_wait = min($smart_wait, $longest_wait);
+
+ $smart_wait = max($minimum, $smart_wait);
+ } else {
+ $smart_wait = $minimum;
+ }
+
+ return $smart_wait;
+ }
+
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
diff --git a/src/docs/user/userguide/diffusion_updates.diviner b/src/docs/user/userguide/diffusion_updates.diviner
new file mode 100644
--- /dev/null
+++ b/src/docs/user/userguide/diffusion_updates.diviner
@@ -0,0 +1,103 @@
+@title Diffusion User Guide: Repository Updates
+@group userguide
+
+Explains how Diffusion updates repositories to discover new changes.
+
+Overview
+========
+
+When Phabricator is configured to import repositories which are hosted
+elsewhere, it needs to poll those repositories for changes. If it polls too
+frequently, it can create too much load locally and on remote services. If it
+polls too rarely, it may take a long time for commits to show up in the web
+interface.
+
+This document describes the rules around polling and how to understand and
+adjust the behavior. In general:
+
+ - Phabricator chooses a default poll interval based on repository
+ activity. These intervals range from every 15 seconds (for active
+ repositories) to every 6 hours (for repositories with no commits in two
+ months).
+ - If you use `arc` to push commits, or you host repositories on Phabricator,
+ repositories automatically update after changes are pushed.
+ - If you don't use `arc` and your repository is hosted elsewhere, this
+ document describes ways you can make polling more responsive.
+
+
+Default Behavior
+================
+
+By default, Phabricator determines how frequently to poll repositories by
+examining how long it has been since the last commit. In most cases this is
+fairly accurate and produces good behavior. In particular, it automatically
+reduces the polling frequency for rarely-used repositories. This dramatically
+reduces load for installs with a large number of inactive repositories, which
+is common.
+
+For repositories with activity in the last 3 days, we wait 1 second for every
+10 minutes without activity. The table below has some examples.
+
+| Time Since Commit | Poll Interval |
+|-------------------|------------------|
+| //Minimum// | 15 seconds |
+| 6h | about 30 seconds |
+| 12h | about 1 minute |
+| 1 day | about 2 minutes |
+| 2 days | about 5 minutes |
+| 3 days | about 7 minutes |
+
+This means that you may need to wait about 2 minutes for the first commit to
+be imported in the morning, and about 5 minutes after a long weekend, but other
+commits to active repositories should usually be recognized in 30 seconds or
+less.
+
+For repositories with no activity in the last 3 days, we wait longer between
+updates (1 second for every 4 minutes without activity). The table below has
+some examples.
+
+| Time Since Commit | Poll Interval |
+|-------------------|------------------|
+| 4 days | about 30 minutes |
+| 7 days | about 45 minutes |
+| 10 days | about 1 hour |
+| 20 days | about 2 hours |
+| 30 days | about 3 hours |
+| //Maximum// | 6 hours |
+
+You can find the exact default poll frequency of a repository in
+Diffusion > (Choose a Repository) > Edit Repository, under "Update Frequency".
+You can also see the time when the repository was last updated in this
+interface.
+
+Repositories that are currently importing are always updated at the minimum
+update frequency so the import finishes as quickly as possible.
+
+
+Triggering Repository Updates
+=============================
+
+If you want Phabricator to update a repository more quickly than the default
+update frequency (for example, because you just pushed a commit to it), you can
+tell Phabricator that it should schedule an update as soon as possible.
+
+There are several ways to do this:
+
+ - If you push changes with `arc land` or `arc commit`, this will be done
+ for you automatically. These commits should normally be recognized within
+ a few seconds.
+ - If your repository is hosted on Phabricator, this will also be done for you
+ automatically.
+ - You can schedule an update from the web interface, in Diffusion >
+ (Choose a Repository) > Edit Repository > Update Now.
+ - You can make a call to the Conduit API method `diffusion.looksoon`. This
+ hints to Phabricator that it should poll a repository as soon as it can.
+ All of the other mechanisms do this under the hood.
+
+In particular, you may be able to add a commit hook to your external repository
+which calls `diffusion.looksoon`. This should make an external repository about
+as responsive as a hosted repository.
+
+If a repository has an update scheduled, the Diffusion > (Choose a
+Repository) > Edit Repository interface will show that the the repository is
+prioritized and will be updated soon.

File Metadata

Mime Type
text/plain
Expires
Wed, Nov 20, 8:12 AM (21 h, 33 m)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
6717336
Default Alt Text
D10323.diff (17 KB)

Event Timeline

Phabricator · Built by Phacility