Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F15387065
D11871.id.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
8 KB
Referenced Files
None
Subscribers
None
D11871.id.diff
View Options
diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
--- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
+++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
@@ -208,6 +208,9 @@
'longer used or supported.'),
'config.mask' => pht(
'Use `config.hide` instead of this option.'),
+ 'phd.start-taskmasters' => pht(
+ 'Taskmasters now use an autoscaling pool. You can configure the '.
+ 'pool size with `phd.taskmasters`.'),
);
return $ancient_config;
diff --git a/src/applications/config/option/PhabricatorPHDConfigOptions.php b/src/applications/config/option/PhabricatorPHDConfigOptions.php
--- a/src/applications/config/option/PhabricatorPHDConfigOptions.php
+++ b/src/applications/config/option/PhabricatorPHDConfigOptions.php
@@ -29,13 +29,13 @@
->setDescription(
pht(
'Directory that the daemons should use to store log files.')),
- $this->newOption('phd.start-taskmasters', 'int', 4)
- ->setSummary(pht('Number of TaskMaster daemons to start by default.'))
+ $this->newOption('phd.taskmasters', 'int', 4)
+ ->setSummary(pht('Maximum taskmaster daemon pool size.'))
->setDescription(
pht(
- "Number of 'TaskMaster' daemons that 'phd start' should start. ".
- "You can raise this if you have a task backlog, or explicitly ".
- "launch more with 'phd launch <N> taskmaster'.")),
+ 'Maximum number of taskmaster daemons to run at once. Raising '.
+ 'this can increase the maximum throughput of the task queue. The '.
+ 'pool will automatically scale down when unutilized.')),
$this->newOption('phd.verbose', 'bool', false)
->setBoolOptions(
array(
diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php
--- a/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php
+++ b/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php
@@ -31,6 +31,7 @@
'Also stop running processes that look like daemons but do '.
'not have corresponding PID files.'),
),
+ $this->getAutoscaleReserveArgument(),
));
}
@@ -46,7 +47,10 @@
return $err;
}
- return $this->executeStartCommand();
+ return $this->executeStartCommand(
+ array(
+ 'reserve' => (float)$args->getArg('autoscale-reserve', 0.0),
+ ));
}
}
diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php
--- a/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php
+++ b/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php
@@ -24,13 +24,17 @@
'help' => pht(
'Start daemons even if daemons are already running.'),
),
+ $this->getAutoscaleReserveArgument(),
));
}
public function execute(PhutilArgumentParser $args) {
return $this->executeStartCommand(
- $args->getArg('keep-leases'),
- $args->getArg('force'));
+ array(
+ 'keep-leases' => $args->getArg('keep-leases'),
+ 'force' => $args->getArg('force'),
+ 'reserve' => (float)$args->getArg('autoscale-reserve', 0.0),
+ ));
}
}
diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
--- a/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
+++ b/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
@@ -290,10 +290,18 @@
/* -( Commands )----------------------------------------------------------- */
- protected final function executeStartCommand($keep_leases, $force) {
+ protected final function executeStartCommand(array $options) {
+ PhutilTypeSpec::checkMap(
+ $options,
+ array(
+ 'keep-leases' => 'optional bool',
+ 'force' => 'optional bool',
+ 'reserve' => 'optional float',
+ ));
+
$console = PhutilConsole::getConsole();
- if (!$force) {
+ if (!idx($options, 'force')) {
$running = $this->loadRunningDaemons();
// This may include daemons which were launched but which are no longer
@@ -315,7 +323,7 @@
}
}
- if ($keep_leases) {
+ if (idx($options, 'keep-leases')) {
$console->writeErr("%s\n", pht('Not touching active task queue leases.'));
} else {
$console->writeErr("%s\n", pht('Freeing active task leases...'));
@@ -335,14 +343,15 @@
array(
'class' => 'PhabricatorTriggerDaemon',
),
- );
-
- $taskmasters = PhabricatorEnv::getEnvConfig('phd.start-taskmasters');
- for ($ii = 0; $ii < $taskmasters; $ii++) {
- $daemons[] = array(
+ array(
'class' => 'PhabricatorTaskmasterDaemon',
- );
- }
+ 'autoscale' => array(
+ 'group' => 'task',
+ 'pool' => PhabricatorEnv::getEnvConfig('phd.taskmasters'),
+ 'reserve' => idx($options, 'reserve', 0),
+ ),
+ ),
+ );
$this->launchDaemons($daemons, $is_debug = false);
@@ -577,7 +586,13 @@
foreach ($daemons as $daemon) {
$is_autoscale = isset($daemon['autoscale']['group']);
if ($is_autoscale) {
- $autoscale = pht('(Autoscaling)');
+ $autoscale = $daemon['autoscale'];
+ foreach ($autoscale as $key => $value) {
+ $autoscale[$key] = $key.'='.$value;
+ }
+ $autoscale = implode(', ', $autoscale);
+
+ $autoscale = pht('(Autoscaling: %s)', $autoscale);
} else {
$autoscale = pht('(Static)');
}
@@ -591,4 +606,16 @@
$console->writeOut("\n");
}
+ protected function getAutoscaleReserveArgument() {
+ return array(
+ 'name' => 'autoscale-reserve',
+ 'param' => 'ratio',
+ 'help' => pht(
+ 'Specify a proportion of machine memory which must be free '.
+ 'before autoscale pools will grow. For example, a value of 0.25 '.
+ 'means that pools will not grow unless the machine has at least '.
+ '25%% of its RAM free.'),
+ );
+ }
+
}
diff --git a/src/docs/user/configuration/managing_daemons.diviner b/src/docs/user/configuration/managing_daemons.diviner
--- a/src/docs/user/configuration/managing_daemons.diviner
+++ b/src/docs/user/configuration/managing_daemons.diviner
@@ -99,8 +99,9 @@
== General Tips ==
- - You can set the number of taskmasters that `phd start` starts by the config
- key `phd.start-taskmasters`. If you have a task backlog, try increasing it.
+ - You can set the maximum number of taskmasters that will run at once
+ by adjusting `phd.taskmasters`. If you have a task backlog, try increasing
+ it.
- When you `phd launch` or `phd debug` a daemon, you can type any unique
substring of its name, so `phd launch pull` will work correctly.
- `phd stop` and `phd restart` stop **all** of the daemons on the machine, not
diff --git a/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
--- a/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
+++ b/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php
@@ -3,9 +3,6 @@
final class PhabricatorTaskmasterDaemon extends PhabricatorDaemon {
protected function run() {
- $taskmaster_count = PhabricatorEnv::getEnvConfig('phd.start-taskmasters');
- $offset = mt_rand(0, $taskmaster_count - 1);
-
do {
$tasks = id(new PhabricatorWorkerLeaseQuery())
->setLimit(1)
@@ -44,22 +41,11 @@
$sleep = 0;
} else {
- // When there's no work, sleep for as many seconds as there are
- // active taskmasters.
-
- // On average, this starts tasks added to an empty queue after one
- // second. This keeps responsiveness high even on small instances
- // without much work to do.
-
- // It also means an empty queue has an average load of one query
- // per second even if there are a very large number of taskmasters
- // launched.
-
- // The first time we sleep, we add a random offset to try to spread
- // the sleep times out somewhat evenly.
+ // When there's no work, sleep for one second. The pool will
+ // autoscale down if we're continuously idle for an extended period
+ // of time.
$this->willBeginIdle();
- $sleep = $taskmaster_count + $offset;
- $offset = 0;
+ $sleep = 1;
}
$this->sleep($sleep);
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Mar 16, 1:46 AM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7705032
Default Alt Text
D11871.id.diff (8 KB)
Attached To
Mode
D11871: Convert taskmasters to use an autoscale pool
Attached
Detach File
Event Timeline
Log In to Comment