diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php index 29e6752bc1..78805c218d 100644 --- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php +++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php @@ -1,215 +1,218 @@ newIssue('config.unknown.'.$key) ->setShortName($short) ->setName($name) ->setSummary($summary); $stack = PhabricatorEnv::getConfigSourceStack(); $stack = $stack->getStack(); $found = array(); $found_local = false; $found_database = false; foreach ($stack as $source_key => $source) { $value = $source->getKeys(array($key)); if ($value) { $found[] = $source->getName(); if ($source instanceof PhabricatorConfigDatabaseSource) { $found_database = true; } if ($source instanceof PhabricatorConfigLocalSource) { $found_local = true; } } } $message = $message."\n\n".pht( 'This configuration value is defined in these %d '. 'configuration source(s): %s.', count($found), implode(', ', $found)); $issue->setMessage($message); if ($found_local) { $command = csprintf('phabricator/ $ ./bin/config delete %s', $key); $issue->addCommand($command); } if ($found_database) { $issue->addPhabricatorConfig($key); } } } /** * Return a map of deleted config options. Keys are option keys; values are * explanations of what happened to the option. */ public static function getAncientConfig() { $reason_auth = pht( 'This option has been migrated to the "Auth" application. Your old '. 'configuration is still in effect, but now stored in "Auth" instead of '. 'configuration. Going forward, you can manage authentication from '. 'the web UI.'); $auth_config = array( 'controller.oauth-registration', 'auth.password-auth-enabled', 'facebook.auth-enabled', 'facebook.registration-enabled', 'facebook.auth-permanent', 'facebook.application-id', 'facebook.application-secret', 'facebook.require-https-auth', 'github.auth-enabled', 'github.registration-enabled', 'github.auth-permanent', 'github.application-id', 'github.application-secret', 'google.auth-enabled', 'google.registration-enabled', 'google.auth-permanent', 'google.application-id', 'google.application-secret', 'ldap.auth-enabled', 'ldap.hostname', 'ldap.port', 'ldap.base_dn', 'ldap.search_attribute', 'ldap.search-first', 'ldap.username-attribute', 'ldap.real_name_attributes', 'ldap.activedirectory_domain', 'ldap.version', 'ldap.referrals', 'ldap.anonymous-user-name', 'ldap.anonymous-user-password', 'ldap.start-tls', 'disqus.auth-enabled', 'disqus.registration-enabled', 'disqus.auth-permanent', 'disqus.application-id', 'disqus.application-secret', 'phabricator.oauth-uri', 'phabricator.auth-enabled', 'phabricator.registration-enabled', 'phabricator.auth-permanent', 'phabricator.application-id', 'phabricator.application-secret', ); $ancient_config = array_fill_keys($auth_config, $reason_auth); $markup_reason = pht( 'Custom remarkup rules are now added by subclassing '. 'PhabricatorRemarkupCustomInlineRule or '. 'PhabricatorRemarkupCustomBlockRule.'); $session_reason = pht( 'Sessions now expire and are garbage collected rather than having an '. 'arbitrary concurrency limit.'); $differential_field_reason = pht( 'All Differential fields are now managed through the configuration '. 'option "%s". Use that option to configure which fields are shown.', 'differential.fields'); $ancient_config += array( 'phid.external-loaders' => pht( 'External loaders have been replaced. Extend `PhabricatorPHIDType` '. 'to implement new PHID and handle types.'), 'maniphest.custom-task-extensions-class' => pht( 'Maniphest fields are now loaded automatically. You can configure '. 'them with `maniphest.fields`.'), 'maniphest.custom-fields' => pht( 'Maniphest fields are now defined in '. '`maniphest.custom-field-definitions`. Existing definitions have '. 'been migrated.'), 'differential.custom-remarkup-rules' => $markup_reason, 'differential.custom-remarkup-block-rules' => $markup_reason, 'auth.sshkeys.enabled' => pht( 'SSH keys are now actually useful, so they are always enabled.'), 'differential.anonymous-access' => pht( 'Phabricator now has meaningful global access controls. See '. '`policy.allow-public`.'), 'celerity.resource-path' => pht( 'An alternate resource map is no longer supported. Instead, use '. 'multiple maps. See T4222.'), 'metamta.send-immediately' => pht( 'Mail is now always delivered by the daemons.'), 'auth.sessions.conduit' => $session_reason, 'auth.sessions.web' => $session_reason, 'tokenizer.ondemand' => pht( 'Phabricator now manages typeahead strategies automatically.'), 'differential.revision-custom-detail-renderer' => pht( 'Obsolete; use standard rendering events instead.'), 'differential.show-host-field' => $differential_field_reason, 'differential.show-test-plan-field' => $differential_field_reason, 'differential.field-selector' => $differential_field_reason, 'phabricator.show-beta-applications' => pht( 'This option has been renamed to `phabricator.show-prototypes` '. 'to emphasize the unfinished nature of many prototype applications. '. 'Your existing setting has been migrated.'), 'notification.user' => pht( 'The notification server no longer requires root permissions. Start '. 'the server as the user you want it to run under.'), 'notification.debug' => pht( 'Notifications no longer have a dedicated debugging mode.'), 'translation.provider' => pht( 'The translation implementation has changed and providers are no '. 'longer used or supported.'), 'config.mask' => pht( 'Use `config.hide` instead of this option.'), + 'phd.start-taskmasters' => pht( + 'Taskmasters now use an autoscaling pool. You can configure the '. + 'pool size with `phd.taskmasters`.'), ); return $ancient_config; } } diff --git a/src/applications/config/option/PhabricatorPHDConfigOptions.php b/src/applications/config/option/PhabricatorPHDConfigOptions.php index ee240d3916..3fb0afdc57 100644 --- a/src/applications/config/option/PhabricatorPHDConfigOptions.php +++ b/src/applications/config/option/PhabricatorPHDConfigOptions.php @@ -1,84 +1,84 @@ newOption('phd.pid-directory', 'string', '/var/tmp/phd/pid') ->setDescription( pht( 'Directory that phd should use to track running daemons.')), $this->newOption('phd.log-directory', 'string', '/var/tmp/phd/log') ->setDescription( pht( 'Directory that the daemons should use to store log files.')), - $this->newOption('phd.start-taskmasters', 'int', 4) - ->setSummary(pht('Number of TaskMaster daemons to start by default.')) + $this->newOption('phd.taskmasters', 'int', 4) + ->setSummary(pht('Maximum taskmaster daemon pool size.')) ->setDescription( pht( - "Number of 'TaskMaster' daemons that 'phd start' should start. ". - "You can raise this if you have a task backlog, or explicitly ". - "launch more with 'phd launch taskmaster'.")), + 'Maximum number of taskmaster daemons to run at once. Raising '. + 'this can increase the maximum throughput of the task queue. The '. + 'pool will automatically scale down when unutilized.')), $this->newOption('phd.verbose', 'bool', false) ->setBoolOptions( array( pht('Verbose mode'), pht('Normal mode'), )) ->setSummary(pht("Launch daemons in 'verbose' mode by default.")) ->setDescription( pht( "Launch daemons in 'verbose' mode by default. This creates a lot ". "of output, but can help debug issues. Daemons launched in debug ". "mode with 'phd debug' are always launched in verbose mode. See ". "also 'phd.trace'.")), $this->newOption('phd.user', 'string', null) ->setLocked(true) ->setSummary(pht('System user to run daemons as.')) ->setDescription( pht( 'Specify a system user to run the daemons as. Primarily, this '. 'user will own the working copies of any repositories that '. 'Phabricator imports or manages. This option is new and '. 'experimental.')), $this->newOption('phd.trace', 'bool', false) ->setBoolOptions( array( pht('Trace mode'), pht('Normal mode'), )) ->setSummary(pht("Launch daemons in 'trace' mode by default.")) ->setDescription( pht( "Launch daemons in 'trace' mode by default. This creates an ". "ENORMOUS amount of output, but can help debug issues. Daemons ". "launched in debug mode with 'phd debug' are always launched in ". "trace mdoe. See also 'phd.verbose'.")), $this->newOption('phd.variant-config', 'list', array()) ->setDescription( pht( 'Specify config keys that can safely vary between the web tier '. 'and the daemons. Primarily, this is a way to suppress the '. '"Daemons and Web Have Different Config" setup issue on a per '. 'config key basis.')), ); } } diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php index cbf02f161c..5d8d6df89e 100644 --- a/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php +++ b/src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php @@ -1,52 +1,56 @@ setName('restart') ->setSynopsis( pht( 'Stop, then start the standard daemon loadout.')) ->setArguments( array( array( 'name' => 'graceful', 'param' => 'seconds', 'help' => pht( 'Grace period for daemons to attempt a clean shutdown, in '. 'seconds. Defaults to __15__ seconds.'), 'default' => 15, ), array( 'name' => 'gently', 'help' => pht( 'Ignore running processes that look like daemons but do not '. 'have corresponding PID files.'), ), array( 'name' => 'force', 'help' => pht( 'Also stop running processes that look like daemons but do '. 'not have corresponding PID files.'), ), + $this->getAutoscaleReserveArgument(), )); } public function execute(PhutilArgumentParser $args) { $err = $this->executeStopCommand( array(), array( 'graceful' => $args->getArg('graceful'), 'force' => $args->getArg('force'), 'gently' => $args->getArg('gently'), )); if ($err) { return $err; } - return $this->executeStartCommand(); + return $this->executeStartCommand( + array( + 'reserve' => (float)$args->getArg('autoscale-reserve', 0.0), + )); } } diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php index 7826dde662..d489dca88b 100644 --- a/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php +++ b/src/applications/daemon/management/PhabricatorDaemonManagementStartWorkflow.php @@ -1,36 +1,40 @@ setName('start') ->setSynopsis( pht( 'Start the standard configured collection of Phabricator daemons. '. 'This is appropriate for most installs. Use **phd launch** to '. 'customize which daemons are launched.')) ->setArguments( array( array( 'name' => 'keep-leases', 'help' => pht( 'By default, **phd start** will free all task leases held by '. 'the daemons. With this flag, this step will be skipped.'), ), array( 'name' => 'force', 'help' => pht( 'Start daemons even if daemons are already running.'), ), + $this->getAutoscaleReserveArgument(), )); } public function execute(PhutilArgumentParser $args) { return $this->executeStartCommand( - $args->getArg('keep-leases'), - $args->getArg('force')); + array( + 'keep-leases' => $args->getArg('keep-leases'), + 'force' => $args->getArg('force'), + 'reserve' => (float)$args->getArg('autoscale-reserve', 0.0), + )); } } diff --git a/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php b/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php index 5e719532c1..6d523c76fd 100644 --- a/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php +++ b/src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php @@ -1,594 +1,621 @@ setAncestorClass('PhutilDaemon') ->setConcreteOnly(true) ->selectSymbolsWithoutLoading(); } protected final function getPIDDirectory() { $path = PhabricatorEnv::getEnvConfig('phd.pid-directory'); return $this->getControlDirectory($path); } protected final function getLogDirectory() { $path = PhabricatorEnv::getEnvConfig('phd.log-directory'); return $this->getControlDirectory($path); } private function getControlDirectory($path) { if (!Filesystem::pathExists($path)) { list($err) = exec_manual('mkdir -p %s', $path); if ($err) { throw new Exception( "phd requires the directory '{$path}' to exist, but it does not ". "exist and could not be created. Create this directory or update ". "'phd.pid-directory' / 'phd.log-directory' in your configuration ". "to point to an existing directory."); } } return $path; } protected final function loadRunningDaemons() { $daemons = array(); $pid_dir = $this->getPIDDirectory(); $pid_files = Filesystem::listDirectory($pid_dir); foreach ($pid_files as $pid_file) { $path = $pid_dir.'/'.$pid_file; $daemons[] = PhabricatorDaemonReference::loadReferencesFromFile($path); } return array_mergev($daemons); } protected final function loadAllRunningDaemons() { $local_daemons = $this->loadRunningDaemons(); $local_ids = array(); foreach ($local_daemons as $daemon) { $daemon_log = $daemon->getDaemonLog(); if ($daemon_log) { $local_ids[] = $daemon_log->getID(); } } $daemon_query = id(new PhabricatorDaemonLogQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE); if ($local_ids) { $daemon_query->withoutIDs($local_ids); } $remote_daemons = $daemon_query->execute(); return array_merge($local_daemons, $remote_daemons); } private function findDaemonClass($substring) { $symbols = $this->loadAvailableDaemonClasses(); $symbols = ipull($symbols, 'name'); $match = array(); foreach ($symbols as $symbol) { if (stripos($symbol, $substring) !== false) { if (strtolower($symbol) == strtolower($substring)) { $match = array($symbol); break; } else { $match[] = $symbol; } } } if (count($match) == 0) { throw new PhutilArgumentUsageException( pht( "No daemons match '%s'! Use 'phd list' for a list of available ". "daemons.", $substring)); } else if (count($match) > 1) { throw new PhutilArgumentUsageException( pht( "Specify a daemon unambiguously. Multiple daemons match '%s': %s.", $substring, implode(', ', $match))); } return head($match); } protected final function launchDaemons( array $daemons, $debug, $run_as_current_user = false) { // Convert any shorthand classnames like "taskmaster" into proper class // names. foreach ($daemons as $key => $daemon) { $class = $this->findDaemonClass($daemon['class']); $daemons[$key]['class'] = $class; } $console = PhutilConsole::getConsole(); if (!$run_as_current_user) { // Check if the script is started as the correct user $phd_user = PhabricatorEnv::getEnvConfig('phd.user'); $current_user = posix_getpwuid(posix_geteuid()); $current_user = $current_user['name']; if ($phd_user && $phd_user != $current_user) { if ($debug) { throw new PhutilArgumentUsageException(pht( 'You are trying to run a daemon as a nonstandard user, '. 'and `phd` was not able to `sudo` to the correct user. '."\n". 'Phabricator is configured to run daemons as "%s", '. 'but the current user is "%s". '."\n". 'Use `sudo` to run as a different user, pass `--as-current-user` '. 'to ignore this warning, or edit `phd.user` '. 'to change the configuration.', $phd_user, $current_user)); } else { $this->runDaemonsAsUser = $phd_user; $console->writeOut(pht('Starting daemons as %s', $phd_user)."\n"); } } } $this->printLaunchingDaemons($daemons, $debug); $flags = array(); if ($debug || PhabricatorEnv::getEnvConfig('phd.trace')) { $flags[] = '--trace'; } if ($debug || PhabricatorEnv::getEnvConfig('phd.verbose')) { $flags[] = '--verbose'; } $instance = PhabricatorEnv::getEnvConfig('cluster.instance'); if ($instance) { $flags[] = '-l'; $flags[] = $instance; } $config = array(); if (!$debug) { $config['daemonize'] = true; } if (!$debug) { $config['log'] = $this->getLogDirectory().'/daemons.log'; } $pid_dir = $this->getPIDDirectory(); // TODO: This should be a much better user experience. Filesystem::assertExists($pid_dir); Filesystem::assertIsDirectory($pid_dir); Filesystem::assertWritable($pid_dir); $config['piddir'] = $pid_dir; $config['daemons'] = $daemons; $command = csprintf('./phd-daemon %Ls', $flags); $phabricator_root = dirname(phutil_get_library_root('phabricator')); $daemon_script_dir = $phabricator_root.'/scripts/daemon/'; if ($debug) { // Don't terminate when the user sends ^C; it will be sent to the // subprocess which will terminate normally. pcntl_signal( SIGINT, array(__CLASS__, 'ignoreSignal')); echo "\n phabricator/scripts/daemon/ \$ {$command}\n\n"; $tempfile = new TempFile('daemon.config'); Filesystem::writeFile($tempfile, json_encode($config)); phutil_passthru( '(cd %s && exec %C < %s)', $daemon_script_dir, $command, $tempfile); } else { try { $this->executeDaemonLaunchCommand( $command, $daemon_script_dir, $config, $this->runDaemonsAsUser); } catch (Exception $e) { // Retry without sudo $console->writeOut(pht( "sudo command failed. Starting daemon as current user\n")); $this->executeDaemonLaunchCommand( $command, $daemon_script_dir, $config); } } } private function executeDaemonLaunchCommand( $command, $daemon_script_dir, array $config, $run_as_user = null) { $is_sudo = false; if ($run_as_user) { // If anything else besides sudo should be // supported then insert it here (runuser, su, ...) $command = csprintf( 'sudo -En -u %s -- %C', $run_as_user, $command); $is_sudo = true; } $future = new ExecFuture('exec %C', $command); // Play games to keep 'ps' looking reasonable. $future->setCWD($daemon_script_dir); $future->write(json_encode($config)); list($stdout, $stderr) = $future->resolvex(); if ($is_sudo) { // On OSX, `sudo -n` exits 0 when the user does not have permission to // switch accounts without a password. This is not consistent with // sudo on Linux, and seems buggy/broken. Check for this by string // matching the output. if (preg_match('/sudo: a password is required/', $stderr)) { throw new Exception( pht( 'sudo exited with a zero exit code, but emitted output '. 'consistent with failure under OSX.')); } } } public static function ignoreSignal($signo) { return; } public static function requireExtensions() { self::mustHaveExtension('pcntl'); self::mustHaveExtension('posix'); } private static function mustHaveExtension($ext) { if (!extension_loaded($ext)) { echo "ERROR: The PHP extension '{$ext}' is not installed. You must ". "install it to run daemons on this machine.\n"; exit(1); } $extension = new ReflectionExtension($ext); foreach ($extension->getFunctions() as $function) { $function = $function->name; if (!function_exists($function)) { echo "ERROR: The PHP function {$function}() is disabled. You must ". "enable it to run daemons on this machine.\n"; exit(1); } } } /* -( Commands )----------------------------------------------------------- */ - protected final function executeStartCommand($keep_leases, $force) { + protected final function executeStartCommand(array $options) { + PhutilTypeSpec::checkMap( + $options, + array( + 'keep-leases' => 'optional bool', + 'force' => 'optional bool', + 'reserve' => 'optional float', + )); + $console = PhutilConsole::getConsole(); - if (!$force) { + if (!idx($options, 'force')) { $running = $this->loadRunningDaemons(); // This may include daemons which were launched but which are no longer // running; check that we actually have active daemons before failing. foreach ($running as $daemon) { if ($daemon->isRunning()) { $message = pht( "phd start: Unable to start daemons because daemons are already ". "running.\n\n". "You can view running daemons with 'phd status'.\n". "You can stop running daemons with 'phd stop'.\n". "You can use 'phd restart' to stop all daemons before starting ". "new daemons.\n". "You can force daemons to start anyway with --force."); $console->writeErr("%s\n", $message); exit(1); } } } - if ($keep_leases) { + if (idx($options, 'keep-leases')) { $console->writeErr("%s\n", pht('Not touching active task queue leases.')); } else { $console->writeErr("%s\n", pht('Freeing active task leases...')); $count = $this->freeActiveLeases(); $console->writeErr( "%s\n", pht('Freed %s task lease(s).', new PhutilNumber($count))); } $daemons = array( array( 'class' => 'PhabricatorRepositoryPullLocalDaemon', ), array( 'class' => 'PhabricatorGarbageCollectorDaemon', ), array( 'class' => 'PhabricatorTriggerDaemon', ), - ); - - $taskmasters = PhabricatorEnv::getEnvConfig('phd.start-taskmasters'); - for ($ii = 0; $ii < $taskmasters; $ii++) { - $daemons[] = array( + array( 'class' => 'PhabricatorTaskmasterDaemon', - ); - } + 'autoscale' => array( + 'group' => 'task', + 'pool' => PhabricatorEnv::getEnvConfig('phd.taskmasters'), + 'reserve' => idx($options, 'reserve', 0), + ), + ), + ); $this->launchDaemons($daemons, $is_debug = false); $console->writeErr(pht('Done.')."\n"); return 0; } protected final function executeStopCommand( array $pids, array $options) { $console = PhutilConsole::getConsole(); $grace_period = idx($options, 'graceful', 15); $force = idx($options, 'force'); $gently = idx($options, 'gently'); if ($gently && $force) { throw new PhutilArgumentUsageException( pht( 'You can not specify conflicting options --gently and --force '. 'together.')); } $daemons = $this->loadRunningDaemons(); if (!$daemons) { $survivors = array(); if (!$pids && !$gently) { $survivors = $this->processRogueDaemons( $grace_period, $warn = true, $force); } if (!$survivors) { $console->writeErr(pht( 'There are no running Phabricator daemons.')."\n"); } return 0; } $running_pids = array_fuse(mpull($daemons, 'getPID')); if (!$pids) { $stop_pids = $running_pids; } else { // We were given a PID or set of PIDs to kill. $stop_pids = array(); foreach ($pids as $key => $pid) { if (!preg_match('/^\d+$/', $pid)) { $console->writeErr(pht("PID '%s' is not a valid PID.", $pid)."\n"); continue; } else if (empty($running_pids[$pid])) { $console->writeErr( pht( 'PID "%d" is not a known Phabricator daemon PID. It will not '. 'be killed.', $pid)."\n"); continue; } else { $stop_pids[$pid] = $pid; } } } if (!$stop_pids) { $console->writeErr(pht('No daemons to kill.')."\n"); return 0; } $survivors = $this->sendStopSignals($stop_pids, $grace_period); // Try to clean up PID files for daemons we killed. $remove = array(); foreach ($daemons as $daemon) { $pid = $daemon->getPID(); if (empty($stop_pids[$pid])) { // We did not try to stop this overseer. continue; } if (isset($survivors[$pid])) { // We weren't able to stop this overseer. continue; } if (!$daemon->getPIDFile()) { // We don't know where the PID file is. continue; } $remove[] = $daemon->getPIDFile(); } foreach (array_unique($remove) as $remove_file) { Filesystem::remove($remove_file); } if (!$gently) { $this->processRogueDaemons($grace_period, !$pids, $force); } return 0; } private function processRogueDaemons($grace_period, $warn, $force_stop) { $console = PhutilConsole::getConsole(); $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons(); if ($rogue_daemons) { if ($force_stop) { $rogue_pids = ipull($rogue_daemons, 'pid'); $survivors = $this->sendStopSignals($rogue_pids, $grace_period); if ($survivors) { $console->writeErr( pht( 'Unable to stop processes running without PID files. '. 'Try running this command again with sudo.')."\n"); } } else if ($warn) { $console->writeErr($this->getForceStopHint($rogue_daemons)."\n"); } } return $rogue_daemons; } private function getForceStopHint($rogue_daemons) { $debug_output = ''; foreach ($rogue_daemons as $rogue) { $debug_output .= $rogue['pid'].' '.$rogue['command']."\n"; } return pht( 'There are processes running that look like Phabricator daemons but '. 'have no corresponding PID files:'."\n\n".'%s'."\n\n". 'Stop these processes by re-running this command with the --force '. 'parameter.', $debug_output); } private function sendStopSignals($pids, $grace_period) { // If we're doing a graceful shutdown, try SIGINT first. if ($grace_period) { $pids = $this->sendSignal($pids, SIGINT, $grace_period); } // If we still have daemons, SIGTERM them. if ($pids) { $pids = $this->sendSignal($pids, SIGTERM, 15); } // If the overseer is still alive, SIGKILL it. if ($pids) { $pids = $this->sendSignal($pids, SIGKILL, 0); } return $pids; } private function sendSignal(array $pids, $signo, $wait) { $console = PhutilConsole::getConsole(); $pids = array_fuse($pids); foreach ($pids as $key => $pid) { if (!$pid) { // NOTE: We must have a PID to signal a daemon, since sending a signal // to PID 0 kills this process. unset($pids[$key]); continue; } switch ($signo) { case SIGINT: $message = pht('Interrupting process %d...', $pid); break; case SIGTERM: $message = pht('Terminating process %d...', $pid); break; case SIGKILL: $message = pht('Killing process %d...', $pid); break; } $console->writeOut("%s\n", $message); posix_kill($pid, $signo); } if ($wait) { $start = PhabricatorTime::getNow(); do { foreach ($pids as $key => $pid) { if (!PhabricatorDaemonReference::isProcessRunning($pid)) { $console->writeOut(pht('Process %d exited.', $pid)."\n"); unset($pids[$key]); } } if (empty($pids)) { break; } usleep(100000); } while (PhabricatorTime::getNow() < $start + $wait); } return $pids; } private function freeActiveLeases() { $task_table = id(new PhabricatorWorkerActiveTask()); $conn_w = $task_table->establishConnection('w'); queryfx( $conn_w, 'UPDATE %T SET leaseExpires = UNIX_TIMESTAMP() WHERE leaseExpires > UNIX_TIMESTAMP()', $task_table->getTableName()); return $conn_w->getAffectedRows(); } private function printLaunchingDaemons(array $daemons, $debug) { $console = PhutilConsole::getConsole(); if ($debug) { $console->writeOut(pht('Launching daemons (in debug mode):')); } else { $console->writeOut(pht('Launching daemons:')); } $log_dir = $this->getLogDirectory().'/daemons.log'; $console->writeOut( "\n%s\n\n", pht('(Logs will appear in "%s".)', $log_dir)); foreach ($daemons as $daemon) { $is_autoscale = isset($daemon['autoscale']['group']); if ($is_autoscale) { - $autoscale = pht('(Autoscaling)'); + $autoscale = $daemon['autoscale']; + foreach ($autoscale as $key => $value) { + $autoscale[$key] = $key.'='.$value; + } + $autoscale = implode(', ', $autoscale); + + $autoscale = pht('(Autoscaling: %s)', $autoscale); } else { $autoscale = pht('(Static)'); } $console->writeOut( " %s %s\n", $daemon['class'], $autoscale, implode(' ', idx($daemon, 'argv', array()))); } $console->writeOut("\n"); } + protected function getAutoscaleReserveArgument() { + return array( + 'name' => 'autoscale-reserve', + 'param' => 'ratio', + 'help' => pht( + 'Specify a proportion of machine memory which must be free '. + 'before autoscale pools will grow. For example, a value of 0.25 '. + 'means that pools will not grow unless the machine has at least '. + '25%% of its RAM free.'), + ); + } + } diff --git a/src/docs/user/configuration/managing_daemons.diviner b/src/docs/user/configuration/managing_daemons.diviner index 988051ecc8..e98af36a24 100644 --- a/src/docs/user/configuration/managing_daemons.diviner +++ b/src/docs/user/configuration/managing_daemons.diviner @@ -1,138 +1,139 @@ @title Managing Daemons with phd @group config Explains Phabricator daemons and the daemon control program ##phd##. = Overview = Phabricator uses daemons (background processing scripts) to handle a number of tasks: - tracking repositories, discovering new commits, and importing and parsing commits; - sending email; and - collecting garbage, like old logs and caches. Daemons are started and stopped with **phd** (the **Ph**abricator **D**aemon launcher). Daemons can be monitored via a web console. You do not need to run daemons for most parts of Phabricator to work, but some features (principally, repository tracking with Diffusion) require them and several features will benefit in performance or stability if you configure daemons. = phd = **phd** is a command-line script (located at ##phabricator/bin/phd##). To get a list of commands, run ##phd help##: phabricator/ $ ./bin/phd help NAME phd - phabricator daemon launcher ... Generally, you will use: - **phd start** to launch all daemons; - **phd restart** to restart all daemons; - **phd status** to get a list of running daemons; and - **phd stop** to stop all daemons. If you want finer-grained control, you can use: - **phd launch** to launch individual daemons; and - **phd debug** to debug problems with daemons. NOTE: When you upgrade Phabricator or change configuration, you should restart the daemons by running `phd restart`. = Daemon Console = You can view status and debugging information for daemons in the Daemon Console via the web interface. Go to ##/daemon/## in your install or click **Daemon Console** from "More Stuff". The Daemon Console shows a list of all the daemons that have ever launched, and allows you to view log information for them. If you have issues with daemons, you may be able to find error information that will help you resolve the problem in the console. NOTE: The easiest way to figure out what's wrong with a daemon is usually to use **phd debug** to launch it instead of **phd start**. This will run it without daemonizing it, so you can see output in your console. = Available Daemons = You can get a list of launchable daemons with **phd list**: - **libphutil test daemons** are not generally useful unless you are developing daemon infrastructure or debugging a daemon problem; - **PhabricatorTaskmasterDaemon** performs work from a task queue; - **PhabricatorRepositoryPullLocalDaemon** daemons track repositories, for more information see @{article:Diffusion User Guide}; and - **PhabricatorGarbageCollectorDaemon** cleans up old logs and caches. = Debugging and Tuning = In most cases, **phd start** handles launching all the daemons you need. However, you may want to use more granular daemon controls to debug daemons, launch custom daemons, or launch special daemons like the IRC bot. To debug a daemon, use `phd debug`: phabricator/bin/ $ ./phd debug You can pass arguments like this (normal arguments are passed to the daemon control mechanism, not to the daemon itself): phabricator/bin/ $ ./phd debug -- --flavor apple In debug mode, daemons do not daemonize, and they print additional debugging output to the console. This should make it easier to debug problems. You can terminate the daemon with `^C`. To launch a nonstandard daemon, use `phd launch`: phabricator/bin/ $ ./phd launch This daemon will daemonize and run normally. == General Tips == - - You can set the number of taskmasters that `phd start` starts by the config - key `phd.start-taskmasters`. If you have a task backlog, try increasing it. + - You can set the maximum number of taskmasters that will run at once + by adjusting `phd.taskmasters`. If you have a task backlog, try increasing + it. - When you `phd launch` or `phd debug` a daemon, you can type any unique substring of its name, so `phd launch pull` will work correctly. - `phd stop` and `phd restart` stop **all** of the daemons on the machine, not just those started with `phd start`. If you're writing a restart script, have it launch any custom daemons explicitly after `phd restart`. - You can write your own daemons and manage them with `phd` by extending @{class:PhabricatorDaemon}. See @{article:libphutil Libraries User Guide}. - See @{article:Diffusion User Guide} for details about tuning the repository daemon. == Multiple Machines == If you have multiple machines, you should use `phd launch` to tweak which daemons launch, and split daemons across machines like this: - `PhabricatorRepositoryPullLocalDaemon`: Run one copy on any machine. On each web frontend which is not running a normal copy, run a copy with the `--no-discovery` flag. - `PhabricatorGarbageCollectorDaemon`: Run one copy on any machine. - `PhabricatorTaskmasterDaemon`: Run as many copies as you need to keep tasks from backing up. You can run them all on one machine or split them across machines. A gratuitously wasteful install might have a dedicated daemon machine which runs `phd start` with a large pool of taskmasters set in the config, and then runs `phd launch PhabricatorRepositoryPullLocalDaemon -- --no-discovery` on each web server. This is grossly excessive in normal cases. = Next Steps = Continue by: - learning about the repository daemon with @{article:Diffusion User Guide}; or - writing your own daemons with @{article:libphutil Libraries User Guide}. diff --git a/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php index f48194bb36..40606c3d61 100644 --- a/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php +++ b/src/infrastructure/daemon/workers/PhabricatorTaskmasterDaemon.php @@ -1,69 +1,55 @@ setLimit(1) ->execute(); if ($tasks) { $this->willBeginWork(); foreach ($tasks as $task) { $id = $task->getID(); $class = $task->getTaskClass(); $this->log("Working on task {$id} ({$class})..."); $task = $task->executeTask(); $ex = $task->getExecutionException(); if ($ex) { if ($ex instanceof PhabricatorWorkerPermanentFailureException) { $this->log( pht( 'Task %s failed permanently: %s', $id, $ex->getMessage())); } else if ($ex instanceof PhabricatorWorkerYieldException) { $this->log(pht('Task %s yielded.', $id)); } else { $this->log("Task {$id} failed!"); throw new PhutilProxyException( "Error while executing task ID {$id} from queue.", $ex); } } else { $this->log("Task {$id} complete! Moved to archive."); } } $sleep = 0; } else { - // When there's no work, sleep for as many seconds as there are - // active taskmasters. - - // On average, this starts tasks added to an empty queue after one - // second. This keeps responsiveness high even on small instances - // without much work to do. - - // It also means an empty queue has an average load of one query - // per second even if there are a very large number of taskmasters - // launched. - - // The first time we sleep, we add a random offset to try to spread - // the sleep times out somewhat evenly. + // When there's no work, sleep for one second. The pool will + // autoscale down if we're continuously idle for an extended period + // of time. $this->willBeginIdle(); - $sleep = $taskmaster_count + $offset; - $offset = 0; + $sleep = 1; } $this->sleep($sleep); } while (!$this->shouldExit()); } }