diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php index 082421f..a0aaf56 100644 --- a/src/daemon/PhutilDaemon.php +++ b/src/daemon/PhutilDaemon.php @@ -1,383 +1,383 @@ shouldExit()) { * if (work_available()) { * $this->willBeginWork(); * do_work(); * $this->sleep(0); * } else { * $this->willBeginIdle(); * $this->sleep(1); * } * } * * In particular, call @{method:willBeginWork} before becoming busy, and * @{method:willBeginIdle} when no work is available. If the daemon is launched * into an autoscale pool, this will cause the pool to automatically scale up * when busy and down when idle. * * See @{class:PhutilHighIntensityIntervalDaemon} for an example of a simple * autoscaling daemon. * * Launching a daemon which does not make these callbacks into an autoscale * pool will have no effect. * * @task overseer Communicating With the Overseer * @task autoscale Autoscaling Daemon Pools */ abstract class PhutilDaemon extends Phobject { const MESSAGETYPE_STDOUT = 'stdout'; const MESSAGETYPE_HEARTBEAT = 'heartbeat'; const MESSAGETYPE_BUSY = 'busy'; const MESSAGETYPE_IDLE = 'idle'; const MESSAGETYPE_DOWN = 'down'; const MESSAGETYPE_HIBERNATE = 'hibernate'; const WORKSTATE_BUSY = 'busy'; const WORKSTATE_IDLE = 'idle'; private $argv; private $traceMode; private $traceMemory; private $verbose; private $notifyReceived; private $inGracefulShutdown; private $workState = null; private $idleSince = null; private $scaledownDuration; final public function setVerbose($verbose) { $this->verbose = $verbose; return $this; } final public function getVerbose() { return $this->verbose; } final public function setScaledownDuration($scaledown_duration) { $this->scaledownDuration = $scaledown_duration; return $this; } final public function getScaledownDuration() { return $this->scaledownDuration; } final public function __construct(array $argv) { $this->argv = $argv; $router = PhutilSignalRouter::getRouter(); $handler_key = 'daemon.term'; if (!$router->getHandler($handler_key)) { $handler = new PhutilCallbackSignalHandler( SIGTERM, __CLASS__.'::onTermSignal'); $router->installHandler($handler_key, $handler); } pcntl_signal(SIGINT, array($this, 'onGracefulSignal')); pcntl_signal(SIGUSR2, array($this, 'onNotifySignal')); // Without discard mode, this consumes unbounded amounts of memory. Keep // memory bounded. PhutilServiceProfiler::getInstance()->enableDiscardMode(); $this->beginStdoutCapture(); } final public function __destruct() { $this->endStdoutCapture(); } final public function stillWorking() { $this->emitOverseerMessage(self::MESSAGETYPE_HEARTBEAT, null); if ($this->traceMemory) { $daemon = get_class($this); fprintf( STDERR, "%s %s %s\n", '', $daemon, pht( 'Memory Usage: %s KB', new PhutilNumber(memory_get_usage() / 1024, 1))); } } final public function shouldExit() { return $this->inGracefulShutdown; } final protected function shouldHibernate($duration) { // Don't hibernate if we don't have very long to sleep. if ($duration < 5) { return false; } // Never hibernate if we're part of a pool and could scale down instead. // We only hibernate the last process to drop the pool size to zero. if ($this->getScaledownDuration()) { return false; } // Don't hibernate for too long. - $duration = max($duration, phutil_units('3 minutes in seconds')); + $duration = min($duration, phutil_units('3 minutes in seconds')); $this->emitOverseerMessage( self::MESSAGETYPE_HIBERNATE, array( 'duration' => $duration, )); $this->log( pht( 'Preparing to hibernate for %s second(s).', new PhutilNumber($duration))); return true; } final protected function sleep($duration) { $this->notifyReceived = false; $this->willSleep($duration); $this->stillWorking(); $scale_down = $this->getScaledownDuration(); $max_sleep = 60; if ($scale_down) { $max_sleep = min($max_sleep, $scale_down); } if ($scale_down) { if ($this->workState == self::WORKSTATE_IDLE) { $dur = (time() - $this->idleSince); $this->log(pht('Idle for %s seconds.', $dur)); } } while ($duration > 0 && !$this->notifyReceived && !$this->shouldExit()) { // If this is an autoscaling clone and we've been idle for too long, // we're going to scale the pool down by exiting and not restarting. The // DOWN message tells the overseer that we don't want to be restarted. if ($scale_down) { if ($this->workState == self::WORKSTATE_IDLE) { if ($this->idleSince && ($this->idleSince + $scale_down < time())) { $this->inGracefulShutdown = true; $this->emitOverseerMessage(self::MESSAGETYPE_DOWN, null); $this->log( pht( 'Daemon was idle for more than %s second(s), '. 'scaling pool down.', new PhutilNumber($scale_down))); break; } } } sleep(min($duration, $max_sleep)); $duration -= $max_sleep; $this->stillWorking(); } } protected function willSleep($duration) { return; } public static function onTermSignal($signo) { self::didCatchSignal($signo); } final protected function getArgv() { return $this->argv; } final public function execute() { $this->willRun(); $this->run(); } abstract protected function run(); final public function setTraceMemory() { $this->traceMemory = true; return $this; } final public function getTraceMemory() { return $this->traceMemory; } final public function setTraceMode() { $this->traceMode = true; PhutilServiceProfiler::installEchoListener(); PhutilConsole::getConsole()->getServer()->setEnableLog(true); $this->didSetTraceMode(); return $this; } final public function getTraceMode() { return $this->traceMode; } final public function onGracefulSignal($signo) { self::didCatchSignal($signo); $this->inGracefulShutdown = true; } final public function onNotifySignal($signo) { self::didCatchSignal($signo); $this->notifyReceived = true; $this->onNotify($signo); } protected function onNotify($signo) { // This is a hook for subclasses. } protected function willRun() { // This is a hook for subclasses. } protected function didSetTraceMode() { // This is a hook for subclasses. } final protected function log($message) { if ($this->verbose) { $daemon = get_class($this); fprintf(STDERR, "%s %s %s\n", '', $daemon, $message); } } private static function didCatchSignal($signo) { $signame = phutil_get_signal_name($signo); fprintf( STDERR, "%s Caught signal %s (%s).\n", '', $signo, $signame); } /* -( Communicating With the Overseer )------------------------------------ */ private function beginStdoutCapture() { ob_start(array($this, 'didReceiveStdout'), 2); } private function endStdoutCapture() { ob_end_flush(); } public function didReceiveStdout($data) { if (!strlen($data)) { return ''; } return $this->encodeOverseerMessage(self::MESSAGETYPE_STDOUT, $data); } private function encodeOverseerMessage($type, $data) { $structure = array($type); if ($data !== null) { $structure[] = $data; } return json_encode($structure)."\n"; } private function emitOverseerMessage($type, $data) { $this->endStdoutCapture(); echo $this->encodeOverseerMessage($type, $data); $this->beginStdoutCapture(); } public static function errorListener($event, $value, array $metadata) { // If the caller has redirected the error log to a file, PHP won't output // messages to stderr, so the overseer can't capture them. Install a // listener which just echoes errors to stderr, so the overseer is always // aware of errors. $console = PhutilConsole::getConsole(); $message = idx($metadata, 'default_message'); if ($message) { $console->writeErr("%s\n", $message); } if (idx($metadata, 'trace')) { $trace = PhutilErrorHandler::formatStacktrace($metadata['trace']); $console->writeErr("%s\n", $trace); } } /* -( Autoscaling )-------------------------------------------------------- */ /** * Prepare to become busy. This may autoscale the pool up. * * This notifies the overseer that the daemon has become busy. If daemons * that are part of an autoscale pool are continuously busy for a prolonged * period of time, the overseer may scale up the pool. * * @return this * @task autoscale */ protected function willBeginWork() { if ($this->workState != self::WORKSTATE_BUSY) { $this->workState = self::WORKSTATE_BUSY; $this->idleSince = null; $this->emitOverseerMessage(self::MESSAGETYPE_BUSY, null); } return $this; } /** * Prepare to idle. This may autoscale the pool down. * * This notifies the overseer that the daemon is no longer busy. If daemons * that are part of an autoscale pool are idle for a prolonged period of * time, they may exit to scale the pool down. * * @return this * @task autoscale */ protected function willBeginIdle() { if ($this->workState != self::WORKSTATE_IDLE) { $this->workState = self::WORKSTATE_IDLE; $this->idleSince = time(); $this->emitOverseerMessage(self::MESSAGETYPE_IDLE, null); } return $this; } } diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php index e1c1813..bf1f864 100644 --- a/src/daemon/PhutilDaemonOverseer.php +++ b/src/daemon/PhutilDaemonOverseer.php @@ -1,514 +1,520 @@ enableDiscardMode(); $args = new PhutilArgumentParser($argv); $args->setTagline(pht('daemon overseer')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'trace-memory', 'help' => pht('Enable debug memory tracing.'), ), array( 'name' => 'verbose', 'help' => pht('Enable verbose activity logging.'), ), array( 'name' => 'label', 'short' => 'l', 'param' => 'label', 'help' => pht( 'Optional process label. Makes "%s" nicer, no behavioral effects.', 'ps'), ), )); $argv = array(); if ($args->getArg('trace')) { $this->traceMode = true; $argv[] = '--trace'; } if ($args->getArg('trace-memory')) { $this->traceMode = true; $this->traceMemory = true; $argv[] = '--trace-memory'; } $verbose = $args->getArg('verbose'); if ($verbose) { $this->verbose = true; $argv[] = '--verbose'; } $label = $args->getArg('label'); if ($label) { $argv[] = '-l'; $argv[] = $label; } $this->argv = $argv; if (function_exists('posix_isatty') && posix_isatty(STDIN)) { fprintf(STDERR, pht('Reading daemon configuration from stdin...')."\n"); } $config = @file_get_contents('php://stdin'); $config = id(new PhutilJSONParser())->parse($config); $this->libraries = idx($config, 'load'); $this->log = idx($config, 'log'); $this->daemonize = idx($config, 'daemonize'); $this->piddir = idx($config, 'piddir'); $this->config = $config; if (self::$instance) { throw new Exception( pht('You may not instantiate more than one Overseer per process.')); } self::$instance = $this; $this->startEpoch = time(); // Check this before we daemonize, since if it's an issue the child will // exit immediately. if ($this->piddir) { $dir = $this->piddir; try { Filesystem::assertWritable($dir); } catch (Exception $ex) { throw new Exception( pht( "Specified daemon PID directory ('%s') does not exist or is ". "not writable by the daemon user!", $dir)); } } if (!idx($config, 'daemons')) { throw new PhutilArgumentUsageException( pht('You must specify at least one daemon to start!')); } if ($this->log) { // NOTE: Now that we're committed to daemonizing, redirect the error // log if we have a `--log` parameter. Do this at the last moment // so as many setup issues as possible are surfaced. ini_set('error_log', $this->log); } if ($this->daemonize) { // We need to get rid of these or the daemon will hang when we TERM it // waiting for something to read the buffers. TODO: Learn how unix works. fclose(STDOUT); fclose(STDERR); ob_start(); $pid = pcntl_fork(); if ($pid === -1) { throw new Exception(pht('Unable to fork!')); } else if ($pid) { exit(0); } $sid = posix_setsid(); if ($sid <= 0) { throw new Exception(pht('Failed to create new process session!')); } } $this->logMessage( 'OVER', pht( 'Started new daemon overseer (with PID "%s").', getmypid())); $this->modules = PhutilDaemonOverseerModule::getAllModules(); $this->installSignalHandlers(); } public function addLibrary($library) { $this->libraries[] = $library; return $this; } public function run() { $this->createDaemonPools(); while (true) { if ($this->shouldReloadDaemons()) { $this->didReceiveSignal(SIGHUP); } $futures = array(); + + $running_pools = false; foreach ($this->getDaemonPools() as $pool) { $pool->updatePool(); if (!$this->shouldShutdown()) { if ($pool->isHibernating()) { if ($this->shouldWakePool($pool)) { $pool->wakeFromHibernation(); } } } foreach ($pool->getFutures() as $future) { $futures[] = $future; } + + if ($pool->getDaemons()) { + $running_pools = true; + } } $this->updatePidfile(); $this->updateMemory(); $this->waitForDaemonFutures($futures); - if (!$futures) { + if (!$futures && !$running_pools) { if ($this->shouldShutdown()) { break; } } } exit($this->err); } private function waitForDaemonFutures(array $futures) { assert_instances_of($futures, 'ExecFuture'); if ($futures) { // TODO: This only wakes if any daemons actually exit. It would be a bit // cleaner to wait on any I/O with Channels. $iter = id(new FutureIterator($futures)) ->setUpdateInterval(1); foreach ($iter as $future) { break; } } else { if (!$this->shouldShutdown()) { sleep(1); } } } private function createDaemonPools() { $configs = $this->config['daemons']; $forced_options = array( 'load' => $this->libraries, 'log' => $this->log, ); foreach ($configs as $config) { $config = $forced_options + $config; $pool = PhutilDaemonPool::newFromConfig($config) ->setOverseer($this) ->setCommandLineArguments($this->argv); $this->pools[] = $pool; } } private function getDaemonPools() { return $this->pools; } /** * Identify running daemons by examining the process table. This isn't * completely reliable, but can be used as a fallback if the pid files fail * or we end up with stray daemons by other means. * * Example output (array keys are process IDs): * * array( * 12345 => array( * 'type' => 'overseer', * 'command' => 'php launch_daemon.php --daemonize ...', * 'pid' => 12345, * ), * 12346 => array( * 'type' => 'daemon', * 'command' => 'php exec_daemon.php ...', * 'pid' => 12346, * ), * ); * * @return dict Map of PIDs to process information, identifying running * daemon processes. */ public static function findRunningDaemons() { $results = array(); list($err, $processes) = exec_manual('ps -o pid,command -a -x -w -w -w'); if ($err) { return $results; } $processes = array_filter(explode("\n", trim($processes))); foreach ($processes as $process) { list($pid, $command) = preg_split('/\s+/', trim($process), 2); $pattern = '/((launch|exec)_daemon.php|phd-daemon)/'; $matches = null; if (!preg_match($pattern, $command, $matches)) { continue; } switch ($matches[1]) { case 'exec_daemon.php': $type = 'daemon'; break; case 'launch_daemon.php': case 'phd-daemon': default: $type = 'overseer'; break; } $results[(int)$pid] = array( 'type' => $type, 'command' => $command, 'pid' => (int)$pid, ); } return $results; } private function updatePidfile() { if (!$this->piddir) { return; } $pidfile = $this->toDictionary(); if ($pidfile !== $this->lastPidfile) { $this->lastPidfile = $pidfile; $pidfile_path = $this->piddir.'/daemon.'.getmypid(); Filesystem::writeFile($pidfile_path, phutil_json_encode($pidfile)); } } public function toDictionary() { $daemons = array(); foreach ($this->getDaemonPools() as $pool) { foreach ($pool->getDaemons() as $daemon) { if (!$daemon->isRunning()) { continue; } $daemons[] = $daemon->toDictionary(); } } return array( 'pid' => getmypid(), 'start' => $this->startEpoch, 'config' => $this->config, 'daemons' => $daemons, ); } private function updateMemory() { if (!$this->traceMemory) { return; } $this->logMessage( 'RAMS', pht( 'Overseer Memory Usage: %s KB', new PhutilNumber(memory_get_usage() / 1024, 1))); } public function logMessage($type, $message, $context = null) { $always_log = false; switch ($type) { case 'OVER': case 'SGNL': $always_log = true; break; } if ($always_log || $this->traceMode || $this->verbose) { error_log(date('Y-m-d g:i:s A').' ['.$type.'] '.$message); } } /* -( Signal Handling )---------------------------------------------------- */ /** * @task signals */ private function installSignalHandlers() { $signals = array( SIGUSR2, SIGHUP, SIGINT, SIGTERM, ); foreach ($signals as $signal) { pcntl_signal($signal, array($this, 'didReceiveSignal')); } } /** * @task signals */ public function didReceiveSignal($signo) { $this->logMessage( 'SGNL', pht( 'Overseer ("%d") received signal %d ("%s").', getmypid(), $signo, phutil_get_signal_name($signo))); switch ($signo) { case SIGUSR2: $signal_type = self::SIGNAL_NOTIFY; break; case SIGHUP: $signal_type = self::SIGNAL_RELOAD; break; case SIGINT: // If we receive SIGINT more than once, interpret it like SIGTERM. if ($this->inGracefulShutdown) { return $this->didReceiveSignal(SIGTERM); } $this->inGracefulShutdown = true; $signal_type = self::SIGNAL_GRACEFUL; break; case SIGTERM: // If we receive SIGTERM more than once, terminate abruptly. $this->err = 128 + $signo; if ($this->inAbruptShutdown) { exit($this->err); } $this->inAbruptShutdown = true; $signal_type = self::SIGNAL_TERMINATE; break; default: throw new Exception( pht( 'Signal handler called with unknown signal type ("%d")!', $signo)); } foreach ($this->getDaemonPools() as $pool) { $pool->didReceiveSignal($signal_type, $signo); } } /* -( Daemon Modules )----------------------------------------------------- */ private function getModules() { return $this->modules; } private function shouldReloadDaemons() { $modules = $this->getModules(); $should_reload = false; foreach ($modules as $module) { try { // NOTE: Even if one module tells us to reload, we call the method on // each module anyway to make calls a little more predictable. if ($module->shouldReloadDaemons()) { $this->logMessage( 'RELO', pht( 'Reloading daemons (triggered by overseer module "%s").', get_class($module))); $should_reload = true; } } catch (Exception $ex) { phlog($ex); } } return $should_reload; } private function shouldWakePool(PhutilDaemonPool $pool) { $modules = $this->getModules(); $should_wake = false; foreach ($modules as $module) { try { if ($module->shouldWakePool($pool)) { $this->logMessage( 'WAKE', pht( 'Waking pool "%s" (triggered by overseer module "%s").', $pool->getPoolLabel(), get_class($module))); $should_wake = true; } } catch (Exception $ex) { phlog($ex); } } return $should_wake; } private function shouldShutdown() { return $this->inGracefulShutdown || $this->inAbruptShutdown; } } diff --git a/src/daemon/PhutilDaemonPool.php b/src/daemon/PhutilDaemonPool.php index 6d7edf2..50b2228 100644 --- a/src/daemon/PhutilDaemonPool.php +++ b/src/daemon/PhutilDaemonPool.php @@ -1,346 +1,360 @@ } public static function newFromConfig(array $config) { PhutilTypeSpec::checkMap( $config, array( 'class' => 'string', 'label' => 'string', 'argv' => 'optional list', 'load' => 'optional list', 'log' => 'optional string|null', 'pool' => 'optional int', 'up' => 'optional int', 'down' => 'optional int', 'reserve' => 'optional int|float', )); $config = $config + array( 'argv' => array(), 'load' => array(), 'log' => null, 'pool' => 1, 'up' => 2, 'down' => 15, 'reserve' => 0, ); $pool = new self(); $pool->properties = $config; return $pool; } public function setOverseer(PhutilDaemonOverseer $overseer) { $this->overseer = $overseer; return $this; } public function getOverseer() { return $this->overseer; } public function setCommandLineArguments(array $arguments) { $this->commandLineArguments = $arguments; return $this; } public function getCommandLineArguments() { return $this->commandLineArguments; } private function shouldShutdown() { return $this->inShutdown; } private function newDaemon() { $config = $this->properties; if (count($this->daemons)) { $down_duration = $this->getPoolScaledownDuration(); } else { // TODO: For now, never scale pools down to 0. $down_duration = 0; } $forced_config = array( 'down' => $down_duration, ); $config = $forced_config + $config; $config = array_select_keys( $config, array( 'class', 'log', 'load', 'argv', 'down', )); $daemon = PhutilDaemonHandle::newFromConfig($config) ->setDaemonPool($this) ->setCommandLineArguments($this->getCommandLineArguments()); $daemon_id = $daemon->getDaemonID(); $this->daemons[$daemon_id] = $daemon; $daemon->didLaunch(); return $daemon; } public function getDaemons() { return $this->daemons; } public function getFutures() { $futures = array(); foreach ($this->getDaemons() as $daemon) { $future = $daemon->getFuture(); if ($future) { $futures[] = $future; } } return $futures; } public function didReceiveSignal($signal, $signo) { + switch ($signal) { + case PhutilDaemonOverseer::SIGNAL_GRACEFUL: + case PhutilDaemonOverseer::SIGNAL_TERMINATE: + $this->inShutdown = true; + break; + } + foreach ($this->getDaemons() as $daemon) { switch ($signal) { case PhutilDaemonOverseer::SIGNAL_NOTIFY: $daemon->didReceiveNotifySignal($signo); break; case PhutilDaemonOverseer::SIGNAL_RELOAD: $daemon->didReceiveReloadSignal($signo); break; case PhutilDaemonOverseer::SIGNAL_GRACEFUL: - $this->inShutdown = true; $daemon->didReceiveGracefulSignal($signo); break; case PhutilDaemonOverseer::SIGNAL_TERMINATE: - $this->inShutdown = true; $daemon->didReceiveTerminateSignal($signo); break; default: throw new Exception( pht( 'Unknown signal "%s" ("%d").', $signal, $signo)); } } } public function getPoolLabel() { return $this->getPoolProperty('label'); } public function getPoolMaximumSize() { return $this->getPoolProperty('pool'); } public function getPoolScaleupDuration() { return $this->getPoolProperty('up'); } public function getPoolScaledownDuration() { return $this->getPoolProperty('down'); } public function getPoolMemoryReserve() { return $this->getPoolProperty('reserve'); } public function getPoolDaemonClass() { return $this->getPoolProperty('class'); } private function getPoolProperty($key) { return idx($this->properties, $key); } public function updatePool() { $daemons = $this->getDaemons(); foreach ($daemons as $key => $daemon) { $daemon->update(); if ($daemon->isDone()) { $daemon->didExit(); unset($this->daemons[$key]); - $this->logMessage( - 'POOL', - pht( - 'Autoscale pool "%s" scaled down to %s daemon(s).', - $this->getPoolLabel(), - new PhutilNumber(count($this->daemons)))); + if ($this->shouldShutdown()) { + $this->logMessage( + 'DOWN', + pht( + 'Pool "%s" is exiting, with %s daemon(s) remaining.', + $this->getPoolLabel(), + new PhutilNumber(count($this->daemons)))); + } else { + $this->logMessage( + 'POOL', + pht( + 'Autoscale pool "%s" scaled down to %s daemon(s).', + $this->getPoolLabel(), + new PhutilNumber(count($this->daemons)))); + } } } $this->updateAutoscale(); } public function isHibernating() { foreach ($this->getDaemons() as $daemon) { if (!$daemon->isHibernating()) { return false; } } return true; } public function wakeFromHibernation() { if (!$this->isHibernating()) { return $this; } $this->logMessage( 'WAKE', pht( 'Autoscale pool "%s" is being awakened from hibernation.', $this->getPoolLabel())); $did_wake_daemons = false; foreach ($this->getDaemons() as $daemon) { if ($daemon->isHibernating()) { $daemon->wakeFromHibernation(); $did_wake_daemons = true; } } if (!$did_wake_daemons) { // TODO: Pools currently can't scale down to 0 daemons, but we should // scale up immediately here once they can. } $this->updatePool(); return $this; } private function updateAutoscale() { if ($this->shouldShutdown()) { return; } // Don't try to autoscale more than once per second. This mostly stops the // logs from getting flooded in verbose mode. $now = time(); if ($this->lastAutoscaleUpdate >= $now) { return; } $this->lastAutoscaleUpdate = $now; $daemons = $this->getDaemons(); // If this pool is already at the maximum size, we can't launch any new // daemons. $max_size = $this->getPoolMaximumSize(); if (count($daemons) >= $max_size) { $this->logMessage( 'POOL', pht( 'Autoscale pool "%s" already at maximum size (%s of %s).', $this->getPoolLabel(), new PhutilNumber(count($daemons)), new PhutilNumber($max_size))); return; } $scaleup_duration = $this->getPoolScaleupDuration(); foreach ($daemons as $daemon) { $busy_epoch = $daemon->getBusyEpoch(); // If any daemons haven't started work yet, don't scale the pool up. if (!$busy_epoch) { $this->logMessage( 'POOL', pht( 'Autoscale pool "%s" has an idle daemon, declining to scale.', $this->getPoolLabel())); return; } // If any daemons started work very recently, wait a little while // to scale the pool up. $busy_for = ($now - $busy_epoch); if ($busy_for < $scaleup_duration) { $this->logMessage( 'POOL', pht( 'Autoscale pool "%s" has not been busy long enough to scale up '. '(busy for %s of %s seconds).', $this->getPoolLabel(), new PhutilNumber($busy_for), new PhutilNumber($scaleup_duration))); return; } } // If we have a configured memory reserve for this pool, it tells us that // we should not scale up unless there's at least that much memory left // on the system (for example, a reserve of 0.25 means that 25% of system // memory must be free to autoscale). // Note that the first daemon is exempt: we'll always launch at least one // daemon, regardless of any memory reservation. if (count($daemons)) { $reserve = $this->getPoolMemoryReserve(); if ($reserve) { // On some systems this may be slightly more expensive than other // checks, so we only do it once we're prepared to scale up. $memory = PhutilSystem::getSystemMemoryInformation(); $free_ratio = ($memory['free'] / $memory['total']); // If we don't have enough free memory, don't scale. if ($free_ratio <= $reserve) { $this->logMessage( 'POOL', pht( 'Autoscale pool "%s" does not have enough free memory to '. 'scale up (%s free of %s reserved).', $this->getPoolLabel(), new PhutilNumber($free_ratio, 3), new PhutilNumber($reserve, 3))); return; } } } $this->logMessage( 'AUTO', pht( 'Scaling pool "%s" up to %s daemon(s).', $this->getPoolLabel(), new PhutilNumber(count($daemons) + 1))); $this->newDaemon(); } public function logMessage($type, $message, $context = null) { return $this->getOverseer()->logMessage($type, $message, $context); } }