diff --git a/scripts/daemon/exec/exec_daemon.php b/scripts/daemon/exec/exec_daemon.php --- a/scripts/daemon/exec/exec_daemon.php +++ b/scripts/daemon/exec/exec_daemon.php @@ -67,7 +67,7 @@ 'log' => 'optional string|null', 'argv' => 'optional list', 'load' => 'optional list', - 'autoscale' => 'optional wild', + 'down' => 'optional int', )); $log = idx($config, 'log'); @@ -123,9 +123,9 @@ $daemon->setVerbose(true); } -$autoscale = idx($config, 'autoscale'); -if ($autoscale) { - $daemon->setAutoscaleProperties($autoscale); +$down_duration = idx($config, 'down'); +if ($down_duration) { + $daemon->setScaledownDuration($down_duration); } $daemon->execute(); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -188,6 +188,7 @@ 'PhutilDaemonHandle' => 'daemon/PhutilDaemonHandle.php', 'PhutilDaemonOverseer' => 'daemon/PhutilDaemonOverseer.php', 'PhutilDaemonOverseerModule' => 'daemon/PhutilDaemonOverseerModule.php', + 'PhutilDaemonPool' => 'daemon/PhutilDaemonPool.php', 'PhutilDefaultSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php', 'PhutilDefaultSyntaxHighlighterEngine' => 'markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php', 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php', @@ -794,6 +795,7 @@ 'PhutilDaemonHandle' => 'Phobject', 'PhutilDaemonOverseer' => 'Phobject', 'PhutilDaemonOverseerModule' => 'Phobject', + 'PhutilDaemonPool' => 'Phobject', 'PhutilDefaultSyntaxHighlighter' => 'Phobject', 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine', 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy', diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php --- a/src/daemon/PhutilDaemon.php +++ b/src/daemon/PhutilDaemon.php @@ -59,7 +59,7 @@ private $inGracefulShutdown; private $workState = null; private $idleSince = null; - private $autoscaleProperties = array(); + private $scaledownDuration; final public function setVerbose($verbose) { $this->verbose = $verbose; @@ -70,6 +70,15 @@ return $this->verbose; } + final public function setScaledownDuration($scaledown_duration) { + $this->scaledownDuration = $scaledown_duration; + return $this; + } + + final public function getScaledownDuration() { + return $this->scaledownDuration; + } + final public function __construct(array $argv) { $this->argv = $argv; @@ -121,15 +130,14 @@ $this->willSleep($duration); $this->stillWorking(); - $is_autoscale = $this->isClonedAutoscaleDaemon(); - $scale_down = $this->getAutoscaleDownDuration(); + $scale_down = $this->getScaledownDuration(); $max_sleep = 60; - if ($is_autoscale) { + if ($scale_down) { $max_sleep = min($max_sleep, $scale_down); } - if ($is_autoscale) { + if ($scale_down) { if ($this->workState == self::WORKSTATE_IDLE) { $dur = (time() - $this->idleSince); $this->log(pht('Idle for %s seconds.', $dur)); @@ -143,7 +151,7 @@ // If this is an autoscaling clone and we've been idle for too long, // we're going to scale the pool down by exiting and not restarting. The // DOWN message tells the overseer that we don't want to be restarted. - if ($is_autoscale) { + if ($scale_down) { if ($this->workState == self::WORKSTATE_IDLE) { if ($this->idleSince && ($this->idleSince + $scale_down < time())) { $this->inGracefulShutdown = true; @@ -327,8 +335,8 @@ * Prepare to idle. This may autoscale the pool down. * * This notifies the overseer that the daemon is no longer busy. If daemons - * that are part of an autoscale pool are idle for a prolonged period of time, - * they may exit to scale the pool down. + * that are part of an autoscale pool are idle for a prolonged period of + * time, they may exit to scale the pool down. * * @return this * @task autoscale @@ -342,66 +350,4 @@ return $this; } - - - /** - * Determine if this is a clone or the original daemon. - * - * @return bool True if this is an cloned autoscaling daemon. - * @task autoscale - */ - private function isClonedAutoscaleDaemon() { - return (bool)$this->getAutoscaleProperty('clone', false); - } - - - /** - * Get the duration (in seconds) which a daemon must be continuously idle - * for before it should exit to scale the pool down. - * - * @return int Duration, in seconds. - * @task autoscale - */ - private function getAutoscaleDownDuration() { - return $this->getAutoscaleProperty('down', 15); - } - - - /** - * Configure autoscaling for this daemon. - * - * @param map Map of autoscale properties. - * @return this - * @task autoscale - */ - public function setAutoscaleProperties(array $autoscale_properties) { - PhutilTypeSpec::checkMap( - $autoscale_properties, - array( - 'group' => 'optional string', - 'up' => 'optional int', - 'down' => 'optional int', - 'pool' => 'optional int', - 'clone' => 'optional bool', - 'reserve' => 'optional int|float', - )); - - $this->autoscaleProperties = $autoscale_properties; - - return $this; - } - - - /** - * Read autoscaling configuration for this daemon. - * - * @param string Property to read. - * @param wild Default value to return if the property is not set. - * @return wild Property value, or `$default` if one is not set. - * @task autoscale - */ - private function getAutoscaleProperty($key, $default = null) { - return idx($this->autoscaleProperties, $key, $default); - } - } diff --git a/src/daemon/PhutilDaemonHandle.php b/src/daemon/PhutilDaemonHandle.php --- a/src/daemon/PhutilDaemonHandle.php +++ b/src/daemon/PhutilDaemonHandle.php @@ -8,40 +8,96 @@ const EVENT_WILL_GRACEFUL = 'daemon.willGraceful'; const EVENT_WILL_EXIT = 'daemon.willExit'; - private $overseer; - private $daemonClass; + private $pool; + private $properties; + private $future; private $argv; - private $config; + + private $restartAt; + private $busyEpoch; + private $pid; private $daemonID; private $deadline; private $heartbeat; private $stdoutBuffer; - private $restartAt; private $shouldRestart = true; private $shouldShutdown; - private $future; - private $traceMemory; - - public function __construct( - PhutilDaemonOverseer $overseer, - $daemon_class, - array $argv, - array $config) { - - $this->overseer = $overseer; - $this->daemonClass = $daemon_class; - $this->argv = $argv; - $this->config = $config; + + private function __construct() { + // + } + + public static function newFromConfig(array $config) { + PhutilTypeSpec::checkMap( + $config, + array( + 'class' => 'string', + 'argv' => 'optional list', + 'load' => 'optional list', + 'log' => 'optional string|null', + 'down' => 'optional int', + )); + + $config = $config + array( + 'argv' => array(), + 'load' => array(), + 'log' => null, + 'down' => 15, + ); + + $daemon = new self(); + $daemon->properties = $config; + $daemon->daemonID = $daemon->generateDaemonID(); + + return $daemon; + } + + public function setDaemonPool(PhutilDaemonPool $daemon_pool) { + $this->pool = $daemon_pool; + return $this; + } + + public function getDaemonPool() { + return $this->pool; + } + + public function getBusyEpoch() { + return $this->busyEpoch; + } + + public function getDaemonClass() { + return $this->getProperty('class'); + } + + private function getProperty($key) { + return idx($this->properties, $key); + } + + public function setCommandLineArguments(array $arguments) { + $this->argv = $arguments; + return $this; + } + + public function getCommandLineArguments() { + return $this->argv; + } + + public function getDaemonArguments() { + return $this->getProperty('argv'); + } + + public function didLaunch() { $this->restartAt = time(); - $this->daemonID = $this->generateDaemonID(); $this->dispatchEvent( self::EVENT_DID_LAUNCH, array( - 'argv' => $this->argv, - 'explicitArgv' => idx($this->config, 'argv'), + 'argv' => $this->getCommandLineArguments(), + 'explicitArgv' => $this->getDaemonArguments(), )); + + return $this; } public function isRunning() { @@ -56,18 +112,7 @@ return $this->future; } - public function setTraceMemory($trace_memory) { - $this->traceMemory = $trace_memory; - return $this; - } - - public function getTraceMemory() { - return $this->traceMemory; - } - public function update() { - $this->updateMemory(); - if (!$this->isRunning()) { if (!$this->shouldRestart) { return; @@ -115,6 +160,7 @@ if ($this->shouldShutdown) { $this->restartAt = null; + $this->dispatchEvent(self::EVENT_WILL_EXIT); } else { $this->scheduleRestart(); } @@ -193,8 +239,8 @@ } private function newExecFuture() { - $class = $this->daemonClass; - $argv = $this->argv; + $class = $this->getDaemonClass(); + $argv = $this->getCommandLineArguments(); $buffer_size = $this->getCaptureBufferSize(); // NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this @@ -210,11 +256,15 @@ // the shell process won't properly posix_setsid() so the pgid of the child // won't be meaningful. + $config = $this->properties; + unset($config['class']); + $config = phutil_json_encode($config); + return id(new ExecFuture('exec ./exec_daemon.php %s %Ls', $class, $argv)) ->setCWD($this->getDaemonCWD()) ->setStdoutSizeLimit($buffer_size) ->setStderrSizeLimit($buffer_size) - ->write(json_encode($this->config)); + ->write($config); } /** @@ -226,9 +276,9 @@ */ private function dispatchEvent($type, array $params = array()) { $data = array( - 'id' => $this->daemonID, - 'daemonClass' => $this->daemonClass, - 'childPID' => $this->pid, + 'id' => $this->getDaemonID(), + 'daemonClass' => $this->getDaemonClass(), + 'childPID' => $this->getPID(), ) + $params; $event = new PhutilEvent($type, $data); @@ -241,7 +291,8 @@ } private function annihilateProcessGroup() { - $pid = $this->pid; + $pid = $this->getPID(); + $pgid = posix_getpgid($pid); if ($pid && $pgid) { posix_kill(-$pgid, SIGTERM); @@ -251,16 +302,6 @@ } } - private function updateMemory() { - if ($this->traceMemory) { - $this->logMessage( - 'RAMS', - pht( - 'Overseer Memory Usage: %s KB', - new PhutilNumber(memory_get_usage() / 1024, 1))); - } - } - private function startDaemonProcess() { $this->logMessage('INIT', pht('Starting process.')); @@ -298,10 +339,12 @@ $this->deadline = time() + $this->getRequiredHeartbeatFrequency(); break; case PhutilDaemon::MESSAGETYPE_BUSY: - $this->overseer->didBeginWork($this); + if (!$this->busyEpoch) { + $this->busyEpoch = time(); + } break; case PhutilDaemon::MESSAGETYPE_IDLE: - $this->overseer->didBeginIdle($this); + $this->busyEpoch = null; break; case PhutilDaemon::MESSAGETYPE_DOWN: // The daemon is exiting because it doesn't have enough work and it @@ -319,7 +362,7 @@ } public function didReceiveNotifySignal($signo) { - $pid = $this->pid; + $pid = $this->getPID(); if ($pid) { posix_kill($pid, $signo); } @@ -349,7 +392,7 @@ // naturally be restarted after it exits, as though it had exited after an // unhandled exception. - posix_kill($this->pid, SIGINT); + posix_kill($this->getPID(), SIGINT); } public function didReceiveGracefulSignal($signo) { @@ -370,10 +413,10 @@ $this->logMessage('DONE', $sigmsg, $signo); - posix_kill($this->pid, SIGINT); + posix_kill($this->getPID(), SIGINT); } - public function didReceiveTerminalSignal($signo) { + public function didReceiveTerminateSignal($signo) { $this->shouldShutdown = true; $this->shouldRestart = false; @@ -392,7 +435,8 @@ } private function logMessage($type, $message, $context = null) { - $this->overseer->logMessage($type, $message, $context); + $this->getDaemonPool()->logMessage($type, $message, $context); + $this->dispatchEvent( self::EVENT_DID_LOG, array( @@ -402,8 +446,13 @@ )); } - public function didRemoveDaemon() { - $this->dispatchEvent(self::EVENT_WILL_EXIT); + public function toDictionary() { + return array( + 'pid' => $this->getPID(), + 'id' => $this->getDaemonID(), + 'config' => $this->properties, + ); } + } diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php --- a/src/daemon/PhutilDaemonOverseer.php +++ b/src/daemon/PhutilDaemonOverseer.php @@ -2,17 +2,16 @@ /** * Oversees a daemon and restarts it if it fails. + * + * @task signals Signal Handling */ final class PhutilDaemonOverseer extends Phobject { private $argv; - private $moreArgs; - private $inAbruptShutdown; - private $inGracefulShutdown; private static $instance; private $config; - private $daemons = array(); + private $pools = array(); private $traceMode; private $traceMemory; private $daemonize; @@ -21,12 +20,20 @@ private $libraries = array(); private $modules = array(); private $verbose; - private $err = 0; private $lastPidfile; private $startEpoch; private $autoscale = array(); private $autoscaleConfig = array(); + const SIGNAL_NOTIFY = 'signal/notify'; + const SIGNAL_RELOAD = 'signal/reload'; + const SIGNAL_GRACEFUL = 'signal/graceful'; + const SIGNAL_TERMINATE = 'signal/terminate'; + + private $err = 0; + private $inAbruptShutdown; + private $inGracefulShutdown; + public function __construct(array $argv) { PhutilServiceProfiler::getInstance()->enableDiscardMode(); @@ -149,11 +156,7 @@ $this->modules = PhutilDaemonOverseerModule::getAllModules(); - pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal')); - - pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal')); - pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal')); - pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); + $this->installSignalHandlers(); } public function addLibrary($library) { @@ -162,273 +165,80 @@ } public function run() { - $this->daemons = array(); - - foreach ($this->config['daemons'] as $config) { - $config += array( - 'argv' => array(), - 'autoscale' => array(), - ); - - $daemon = new PhutilDaemonHandle( - $this, - $config['class'], - $this->argv, - array( - 'log' => $this->log, - 'argv' => $config['argv'], - 'load' => $this->libraries, - 'autoscale' => $config['autoscale'], - )); - - $daemon->setTraceMemory($this->traceMemory); - - $this->addDaemon($daemon, $config); - - $group = idx($config['autoscale'], 'group'); - if (strlen($group)) { - if (isset($this->autoscaleConfig[$group])) { - throw new Exception( - pht( - 'Two daemons are part of the same autoscale group ("%s"). '. - 'Each daemon autoscale group must be unique.', - $group)); - } - $this->autoscaleConfig[$group] = $config; - } - } - - $should_reload = false; + $this->createDaemonPools(); while (true) { - foreach ($this->modules as $module) { - try { - if ($module->shouldReloadDaemons()) { - $this->logMessage( - 'RELO', - pht( - 'Reloading daemons (triggered by overseer module "%s").', - get_class($module))); - $should_reload = true; - } - } catch (Exception $ex) { - phlog($ex); - } - } - - if ($should_reload) { - $this->didReceiveReloadSignal(SIGHUP); - $should_reload = false; + if ($this->shouldReloadDaemons()) { + $this->didReceiveSignal(SIGHUP); } $futures = array(); - foreach ($this->getDaemonHandles() as $daemon) { - $daemon->update(); - if ($daemon->isRunning()) { - $futures[] = $daemon->getFuture(); - } + foreach ($this->getDaemonPools() as $pool) { + $pool->updatePool(); - if ($daemon->isDone()) { - $this->removeDaemon($daemon); + foreach ($pool->getFutures() as $future) { + $futures[] = $future; } } $this->updatePidfile(); - $this->updateAutoscale(); + $this->updateMemory(); - if ($futures) { - $iter = id(new FutureIterator($futures)) - ->setUpdateInterval(1); - foreach ($iter as $future) { - break; - } - } else { + $this->waitForDaemonFutures($futures); + + if (!$futures) { if ($this->inGracefulShutdown) { break; } - sleep(1); } } exit($this->err); } - private function addDaemon(PhutilDaemonHandle $daemon, array $config) { - $id = $daemon->getDaemonID(); - $this->daemons[$id] = array( - 'handle' => $daemon, - 'config' => $config, - ); - $autoscale_group = $this->getAutoscaleGroup($daemon); - if ($autoscale_group) { - $this->autoscale[$autoscale_group][$id] = true; - } - - return $this; - } - - private function removeDaemon(PhutilDaemonHandle $daemon) { - $id = $daemon->getDaemonID(); - - $autoscale_group = $this->getAutoscaleGroup($daemon); - if ($autoscale_group) { - unset($this->autoscale[$autoscale_group][$id]); - } + private function waitForDaemonFutures(array $futures) { + assert_instances_of($futures, 'ExecFuture'); - unset($this->daemons[$id]); - - $daemon->didRemoveDaemon(); - - return $this; - } - - private function getAutoscaleGroup(PhutilDaemonHandle $daemon) { - $id = $daemon->getDaemonID(); - $autoscale = $this->daemons[$id]['config']['autoscale']; - return idx($autoscale, 'group'); - } - - private function getAutoscaleProperty($group_key, $key, $default = null) { - $config = $this->autoscaleConfig[$group_key]['autoscale']; - return idx($config, $key, $default); - } - - public function didBeginWork(PhutilDaemonHandle $daemon) { - $id = $daemon->getDaemonID(); - $busy = idx($this->daemons[$daemon->getDaemonID()], 'busy'); - if (!$busy) { - $this->daemons[$id]['busy'] = time(); - } - } - - public function didBeginIdle(PhutilDaemonHandle $daemon) { - $id = $daemon->getDaemonID(); - unset($this->daemons[$id]['busy']); - } - - public function updateAutoscale() { - if ($this->inGracefulShutdown) { - return; - } - - foreach ($this->autoscale as $group => $daemons) { - $scaleup_duration = $this->getAutoscaleProperty($group, 'up', 2); - $max_pool_size = $this->getAutoscaleProperty($group, 'pool', 8); - $reserve = $this->getAutoscaleProperty($group, 'reserve', 0); - - // Don't scale a group if it is already at the maximum pool size. - if (count($daemons) >= $max_pool_size) { - continue; + if ($futures) { + // TODO: This only wakes if any daemons actually exit. It would be a bit + // cleaner to wait on any I/O with Channels. + $iter = id(new FutureIterator($futures)) + ->setUpdateInterval(1); + foreach ($iter as $future) { + break; } - - $should_scale = true; - foreach ($daemons as $daemon_id => $ignored) { - $busy = idx($this->daemons[$daemon_id], 'busy'); - if (!$busy) { - // At least one daemon in the group hasn't reported that it has - // started work. - $should_scale = false; - break; - } - - if ((time() - $busy) < $scaleup_duration) { - // At least one daemon in the group was idle recently, so we have - // not fullly - $should_scale = false; - break; - } - } - - // If we have a configured memory reserve for this pool, it tells us that - // we should not scale up unless there's at least that much memory left - // on the system (for example, a reserve of 0.25 means that 25% of system - // memory must be free to autoscale). - if ($should_scale && $reserve) { - // On some systems this may be slightly more expensive than other - // checks, so only do it once we're prepared to scale up. - $memory = PhutilSystem::getSystemMemoryInformation(); - $free_ratio = ($memory['free'] / $memory['total']); - - // If we don't have enough free memory, don't scale. - if ($free_ratio <= $reserve) { - continue; - } - } - - if ($should_scale) { - $config = $this->autoscaleConfig[$group]; - - $config['autoscale']['clone'] = true; - - $clone = new PhutilDaemonHandle( - $this, - $config['class'], - $this->argv, - array( - 'log' => $this->log, - 'argv' => $config['argv'], - 'load' => $this->libraries, - 'autoscale' => $config['autoscale'], - )); - - $this->logMessage( - 'AUTO', - pht( - 'Scaling pool "%s" up to %s daemon(s).', - $group, - new PhutilNumber(count($daemons) + 1))); - - $this->addDaemon($clone, $config); - - // Don't scale more than one pool up per iteration. Otherwise, we could - // break the memory barrier if we have a lot of pools and scale them - // all up at once. - return; + } else { + if (!$this->inGracefulShutdown) { + sleep(1); } } } - public function didReceiveNotifySignal($signo) { - foreach ($this->getDaemonHandles() as $daemon) { - $daemon->didReceiveNotifySignal($signo); - } - } + private function createDaemonPools() { + $configs = $this->config['daemons']; - public function didReceiveReloadSignal($signo) { - foreach ($this->getDaemonHandles() as $daemon) { - $daemon->didReceiveReloadSignal($signo); - } - } + $forced_options = array( + 'load' => $this->libraries, + 'log' => $this->log, + ); - public function didReceiveGracefulSignal($signo) { - // If we receive SIGINT more than once, interpret it like SIGTERM. - if ($this->inGracefulShutdown) { - return $this->didReceiveTerminalSignal($signo); - } - $this->inGracefulShutdown = true; + foreach ($configs as $config) { + $config = $forced_options + $config; - foreach ($this->getDaemonHandles() as $daemon) { - $daemon->didReceiveGracefulSignal($signo); - } - } + $pool = PhutilDaemonPool::newFromConfig($config) + ->setOverseer($this) + ->setCommandLineArguments($this->argv); - public function didReceiveTerminalSignal($signo) { - $this->err = 128 + $signo; - if ($this->inAbruptShutdown) { - exit($this->err); - } - $this->inAbruptShutdown = true; - - foreach ($this->getDaemonHandles() as $daemon) { - $daemon->didReceiveTerminalSignal($signo); + $this->pools[] = $pool; } } - private function getDaemonHandles() { - return ipull($this->daemons, 'handle'); + private function getDaemonPools() { + return $this->pools; } + /** * Identify running daemons by examining the process table. This isn't * completely reliable, but can be used as a fallback if the pid files fail @@ -496,35 +306,45 @@ return; } - $daemons = array(); + $pidfile = $this->toDictionary(); - foreach ($this->daemons as $daemon) { - $handle = $daemon['handle']; - $config = $daemon['config']; + if ($pidfile !== $this->lastPidfile) { + $this->lastPidfile = $pidfile; + $pidfile_path = $this->piddir.'/daemon.'.getmypid(); + Filesystem::writeFile($pidfile_path, phutil_json_encode($pidfile)); + } + } - if (!$handle->isRunning()) { - continue; - } + public function toDictionary() { + $daemons = array(); + foreach ($this->getDaemonPools() as $pool) { + foreach ($pool->getDaemons() as $daemon) { + if (!$daemon->isRunning()) { + continue; + } - $daemons[] = array( - 'pid' => $handle->getPID(), - 'id' => $handle->getDaemonID(), - 'config' => $config, - ); + $daemons[] = $daemon->toDictionary(); + } } - $pidfile = array( + return array( 'pid' => getmypid(), 'start' => $this->startEpoch, 'config' => $this->config, 'daemons' => $daemons, ); + } - if ($pidfile !== $this->lastPidfile) { - $this->lastPidfile = $pidfile; - $pidfile_path = $this->piddir.'/daemon.'.getmypid(); - Filesystem::writeFile($pidfile_path, json_encode($pidfile)); + private function updateMemory() { + if (!$this->traceMemory) { + return; } + + $this->logMessage( + 'RAMS', + pht( + 'Overseer Memory Usage: %s KB', + new PhutilNumber(memory_get_usage() / 1024, 1))); } public function logMessage($type, $message, $context = null) { @@ -533,4 +353,101 @@ } } + +/* -( Signal Handling )---------------------------------------------------- */ + + + /** + * @task signals + */ + private function installSignalHandlers() { + $signals = array( + SIGUSR2, + SIGHUP, + SIGINT, + SIGTERM, + ); + + foreach ($signals as $signal) { + pcntl_signal($signal, array($this, 'didReceiveSignal')); + } + } + + + /** + * @task signals + */ + public function didReceiveSignal($signo) { + switch ($signo) { + case SIGUSR2: + $signal_type = self::SIGNAL_NOTIFY; + break; + case SIGHUP: + $signal_type = self::SIGNAL_RELOAD; + break; + case SIGINT: + // If we receive SIGINT more than once, interpret it like SIGTERM. + if ($this->inGracefulShutdown) { + return $this->didReceiveSignal(SIGTERM); + } + + $this->inGracefulShutdown = true; + $signal_type = self::SIGNAL_GRACEFUL; + break; + case SIGTERM: + // If we receive SIGTERM more than once, terminate abruptly. + $this->err = 128 + $signo; + if ($this->inAbruptShutdown) { + exit($this->err); + } + + $this->inAbruptShutdown = true; + $signal_type = self::SIGNAL_TERMINATE; + break; + default: + throw new Exception( + pht( + 'Signal handler called with unknown signal type ("%d")!', + $signo)); + } + + foreach ($this->getDaemonPools() as $pool) { + $pool->didReceiveSignal($signal_type, $signo); + } + } + + +/* -( Daemon Modules )----------------------------------------------------- */ + + + private function getModules() { + return $this->modules; + } + + private function shouldReloadDaemons() { + $modules = $this->getModules(); + + $should_reload = false; + foreach ($modules as $module) { + try { + // NOTE: Even if one module tells us to reload, we call the method on + // each module anyway to make calls a little more predictable. + + if ($module->shouldReloadDaemons()) { + $this->logMessage( + 'RELO', + pht( + 'Reloading daemons (triggered by overseer module "%s").', + get_class($module))); + $should_reload = true; + } + } catch (Exception $ex) { + phlog($ex); + } + } + + return $should_reload; + } + + } diff --git a/src/daemon/PhutilDaemonPool.php b/src/daemon/PhutilDaemonPool.php new file mode 100644 --- /dev/null +++ b/src/daemon/PhutilDaemonPool.php @@ -0,0 +1,245 @@ + + } + + public static function newFromConfig(array $config) { + PhutilTypeSpec::checkMap( + $config, + array( + 'class' => 'string', + 'label' => 'string', + 'argv' => 'optional list', + 'load' => 'optional list', + 'log' => 'optional string|null', + 'pool' => 'optional int', + 'up' => 'optional int', + 'down' => 'optional int', + 'reserve' => 'optional int|float', + )); + + $config = $config + array( + 'argv' => array(), + 'load' => array(), + 'log' => null, + 'pool' => 1, + 'up' => 2, + 'down' => 15, + 'reserve' => 0, + ); + + $pool = new self(); + $pool->properties = $config; + + return $pool; + } + + public function setOverseer(PhutilDaemonOverseer $overseer) { + $this->overseer = $overseer; + return $this; + } + + public function getOverseer() { + return $this->overseer; + } + + public function setCommandLineArguments(array $arguments) { + $this->commandLineArguments = $arguments; + return $this; + } + + public function getCommandLineArguments() { + return $this->commandLineArguments; + } + + private function newDaemon() { + $config = $this->properties; + + if (count($this->daemons)) { + $down_duration = $this->getPoolScaledownDuration(); + } else { + // TODO: For now, never scale pools down to 0. + $down_duration = 0; + } + + $forced_config = array( + 'down' => $down_duration, + ); + + $config = $forced_config + $config; + + $config = array_select_keys( + $config, + array( + 'class', + 'log', + 'load', + 'argv', + 'down', + )); + + $daemon = PhutilDaemonHandle::newFromConfig($config) + ->setDaemonPool($this) + ->setCommandLineArguments($this->getCommandLineArguments()); + + $daemon_id = $daemon->getDaemonID(); + $this->daemons[$daemon_id] = $daemon; + + $daemon->didLaunch(); + + return $daemon; + } + + public function getDaemons() { + return $this->daemons; + } + + public function getFutures() { + $futures = array(); + foreach ($this->getDaemons() as $daemon) { + $future = $daemon->getFuture(); + if ($future) { + $futures[] = $future; + } + } + + return $futures; + } + + public function didReceiveSignal($signal, $signo) { + foreach ($this->getDaemons() as $daemon) { + switch ($signal) { + case PhutilDaemonOverseer::SIGNAL_NOTIFY: + $daemon->didReceiveNotifySignal($signo); + break; + case PhutilDaemonOverseer::SIGNAL_RELOAD: + $daemon->didReceiveReloadSignal($signo); + break; + case PhutilDaemonOverseer::SIGNAL_GRACEFUL: + $daemon->didReceiveGracefulSignal($signo); + break; + case PhutilDaemonOverseer::SIGNAL_TERMINATE: + $daemon->didReceiveTerminateSignal($signo); + break; + default: + throw new Exception( + pht( + 'Unknown signal "%s" ("%d").', + $signal, + $signo)); + } + } + } + + public function getPoolLabel() { + return $this->getPoolProperty('label'); + } + + public function getPoolMaximumSize() { + return $this->getPoolProperty('pool'); + } + + public function getPoolScaleupDuration() { + return $this->getPoolProperty('up'); + } + + public function getPoolScaledownDuration() { + return $this->getPoolProperty('down'); + } + + public function getPoolMemoryReserve() { + return $this->getPoolProperty('reserve'); + } + + public function getPoolDaemonClass() { + return $this->getPoolProperty('class'); + } + + private function getPoolProperty($key) { + return idx($this->properties, $key); + } + + public function updatePool() { + $daemons = $this->getDaemons(); + + foreach ($daemons as $key => $daemon) { + $daemon->update(); + + if ($daemon->isDone()) { + unset($daemons[$key]); + } + } + + $this->updateAutoscale(); + } + + private function updateAutoscale() { + $daemons = $this->getDaemons(); + + // If this pool is already at the maximum size, we can't launch any new + // daemons. + $max_size = $this->getPoolMaximumSize(); + if (count($daemons) >= $max_size) { + return; + } + + $now = time(); + $scaleup_duration = $this->getPoolScaleupDuration(); + + foreach ($daemons as $daemon) { + $busy_epoch = $daemon->getBusyEpoch(); + // If any daemons haven't started work yet, don't scale the pool up. + if (!$busy_epoch) { + return; + } + + // If any daemons started work very recently, wait a little while + // to scale the pool up. + $busy_for = ($now - $busy_epoch); + if ($busy_for < $scaleup_duration) { + return; + } + } + + // If we have a configured memory reserve for this pool, it tells us that + // we should not scale up unless there's at least that much memory left + // on the system (for example, a reserve of 0.25 means that 25% of system + // memory must be free to autoscale). + $reserve = $this->getPoolMemoryReserve(); + if ($reserve) { + // On some systems this may be slightly more expensive than other checks, + // so we only do it once we're prepared to scale up. + $memory = PhutilSystem::getSystemMemoryInformation(); + $free_ratio = ($memory['free'] / $memory['total']); + + // If we don't have enough free memory, don't scale. + if ($free_ratio <= $reserve) { + return; + } + } + + $this->logMessage( + 'AUTO', + pht( + 'Scaling pool "%s" up to %s daemon(s).', + $this->getPoolLabel(), + new PhutilNumber(count($daemons) + 1))); + + $this->newDaemon(); + } + + public function logMessage($type, $message, $context = null) { + return $this->getOverseer()->logMessage($type, $message, $context); + } + +}