Page MenuHomePhabricator
Files F15335341

D17389.id41813.diff
No OneTemporary
Actions

D17389.id41813.diff
View Options

diff --git a/scripts/daemon/exec/exec_daemon.php b/scripts/daemon/exec/exec_daemon.php
--- a/scripts/daemon/exec/exec_daemon.php
+++ b/scripts/daemon/exec/exec_daemon.php
@@ -67,7 +67,7 @@
'log' => 'optional string|null',
'argv' => 'optional list<wild>',
'load' => 'optional list<string>',
- 'autoscale' => 'optional wild',
+ 'down' => 'optional int',
));
$log = idx($config, 'log');
@@ -123,9 +123,9 @@
$daemon->setVerbose(true);
}
-$autoscale = idx($config, 'autoscale');
-if ($autoscale) {
- $daemon->setAutoscaleProperties($autoscale);
+$down_duration = idx($config, 'down');
+if ($down_duration) {
+ $daemon->setScaledownDuration($down_duration);
}
$daemon->execute();
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -188,6 +188,7 @@
'PhutilDaemonHandle' => 'daemon/PhutilDaemonHandle.php',
'PhutilDaemonOverseer' => 'daemon/PhutilDaemonOverseer.php',
'PhutilDaemonOverseerModule' => 'daemon/PhutilDaemonOverseerModule.php',
+ 'PhutilDaemonPool' => 'daemon/PhutilDaemonPool.php',
'PhutilDefaultSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php',
'PhutilDefaultSyntaxHighlighterEngine' => 'markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php',
'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php',
@@ -794,6 +795,7 @@
'PhutilDaemonHandle' => 'Phobject',
'PhutilDaemonOverseer' => 'Phobject',
'PhutilDaemonOverseerModule' => 'Phobject',
+ 'PhutilDaemonPool' => 'Phobject',
'PhutilDefaultSyntaxHighlighter' => 'Phobject',
'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine',
'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy',
diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php
--- a/src/daemon/PhutilDaemon.php
+++ b/src/daemon/PhutilDaemon.php
@@ -59,7 +59,7 @@
private $inGracefulShutdown;
private $workState = null;
private $idleSince = null;
- private $autoscaleProperties = array();
+ private $scaledownDuration;
final public function setVerbose($verbose) {
$this->verbose = $verbose;
@@ -70,6 +70,15 @@
return $this->verbose;
}
+ final public function setScaledownDuration($scaledown_duration) {
+ $this->scaledownDuration = $scaledown_duration;
+ return $this;
+ }
+
+ final public function getScaledownDuration() {
+ return $this->scaledownDuration;
+ }
+
final public function __construct(array $argv) {
$this->argv = $argv;
@@ -121,15 +130,14 @@
$this->willSleep($duration);
$this->stillWorking();
- $is_autoscale = $this->isClonedAutoscaleDaemon();
- $scale_down = $this->getAutoscaleDownDuration();
+ $scale_down = $this->getScaledownDuration();
$max_sleep = 60;
- if ($is_autoscale) {
+ if ($scale_down) {
$max_sleep = min($max_sleep, $scale_down);
}
- if ($is_autoscale) {
+ if ($scale_down) {
if ($this->workState == self::WORKSTATE_IDLE) {
$dur = (time() - $this->idleSince);
$this->log(pht('Idle for %s seconds.', $dur));
@@ -143,7 +151,7 @@
// If this is an autoscaling clone and we've been idle for too long,
// we're going to scale the pool down by exiting and not restarting. The
// DOWN message tells the overseer that we don't want to be restarted.
- if ($is_autoscale) {
+ if ($scale_down) {
if ($this->workState == self::WORKSTATE_IDLE) {
if ($this->idleSince && ($this->idleSince + $scale_down < time())) {
$this->inGracefulShutdown = true;
@@ -327,8 +335,8 @@
* Prepare to idle. This may autoscale the pool down.
*
* This notifies the overseer that the daemon is no longer busy. If daemons
- * that are part of an autoscale pool are idle for a prolonged period of time,
- * they may exit to scale the pool down.
+ * that are part of an autoscale pool are idle for a prolonged period of
+ * time, they may exit to scale the pool down.
*
* @return this
* @task autoscale
@@ -342,66 +350,4 @@
return $this;
}
-
-
- /**
- * Determine if this is a clone or the original daemon.
- *
- * @return bool True if this is an cloned autoscaling daemon.
- * @task autoscale
- */
- private function isClonedAutoscaleDaemon() {
- return (bool)$this->getAutoscaleProperty('clone', false);
- }
-
-
- /**
- * Get the duration (in seconds) which a daemon must be continuously idle
- * for before it should exit to scale the pool down.
- *
- * @return int Duration, in seconds.
- * @task autoscale
- */
- private function getAutoscaleDownDuration() {
- return $this->getAutoscaleProperty('down', 15);
- }
-
-
- /**
- * Configure autoscaling for this daemon.
- *
- * @param map<string, wild> Map of autoscale properties.
- * @return this
- * @task autoscale
- */
- public function setAutoscaleProperties(array $autoscale_properties) {
- PhutilTypeSpec::checkMap(
- $autoscale_properties,
- array(
- 'group' => 'optional string',
- 'up' => 'optional int',
- 'down' => 'optional int',
- 'pool' => 'optional int',
- 'clone' => 'optional bool',
- 'reserve' => 'optional int|float',
- ));
-
- $this->autoscaleProperties = $autoscale_properties;
-
- return $this;
- }
-
-
- /**
- * Read autoscaling configuration for this daemon.
- *
- * @param string Property to read.
- * @param wild Default value to return if the property is not set.
- * @return wild Property value, or `$default` if one is not set.
- * @task autoscale
- */
- private function getAutoscaleProperty($key, $default = null) {
- return idx($this->autoscaleProperties, $key, $default);
- }
-
}
diff --git a/src/daemon/PhutilDaemonHandle.php b/src/daemon/PhutilDaemonHandle.php
--- a/src/daemon/PhutilDaemonHandle.php
+++ b/src/daemon/PhutilDaemonHandle.php
@@ -8,40 +8,96 @@
const EVENT_WILL_GRACEFUL = 'daemon.willGraceful';
const EVENT_WILL_EXIT = 'daemon.willExit';
- private $overseer;
- private $daemonClass;
+ private $pool;
+ private $properties;
+ private $future;
private $argv;
- private $config;
+
+ private $restartAt;
+ private $busyEpoch;
+
private $pid;
private $daemonID;
private $deadline;
private $heartbeat;
private $stdoutBuffer;
- private $restartAt;
private $shouldRestart = true;
private $shouldShutdown;
- private $future;
- private $traceMemory;
-
- public function __construct(
- PhutilDaemonOverseer $overseer,
- $daemon_class,
- array $argv,
- array $config) {
-
- $this->overseer = $overseer;
- $this->daemonClass = $daemon_class;
- $this->argv = $argv;
- $this->config = $config;
+
+ private function __construct() {
+ // <empty>
+ }
+
+ public static function newFromConfig(array $config) {
+ PhutilTypeSpec::checkMap(
+ $config,
+ array(
+ 'class' => 'string',
+ 'argv' => 'optional list<string>',
+ 'load' => 'optional list<string>',
+ 'log' => 'optional string|null',
+ 'down' => 'optional int',
+ ));
+
+ $config = $config + array(
+ 'argv' => array(),
+ 'load' => array(),
+ 'log' => null,
+ 'down' => 15,
+ );
+
+ $daemon = new self();
+ $daemon->properties = $config;
+ $daemon->daemonID = $daemon->generateDaemonID();
+
+ return $daemon;
+ }
+
+ public function setDaemonPool(PhutilDaemonPool $daemon_pool) {
+ $this->pool = $daemon_pool;
+ return $this;
+ }
+
+ public function getDaemonPool() {
+ return $this->pool;
+ }
+
+ public function getBusyEpoch() {
+ return $this->busyEpoch;
+ }
+
+ public function getDaemonClass() {
+ return $this->getProperty('class');
+ }
+
+ private function getProperty($key) {
+ return idx($this->properties, $key);
+ }
+
+ public function setCommandLineArguments(array $arguments) {
+ $this->argv = $arguments;
+ return $this;
+ }
+
+ public function getCommandLineArguments() {
+ return $this->argv;
+ }
+
+ public function getDaemonArguments() {
+ return $this->getProperty('argv');
+ }
+
+ public function didLaunch() {
$this->restartAt = time();
- $this->daemonID = $this->generateDaemonID();
$this->dispatchEvent(
self::EVENT_DID_LAUNCH,
array(
- 'argv' => $this->argv,
- 'explicitArgv' => idx($this->config, 'argv'),
+ 'argv' => $this->getCommandLineArguments(),
+ 'explicitArgv' => $this->getDaemonArguments(),
));
+
+ return $this;
}
public function isRunning() {
@@ -56,18 +112,7 @@
return $this->future;
}
- public function setTraceMemory($trace_memory) {
- $this->traceMemory = $trace_memory;
- return $this;
- }
-
- public function getTraceMemory() {
- return $this->traceMemory;
- }
-
public function update() {
- $this->updateMemory();
-
if (!$this->isRunning()) {
if (!$this->shouldRestart) {
return;
@@ -115,6 +160,7 @@
if ($this->shouldShutdown) {
$this->restartAt = null;
+ $this->dispatchEvent(self::EVENT_WILL_EXIT);
} else {
$this->scheduleRestart();
}
@@ -193,8 +239,8 @@
}
private function newExecFuture() {
- $class = $this->daemonClass;
- $argv = $this->argv;
+ $class = $this->getDaemonClass();
+ $argv = $this->getCommandLineArguments();
$buffer_size = $this->getCaptureBufferSize();
// NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this
@@ -210,11 +256,15 @@
// the shell process won't properly posix_setsid() so the pgid of the child
// won't be meaningful.
+ $config = $this->properties;
+ unset($config['class']);
+ $config = phutil_json_encode($config);
+
return id(new ExecFuture('exec ./exec_daemon.php %s %Ls', $class, $argv))
->setCWD($this->getDaemonCWD())
->setStdoutSizeLimit($buffer_size)
->setStderrSizeLimit($buffer_size)
- ->write(json_encode($this->config));
+ ->write($config);
}
/**
@@ -226,9 +276,9 @@
*/
private function dispatchEvent($type, array $params = array()) {
$data = array(
- 'id' => $this->daemonID,
- 'daemonClass' => $this->daemonClass,
- 'childPID' => $this->pid,
+ 'id' => $this->getDaemonID(),
+ 'daemonClass' => $this->getDaemonClass(),
+ 'childPID' => $this->getPID(),
) + $params;
$event = new PhutilEvent($type, $data);
@@ -241,7 +291,8 @@
}
private function annihilateProcessGroup() {
- $pid = $this->pid;
+ $pid = $this->getPID();
+
$pgid = posix_getpgid($pid);
if ($pid && $pgid) {
posix_kill(-$pgid, SIGTERM);
@@ -251,16 +302,6 @@
}
}
- private function updateMemory() {
- if ($this->traceMemory) {
- $this->logMessage(
- 'RAMS',
- pht(
- 'Overseer Memory Usage: %s KB',
- new PhutilNumber(memory_get_usage() / 1024, 1)));
- }
- }
-
private function startDaemonProcess() {
$this->logMessage('INIT', pht('Starting process.'));
@@ -298,10 +339,12 @@
$this->deadline = time() + $this->getRequiredHeartbeatFrequency();
break;
case PhutilDaemon::MESSAGETYPE_BUSY:
- $this->overseer->didBeginWork($this);
+ if (!$this->busyEpoch) {
+ $this->busyEpoch = time();
+ }
break;
case PhutilDaemon::MESSAGETYPE_IDLE:
- $this->overseer->didBeginIdle($this);
+ $this->busyEpoch = null;
break;
case PhutilDaemon::MESSAGETYPE_DOWN:
// The daemon is exiting because it doesn't have enough work and it
@@ -319,7 +362,7 @@
}
public function didReceiveNotifySignal($signo) {
- $pid = $this->pid;
+ $pid = $this->getPID();
if ($pid) {
posix_kill($pid, $signo);
}
@@ -349,7 +392,7 @@
// naturally be restarted after it exits, as though it had exited after an
// unhandled exception.
- posix_kill($this->pid, SIGINT);
+ posix_kill($this->getPID(), SIGINT);
}
public function didReceiveGracefulSignal($signo) {
@@ -370,10 +413,10 @@
$this->logMessage('DONE', $sigmsg, $signo);
- posix_kill($this->pid, SIGINT);
+ posix_kill($this->getPID(), SIGINT);
}
- public function didReceiveTerminalSignal($signo) {
+ public function didReceiveTerminateSignal($signo) {
$this->shouldShutdown = true;
$this->shouldRestart = false;
@@ -392,7 +435,8 @@
}
private function logMessage($type, $message, $context = null) {
- $this->overseer->logMessage($type, $message, $context);
+ $this->getDaemonPool()->logMessage($type, $message, $context);
+
$this->dispatchEvent(
self::EVENT_DID_LOG,
array(
@@ -402,8 +446,13 @@
));
}
- public function didRemoveDaemon() {
- $this->dispatchEvent(self::EVENT_WILL_EXIT);
+ public function toDictionary() {
+ return array(
+ 'pid' => $this->getPID(),
+ 'id' => $this->getDaemonID(),
+ 'config' => $this->properties,
+ );
}
+
}
diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php
--- a/src/daemon/PhutilDaemonOverseer.php
+++ b/src/daemon/PhutilDaemonOverseer.php
@@ -2,17 +2,16 @@
/**
* Oversees a daemon and restarts it if it fails.
+ *
+ * @task signals Signal Handling
*/
final class PhutilDaemonOverseer extends Phobject {
private $argv;
- private $moreArgs;
- private $inAbruptShutdown;
- private $inGracefulShutdown;
private static $instance;
private $config;
- private $daemons = array();
+ private $pools = array();
private $traceMode;
private $traceMemory;
private $daemonize;
@@ -21,12 +20,20 @@
private $libraries = array();
private $modules = array();
private $verbose;
- private $err = 0;
private $lastPidfile;
private $startEpoch;
private $autoscale = array();
private $autoscaleConfig = array();
+ const SIGNAL_NOTIFY = 'signal/notify';
+ const SIGNAL_RELOAD = 'signal/reload';
+ const SIGNAL_GRACEFUL = 'signal/graceful';
+ const SIGNAL_TERMINATE = 'signal/terminate';
+
+ private $err = 0;
+ private $inAbruptShutdown;
+ private $inGracefulShutdown;
+
public function __construct(array $argv) {
PhutilServiceProfiler::getInstance()->enableDiscardMode();
@@ -149,11 +156,7 @@
$this->modules = PhutilDaemonOverseerModule::getAllModules();
- pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal'));
-
- pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal'));
- pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal'));
- pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal'));
+ $this->installSignalHandlers();
}
public function addLibrary($library) {
@@ -162,273 +165,80 @@
}
public function run() {
- $this->daemons = array();
-
- foreach ($this->config['daemons'] as $config) {
- $config += array(
- 'argv' => array(),
- 'autoscale' => array(),
- );
-
- $daemon = new PhutilDaemonHandle(
- $this,
- $config['class'],
- $this->argv,
- array(
- 'log' => $this->log,
- 'argv' => $config['argv'],
- 'load' => $this->libraries,
- 'autoscale' => $config['autoscale'],
- ));
-
- $daemon->setTraceMemory($this->traceMemory);
-
- $this->addDaemon($daemon, $config);
-
- $group = idx($config['autoscale'], 'group');
- if (strlen($group)) {
- if (isset($this->autoscaleConfig[$group])) {
- throw new Exception(
- pht(
- 'Two daemons are part of the same autoscale group ("%s"). '.
- 'Each daemon autoscale group must be unique.',
- $group));
- }
- $this->autoscaleConfig[$group] = $config;
- }
- }
-
- $should_reload = false;
+ $this->createDaemonPools();
while (true) {
- foreach ($this->modules as $module) {
- try {
- if ($module->shouldReloadDaemons()) {
- $this->logMessage(
- 'RELO',
- pht(
- 'Reloading daemons (triggered by overseer module "%s").',
- get_class($module)));
- $should_reload = true;
- }
- } catch (Exception $ex) {
- phlog($ex);
- }
- }
-
- if ($should_reload) {
- $this->didReceiveReloadSignal(SIGHUP);
- $should_reload = false;
+ if ($this->shouldReloadDaemons()) {
+ $this->didReceiveSignal(SIGHUP);
}
$futures = array();
- foreach ($this->getDaemonHandles() as $daemon) {
- $daemon->update();
- if ($daemon->isRunning()) {
- $futures[] = $daemon->getFuture();
- }
+ foreach ($this->getDaemonPools() as $pool) {
+ $pool->updatePool();
- if ($daemon->isDone()) {
- $this->removeDaemon($daemon);
+ foreach ($pool->getFutures() as $future) {
+ $futures[] = $future;
}
}
$this->updatePidfile();
- $this->updateAutoscale();
+ $this->updateMemory();
- if ($futures) {
- $iter = id(new FutureIterator($futures))
- ->setUpdateInterval(1);
- foreach ($iter as $future) {
- break;
- }
- } else {
+ $this->waitForDaemonFutures($futures);
+
+ if (!$futures) {
if ($this->inGracefulShutdown) {
break;
}
- sleep(1);
}
}
exit($this->err);
}
- private function addDaemon(PhutilDaemonHandle $daemon, array $config) {
- $id = $daemon->getDaemonID();
- $this->daemons[$id] = array(
- 'handle' => $daemon,
- 'config' => $config,
- );
- $autoscale_group = $this->getAutoscaleGroup($daemon);
- if ($autoscale_group) {
- $this->autoscale[$autoscale_group][$id] = true;
- }
-
- return $this;
- }
-
- private function removeDaemon(PhutilDaemonHandle $daemon) {
- $id = $daemon->getDaemonID();
-
- $autoscale_group = $this->getAutoscaleGroup($daemon);
- if ($autoscale_group) {
- unset($this->autoscale[$autoscale_group][$id]);
- }
+ private function waitForDaemonFutures(array $futures) {
+ assert_instances_of($futures, 'ExecFuture');
- unset($this->daemons[$id]);
-
- $daemon->didRemoveDaemon();
-
- return $this;
- }
-
- private function getAutoscaleGroup(PhutilDaemonHandle $daemon) {
- $id = $daemon->getDaemonID();
- $autoscale = $this->daemons[$id]['config']['autoscale'];
- return idx($autoscale, 'group');
- }
-
- private function getAutoscaleProperty($group_key, $key, $default = null) {
- $config = $this->autoscaleConfig[$group_key]['autoscale'];
- return idx($config, $key, $default);
- }
-
- public function didBeginWork(PhutilDaemonHandle $daemon) {
- $id = $daemon->getDaemonID();
- $busy = idx($this->daemons[$daemon->getDaemonID()], 'busy');
- if (!$busy) {
- $this->daemons[$id]['busy'] = time();
- }
- }
-
- public function didBeginIdle(PhutilDaemonHandle $daemon) {
- $id = $daemon->getDaemonID();
- unset($this->daemons[$id]['busy']);
- }
-
- public function updateAutoscale() {
- if ($this->inGracefulShutdown) {
- return;
- }
-
- foreach ($this->autoscale as $group => $daemons) {
- $scaleup_duration = $this->getAutoscaleProperty($group, 'up', 2);
- $max_pool_size = $this->getAutoscaleProperty($group, 'pool', 8);
- $reserve = $this->getAutoscaleProperty($group, 'reserve', 0);
-
- // Don't scale a group if it is already at the maximum pool size.
- if (count($daemons) >= $max_pool_size) {
- continue;
+ if ($futures) {
+ // TODO: This only wakes if any daemons actually exit. It would be a bit
+ // cleaner to wait on any I/O with Channels.
+ $iter = id(new FutureIterator($futures))
+ ->setUpdateInterval(1);
+ foreach ($iter as $future) {
+ break;
}
-
- $should_scale = true;
- foreach ($daemons as $daemon_id => $ignored) {
- $busy = idx($this->daemons[$daemon_id], 'busy');
- if (!$busy) {
- // At least one daemon in the group hasn't reported that it has
- // started work.
- $should_scale = false;
- break;
- }
-
- if ((time() - $busy) < $scaleup_duration) {
- // At least one daemon in the group was idle recently, so we have
- // not fullly
- $should_scale = false;
- break;
- }
- }
-
- // If we have a configured memory reserve for this pool, it tells us that
- // we should not scale up unless there's at least that much memory left
- // on the system (for example, a reserve of 0.25 means that 25% of system
- // memory must be free to autoscale).
- if ($should_scale && $reserve) {
- // On some systems this may be slightly more expensive than other
- // checks, so only do it once we're prepared to scale up.
- $memory = PhutilSystem::getSystemMemoryInformation();
- $free_ratio = ($memory['free'] / $memory['total']);
-
- // If we don't have enough free memory, don't scale.
- if ($free_ratio <= $reserve) {
- continue;
- }
- }
-
- if ($should_scale) {
- $config = $this->autoscaleConfig[$group];
-
- $config['autoscale']['clone'] = true;
-
- $clone = new PhutilDaemonHandle(
- $this,
- $config['class'],
- $this->argv,
- array(
- 'log' => $this->log,
- 'argv' => $config['argv'],
- 'load' => $this->libraries,
- 'autoscale' => $config['autoscale'],
- ));
-
- $this->logMessage(
- 'AUTO',
- pht(
- 'Scaling pool "%s" up to %s daemon(s).',
- $group,
- new PhutilNumber(count($daemons) + 1)));
-
- $this->addDaemon($clone, $config);
-
- // Don't scale more than one pool up per iteration. Otherwise, we could
- // break the memory barrier if we have a lot of pools and scale them
- // all up at once.
- return;
+ } else {
+ if (!$this->inGracefulShutdown) {
+ sleep(1);
}
}
}
- public function didReceiveNotifySignal($signo) {
- foreach ($this->getDaemonHandles() as $daemon) {
- $daemon->didReceiveNotifySignal($signo);
- }
- }
+ private function createDaemonPools() {
+ $configs = $this->config['daemons'];
- public function didReceiveReloadSignal($signo) {
- foreach ($this->getDaemonHandles() as $daemon) {
- $daemon->didReceiveReloadSignal($signo);
- }
- }
+ $forced_options = array(
+ 'load' => $this->libraries,
+ 'log' => $this->log,
+ );
- public function didReceiveGracefulSignal($signo) {
- // If we receive SIGINT more than once, interpret it like SIGTERM.
- if ($this->inGracefulShutdown) {
- return $this->didReceiveTerminalSignal($signo);
- }
- $this->inGracefulShutdown = true;
+ foreach ($configs as $config) {
+ $config = $forced_options + $config;
- foreach ($this->getDaemonHandles() as $daemon) {
- $daemon->didReceiveGracefulSignal($signo);
- }
- }
+ $pool = PhutilDaemonPool::newFromConfig($config)
+ ->setOverseer($this)
+ ->setCommandLineArguments($this->argv);
- public function didReceiveTerminalSignal($signo) {
- $this->err = 128 + $signo;
- if ($this->inAbruptShutdown) {
- exit($this->err);
- }
- $this->inAbruptShutdown = true;
-
- foreach ($this->getDaemonHandles() as $daemon) {
- $daemon->didReceiveTerminalSignal($signo);
+ $this->pools[] = $pool;
}
}
- private function getDaemonHandles() {
- return ipull($this->daemons, 'handle');
+ private function getDaemonPools() {
+ return $this->pools;
}
+
/**
* Identify running daemons by examining the process table. This isn't
* completely reliable, but can be used as a fallback if the pid files fail
@@ -496,35 +306,45 @@
return;
}
- $daemons = array();
+ $pidfile = $this->toDictionary();
- foreach ($this->daemons as $daemon) {
- $handle = $daemon['handle'];
- $config = $daemon['config'];
+ if ($pidfile !== $this->lastPidfile) {
+ $this->lastPidfile = $pidfile;
+ $pidfile_path = $this->piddir.'/daemon.'.getmypid();
+ Filesystem::writeFile($pidfile_path, phutil_json_encode($pidfile));
+ }
+ }
- if (!$handle->isRunning()) {
- continue;
- }
+ public function toDictionary() {
+ $daemons = array();
+ foreach ($this->getDaemonPools() as $pool) {
+ foreach ($pool->getDaemons() as $daemon) {
+ if (!$daemon->isRunning()) {
+ continue;
+ }
- $daemons[] = array(
- 'pid' => $handle->getPID(),
- 'id' => $handle->getDaemonID(),
- 'config' => $config,
- );
+ $daemons[] = $daemon->toDictionary();
+ }
}
- $pidfile = array(
+ return array(
'pid' => getmypid(),
'start' => $this->startEpoch,
'config' => $this->config,
'daemons' => $daemons,
);
+ }
- if ($pidfile !== $this->lastPidfile) {
- $this->lastPidfile = $pidfile;
- $pidfile_path = $this->piddir.'/daemon.'.getmypid();
- Filesystem::writeFile($pidfile_path, json_encode($pidfile));
+ private function updateMemory() {
+ if (!$this->traceMemory) {
+ return;
}
+
+ $this->logMessage(
+ 'RAMS',
+ pht(
+ 'Overseer Memory Usage: %s KB',
+ new PhutilNumber(memory_get_usage() / 1024, 1)));
}
public function logMessage($type, $message, $context = null) {
@@ -533,4 +353,101 @@
}
}
+
+/* -( Signal Handling )---------------------------------------------------- */
+
+
+ /**
+ * @task signals
+ */
+ private function installSignalHandlers() {
+ $signals = array(
+ SIGUSR2,
+ SIGHUP,
+ SIGINT,
+ SIGTERM,
+ );
+
+ foreach ($signals as $signal) {
+ pcntl_signal($signal, array($this, 'didReceiveSignal'));
+ }
+ }
+
+
+ /**
+ * @task signals
+ */
+ public function didReceiveSignal($signo) {
+ switch ($signo) {
+ case SIGUSR2:
+ $signal_type = self::SIGNAL_NOTIFY;
+ break;
+ case SIGHUP:
+ $signal_type = self::SIGNAL_RELOAD;
+ break;
+ case SIGINT:
+ // If we receive SIGINT more than once, interpret it like SIGTERM.
+ if ($this->inGracefulShutdown) {
+ return $this->didReceiveSignal(SIGTERM);
+ }
+
+ $this->inGracefulShutdown = true;
+ $signal_type = self::SIGNAL_GRACEFUL;
+ break;
+ case SIGTERM:
+ // If we receive SIGTERM more than once, terminate abruptly.
+ $this->err = 128 + $signo;
+ if ($this->inAbruptShutdown) {
+ exit($this->err);
+ }
+
+ $this->inAbruptShutdown = true;
+ $signal_type = self::SIGNAL_TERMINATE;
+ break;
+ default:
+ throw new Exception(
+ pht(
+ 'Signal handler called with unknown signal type ("%d")!',
+ $signo));
+ }
+
+ foreach ($this->getDaemonPools() as $pool) {
+ $pool->didReceiveSignal($signal_type, $signo);
+ }
+ }
+
+
+/* -( Daemon Modules )----------------------------------------------------- */
+
+
+ private function getModules() {
+ return $this->modules;
+ }
+
+ private function shouldReloadDaemons() {
+ $modules = $this->getModules();
+
+ $should_reload = false;
+ foreach ($modules as $module) {
+ try {
+ // NOTE: Even if one module tells us to reload, we call the method on
+ // each module anyway to make calls a little more predictable.
+
+ if ($module->shouldReloadDaemons()) {
+ $this->logMessage(
+ 'RELO',
+ pht(
+ 'Reloading daemons (triggered by overseer module "%s").',
+ get_class($module)));
+ $should_reload = true;
+ }
+ } catch (Exception $ex) {
+ phlog($ex);
+ }
+ }
+
+ return $should_reload;
+ }
+
+
}
diff --git a/src/daemon/PhutilDaemonPool.php b/src/daemon/PhutilDaemonPool.php
new file mode 100644
--- /dev/null
+++ b/src/daemon/PhutilDaemonPool.php
@@ -0,0 +1,245 @@
+<?php
+
+final class PhutilDaemonPool extends Phobject {
+
+ private $properties = array();
+ private $commandLineArguments;
+
+ private $overseer;
+ private $daemons = array();
+ private $argv;
+
+ private function __construct() {
+ // <empty>
+ }
+
+ public static function newFromConfig(array $config) {
+ PhutilTypeSpec::checkMap(
+ $config,
+ array(
+ 'class' => 'string',
+ 'label' => 'string',
+ 'argv' => 'optional list<string>',
+ 'load' => 'optional list<string>',
+ 'log' => 'optional string|null',
+ 'pool' => 'optional int',
+ 'up' => 'optional int',
+ 'down' => 'optional int',
+ 'reserve' => 'optional int|float',
+ ));
+
+ $config = $config + array(
+ 'argv' => array(),
+ 'load' => array(),
+ 'log' => null,
+ 'pool' => 1,
+ 'up' => 2,
+ 'down' => 15,
+ 'reserve' => 0,
+ );
+
+ $pool = new self();
+ $pool->properties = $config;
+
+ return $pool;
+ }
+
+ public function setOverseer(PhutilDaemonOverseer $overseer) {
+ $this->overseer = $overseer;
+ return $this;
+ }
+
+ public function getOverseer() {
+ return $this->overseer;
+ }
+
+ public function setCommandLineArguments(array $arguments) {
+ $this->commandLineArguments = $arguments;
+ return $this;
+ }
+
+ public function getCommandLineArguments() {
+ return $this->commandLineArguments;
+ }
+
+ private function newDaemon() {
+ $config = $this->properties;
+
+ if (count($this->daemons)) {
+ $down_duration = $this->getPoolScaledownDuration();
+ } else {
+ // TODO: For now, never scale pools down to 0.
+ $down_duration = 0;
+ }
+
+ $forced_config = array(
+ 'down' => $down_duration,
+ );
+
+ $config = $forced_config + $config;
+
+ $config = array_select_keys(
+ $config,
+ array(
+ 'class',
+ 'log',
+ 'load',
+ 'argv',
+ 'down',
+ ));
+
+ $daemon = PhutilDaemonHandle::newFromConfig($config)
+ ->setDaemonPool($this)
+ ->setCommandLineArguments($this->getCommandLineArguments());
+
+ $daemon_id = $daemon->getDaemonID();
+ $this->daemons[$daemon_id] = $daemon;
+
+ $daemon->didLaunch();
+
+ return $daemon;
+ }
+
+ public function getDaemons() {
+ return $this->daemons;
+ }
+
+ public function getFutures() {
+ $futures = array();
+ foreach ($this->getDaemons() as $daemon) {
+ $future = $daemon->getFuture();
+ if ($future) {
+ $futures[] = $future;
+ }
+ }
+
+ return $futures;
+ }
+
+ public function didReceiveSignal($signal, $signo) {
+ foreach ($this->getDaemons() as $daemon) {
+ switch ($signal) {
+ case PhutilDaemonOverseer::SIGNAL_NOTIFY:
+ $daemon->didReceiveNotifySignal($signo);
+ break;
+ case PhutilDaemonOverseer::SIGNAL_RELOAD:
+ $daemon->didReceiveReloadSignal($signo);
+ break;
+ case PhutilDaemonOverseer::SIGNAL_GRACEFUL:
+ $daemon->didReceiveGracefulSignal($signo);
+ break;
+ case PhutilDaemonOverseer::SIGNAL_TERMINATE:
+ $daemon->didReceiveTerminateSignal($signo);
+ break;
+ default:
+ throw new Exception(
+ pht(
+ 'Unknown signal "%s" ("%d").',
+ $signal,
+ $signo));
+ }
+ }
+ }
+
+ public function getPoolLabel() {
+ return $this->getPoolProperty('label');
+ }
+
+ public function getPoolMaximumSize() {
+ return $this->getPoolProperty('pool');
+ }
+
+ public function getPoolScaleupDuration() {
+ return $this->getPoolProperty('up');
+ }
+
+ public function getPoolScaledownDuration() {
+ return $this->getPoolProperty('down');
+ }
+
+ public function getPoolMemoryReserve() {
+ return $this->getPoolProperty('reserve');
+ }
+
+ public function getPoolDaemonClass() {
+ return $this->getPoolProperty('class');
+ }
+
+ private function getPoolProperty($key) {
+ return idx($this->properties, $key);
+ }
+
+ public function updatePool() {
+ $daemons = $this->getDaemons();
+
+ foreach ($daemons as $key => $daemon) {
+ $daemon->update();
+
+ if ($daemon->isDone()) {
+ unset($daemons[$key]);
+ }
+ }
+
+ $this->updateAutoscale();
+ }
+
+ private function updateAutoscale() {
+ $daemons = $this->getDaemons();
+
+ // If this pool is already at the maximum size, we can't launch any new
+ // daemons.
+ $max_size = $this->getPoolMaximumSize();
+ if (count($daemons) >= $max_size) {
+ return;
+ }
+
+ $now = time();
+ $scaleup_duration = $this->getPoolScaleupDuration();
+
+ foreach ($daemons as $daemon) {
+ $busy_epoch = $daemon->getBusyEpoch();
+ // If any daemons haven't started work yet, don't scale the pool up.
+ if (!$busy_epoch) {
+ return;
+ }
+
+ // If any daemons started work very recently, wait a little while
+ // to scale the pool up.
+ $busy_for = ($now - $busy_epoch);
+ if ($busy_for < $scaleup_duration) {
+ return;
+ }
+ }
+
+ // If we have a configured memory reserve for this pool, it tells us that
+ // we should not scale up unless there's at least that much memory left
+ // on the system (for example, a reserve of 0.25 means that 25% of system
+ // memory must be free to autoscale).
+ $reserve = $this->getPoolMemoryReserve();
+ if ($reserve) {
+ // On some systems this may be slightly more expensive than other checks,
+ // so we only do it once we're prepared to scale up.
+ $memory = PhutilSystem::getSystemMemoryInformation();
+ $free_ratio = ($memory['free'] / $memory['total']);
+
+ // If we don't have enough free memory, don't scale.
+ if ($free_ratio <= $reserve) {
+ return;
+ }
+ }
+
+ $this->logMessage(
+ 'AUTO',
+ pht(
+ 'Scaling pool "%s" up to %s daemon(s).',
+ $this->getPoolLabel(),
+ new PhutilNumber(count($daemons) + 1)));
+
+ $this->newDaemon();
+ }
+
+ public function logMessage($type, $message, $context = null) {
+ return $this->getOverseer()->logMessage($type, $message, $context);
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Sun, Mar 9, 3:06 PM (2 d, 21 h ago)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/yr/p4/i4dzs5q4jltwoclu
Default Alt Text
D17389.id41813.diff (33 KB)

Event Timeline

Phabricator · Built by Phacility