D17389.diff
No OneTemporary
Actions

Size

33 KB

Referenced Files

None

Subscribers

None

D17389.diff
View Options

	diff --git a/scripts/daemon/exec/exec_daemon.php b/scripts/daemon/exec/exec_daemon.php
	--- a/scripts/daemon/exec/exec_daemon.php
	+++ b/scripts/daemon/exec/exec_daemon.php
	@@ -67,7 +67,7 @@
	'log' => 'optional string\|null',
	'argv' => 'optional list<wild>',
	'load' => 'optional list<string>',
	- 'autoscale' => 'optional wild',
	+ 'down' => 'optional int',
	));

	$log = idx($config, 'log');
	@@ -123,9 +123,9 @@
	$daemon->setVerbose(true);
	}

	-$autoscale = idx($config, 'autoscale');
	-if ($autoscale) {
	- $daemon->setAutoscaleProperties($autoscale);
	+$down_duration = idx($config, 'down');
	+if ($down_duration) {
	+ $daemon->setScaledownDuration($down_duration);
	}

	$daemon->execute();
	diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
	--- a/src/__phutil_library_map__.php
	+++ b/src/__phutil_library_map__.php
	@@ -188,6 +188,7 @@
	'PhutilDaemonHandle' => 'daemon/PhutilDaemonHandle.php',
	'PhutilDaemonOverseer' => 'daemon/PhutilDaemonOverseer.php',
	'PhutilDaemonOverseerModule' => 'daemon/PhutilDaemonOverseerModule.php',
	+ 'PhutilDaemonPool' => 'daemon/PhutilDaemonPool.php',
	'PhutilDefaultSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php',
	'PhutilDefaultSyntaxHighlighterEngine' => 'markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php',
	'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php',
	@@ -794,6 +795,7 @@
	'PhutilDaemonHandle' => 'Phobject',
	'PhutilDaemonOverseer' => 'Phobject',
	'PhutilDaemonOverseerModule' => 'Phobject',
	+ 'PhutilDaemonPool' => 'Phobject',
	'PhutilDefaultSyntaxHighlighter' => 'Phobject',
	'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine',
	'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy',
	diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php
	--- a/src/daemon/PhutilDaemon.php
	+++ b/src/daemon/PhutilDaemon.php
	@@ -59,7 +59,7 @@
	private $inGracefulShutdown;
	private $workState = null;
	private $idleSince = null;
	- private $autoscaleProperties = array();
	+ private $scaledownDuration;

	final public function setVerbose($verbose) {
	$this->verbose = $verbose;
	@@ -70,6 +70,15 @@
	return $this->verbose;
	}

	+ final public function setScaledownDuration($scaledown_duration) {
	+ $this->scaledownDuration = $scaledown_duration;
	+ return $this;
	+ }
	+
	+ final public function getScaledownDuration() {
	+ return $this->scaledownDuration;
	+ }
	+
	final public function __construct(array $argv) {
	$this->argv = $argv;

	@@ -121,15 +130,14 @@
	$this->willSleep($duration);
	$this->stillWorking();

	- $is_autoscale = $this->isClonedAutoscaleDaemon();
	- $scale_down = $this->getAutoscaleDownDuration();
	+ $scale_down = $this->getScaledownDuration();

	$max_sleep = 60;
	- if ($is_autoscale) {
	+ if ($scale_down) {
	$max_sleep = min($max_sleep, $scale_down);
	}

	- if ($is_autoscale) {
	+ if ($scale_down) {
	if ($this->workState == self::WORKSTATE_IDLE) {
	$dur = (time() - $this->idleSince);
	$this->log(pht('Idle for %s seconds.', $dur));
	@@ -143,7 +151,7 @@
	// If this is an autoscaling clone and we've been idle for too long,
	// we're going to scale the pool down by exiting and not restarting. The
	// DOWN message tells the overseer that we don't want to be restarted.
	- if ($is_autoscale) {
	+ if ($scale_down) {
	if ($this->workState == self::WORKSTATE_IDLE) {
	if ($this->idleSince && ($this->idleSince + $scale_down < time())) {
	$this->inGracefulShutdown = true;
	@@ -327,8 +335,8 @@
	* Prepare to idle. This may autoscale the pool down.
	*
	* This notifies the overseer that the daemon is no longer busy. If daemons
	- * that are part of an autoscale pool are idle for a prolonged period of time,
	- * they may exit to scale the pool down.
	+ * that are part of an autoscale pool are idle for a prolonged period of
	+ * time, they may exit to scale the pool down.
	*
	* @return this
	* @task autoscale
	@@ -342,66 +350,4 @@

	return $this;
	}
	-
	-
	- /**
	- * Determine if this is a clone or the original daemon.
	- *
	- * @return bool True if this is an cloned autoscaling daemon.
	- * @task autoscale
	- */
	- private function isClonedAutoscaleDaemon() {
	- return (bool)$this->getAutoscaleProperty('clone', false);
	- }
	-
	-
	- /**
	- * Get the duration (in seconds) which a daemon must be continuously idle
	- * for before it should exit to scale the pool down.
	- *
	- * @return int Duration, in seconds.
	- * @task autoscale
	- */
	- private function getAutoscaleDownDuration() {
	- return $this->getAutoscaleProperty('down', 15);
	- }
	-
	-
	- /**
	- * Configure autoscaling for this daemon.
	- *
	- * @param map<string, wild> Map of autoscale properties.
	- * @return this
	- * @task autoscale
	- */
	- public function setAutoscaleProperties(array $autoscale_properties) {
	- PhutilTypeSpec::checkMap(
	- $autoscale_properties,
	- array(
	- 'group' => 'optional string',
	- 'up' => 'optional int',
	- 'down' => 'optional int',
	- 'pool' => 'optional int',
	- 'clone' => 'optional bool',
	- 'reserve' => 'optional int\|float',
	- ));
	-
	- $this->autoscaleProperties = $autoscale_properties;
	-
	- return $this;
	- }
	-
	-
	- /**
	- * Read autoscaling configuration for this daemon.
	- *
	- * @param string Property to read.
	- * @param wild Default value to return if the property is not set.
	- * @return wild Property value, or `$default` if one is not set.
	- * @task autoscale
	- */
	- private function getAutoscaleProperty($key, $default = null) {
	- return idx($this->autoscaleProperties, $key, $default);
	- }
	-
	}
	diff --git a/src/daemon/PhutilDaemonHandle.php b/src/daemon/PhutilDaemonHandle.php
	--- a/src/daemon/PhutilDaemonHandle.php
	+++ b/src/daemon/PhutilDaemonHandle.php
	@@ -8,40 +8,96 @@
	const EVENT_WILL_GRACEFUL = 'daemon.willGraceful';
	const EVENT_WILL_EXIT = 'daemon.willExit';

	- private $overseer;
	- private $daemonClass;
	+ private $pool;
	+ private $properties;
	+ private $future;
	private $argv;
	- private $config;
	+
	+ private $restartAt;
	+ private $busyEpoch;
	+
	private $pid;
	private $daemonID;
	private $deadline;
	private $heartbeat;
	private $stdoutBuffer;
	- private $restartAt;
	private $shouldRestart = true;
	private $shouldShutdown;
	- private $future;
	- private $traceMemory;
	-
	- public function __construct(
	- PhutilDaemonOverseer $overseer,
	- $daemon_class,
	- array $argv,
	- array $config) {
	-
	- $this->overseer = $overseer;
	- $this->daemonClass = $daemon_class;
	- $this->argv = $argv;
	- $this->config = $config;
	+
	+ private function __construct() {
	+ // <empty>
	+ }
	+
	+ public static function newFromConfig(array $config) {
	+ PhutilTypeSpec::checkMap(
	+ $config,
	+ array(
	+ 'class' => 'string',
	+ 'argv' => 'optional list<string>',
	+ 'load' => 'optional list<string>',
	+ 'log' => 'optional string\|null',
	+ 'down' => 'optional int',
	+ ));
	+
	+ $config = $config + array(
	+ 'argv' => array(),
	+ 'load' => array(),
	+ 'log' => null,
	+ 'down' => 15,
	+ );
	+
	+ $daemon = new self();
	+ $daemon->properties = $config;
	+ $daemon->daemonID = $daemon->generateDaemonID();
	+
	+ return $daemon;
	+ }
	+
	+ public function setDaemonPool(PhutilDaemonPool $daemon_pool) {
	+ $this->pool = $daemon_pool;
	+ return $this;
	+ }
	+
	+ public function getDaemonPool() {
	+ return $this->pool;
	+ }
	+
	+ public function getBusyEpoch() {
	+ return $this->busyEpoch;
	+ }
	+
	+ public function getDaemonClass() {
	+ return $this->getProperty('class');
	+ }
	+
	+ private function getProperty($key) {
	+ return idx($this->properties, $key);
	+ }
	+
	+ public function setCommandLineArguments(array $arguments) {
	+ $this->argv = $arguments;
	+ return $this;
	+ }
	+
	+ public function getCommandLineArguments() {
	+ return $this->argv;
	+ }
	+
	+ public function getDaemonArguments() {
	+ return $this->getProperty('argv');
	+ }
	+
	+ public function didLaunch() {
	$this->restartAt = time();

	- $this->daemonID = $this->generateDaemonID();
	$this->dispatchEvent(
	self::EVENT_DID_LAUNCH,
	array(
	- 'argv' => $this->argv,
	- 'explicitArgv' => idx($this->config, 'argv'),
	+ 'argv' => $this->getCommandLineArguments(),
	+ 'explicitArgv' => $this->getDaemonArguments(),
	));
	+
	+ return $this;
	}

	public function isRunning() {
	@@ -56,18 +112,7 @@
	return $this->future;
	}

	- public function setTraceMemory($trace_memory) {
	- $this->traceMemory = $trace_memory;
	- return $this;
	- }
	-
	- public function getTraceMemory() {
	- return $this->traceMemory;
	- }
	-
	public function update() {
	- $this->updateMemory();
	-
	if (!$this->isRunning()) {
	if (!$this->shouldRestart) {
	return;
	@@ -115,6 +160,7 @@

	if ($this->shouldShutdown) {
	$this->restartAt = null;
	+ $this->dispatchEvent(self::EVENT_WILL_EXIT);
	} else {
	$this->scheduleRestart();
	}
	@@ -193,8 +239,8 @@
	}

	private function newExecFuture() {
	- $class = $this->daemonClass;
	- $argv = $this->argv;
	+ $class = $this->getDaemonClass();
	+ $argv = $this->getCommandLineArguments();
	$buffer_size = $this->getCaptureBufferSize();

	// NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this
	@@ -210,11 +256,15 @@
	// the shell process won't properly posix_setsid() so the pgid of the child
	// won't be meaningful.

	+ $config = $this->properties;
	+ unset($config['class']);
	+ $config = phutil_json_encode($config);
	+
	return id(new ExecFuture('exec ./exec_daemon.php %s %Ls', $class, $argv))
	->setCWD($this->getDaemonCWD())
	->setStdoutSizeLimit($buffer_size)
	->setStderrSizeLimit($buffer_size)
	- ->write(json_encode($this->config));
	+ ->write($config);
	}

	/**
	@@ -226,9 +276,9 @@
	*/
	private function dispatchEvent($type, array $params = array()) {
	$data = array(
	- 'id' => $this->daemonID,
	- 'daemonClass' => $this->daemonClass,
	- 'childPID' => $this->pid,
	+ 'id' => $this->getDaemonID(),
	+ 'daemonClass' => $this->getDaemonClass(),
	+ 'childPID' => $this->getPID(),
	) + $params;

	$event = new PhutilEvent($type, $data);
	@@ -241,7 +291,8 @@
	}

	private function annihilateProcessGroup() {
	- $pid = $this->pid;
	+ $pid = $this->getPID();
	+
	$pgid = posix_getpgid($pid);
	if ($pid && $pgid) {
	posix_kill(-$pgid, SIGTERM);
	@@ -251,16 +302,6 @@
	}
	}

	- private function updateMemory() {
	- if ($this->traceMemory) {
	- $this->logMessage(
	- 'RAMS',
	- pht(
	- 'Overseer Memory Usage: %s KB',
	- new PhutilNumber(memory_get_usage() / 1024, 1)));
	- }
	- }
	-
	private function startDaemonProcess() {
	$this->logMessage('INIT', pht('Starting process.'));

	@@ -298,10 +339,12 @@
	$this->deadline = time() + $this->getRequiredHeartbeatFrequency();
	break;
	case PhutilDaemon::MESSAGETYPE_BUSY:
	- $this->overseer->didBeginWork($this);
	+ if (!$this->busyEpoch) {
	+ $this->busyEpoch = time();
	+ }
	break;
	case PhutilDaemon::MESSAGETYPE_IDLE:
	- $this->overseer->didBeginIdle($this);
	+ $this->busyEpoch = null;
	break;
	case PhutilDaemon::MESSAGETYPE_DOWN:
	// The daemon is exiting because it doesn't have enough work and it
	@@ -319,7 +362,7 @@
	}

	public function didReceiveNotifySignal($signo) {
	- $pid = $this->pid;
	+ $pid = $this->getPID();
	if ($pid) {
	posix_kill($pid, $signo);
	}
	@@ -349,7 +392,7 @@
	// naturally be restarted after it exits, as though it had exited after an
	// unhandled exception.

	- posix_kill($this->pid, SIGINT);
	+ posix_kill($this->getPID(), SIGINT);
	}

	public function didReceiveGracefulSignal($signo) {
	@@ -370,10 +413,10 @@

	$this->logMessage('DONE', $sigmsg, $signo);

	- posix_kill($this->pid, SIGINT);
	+ posix_kill($this->getPID(), SIGINT);
	}

	- public function didReceiveTerminalSignal($signo) {
	+ public function didReceiveTerminateSignal($signo) {
	$this->shouldShutdown = true;
	$this->shouldRestart = false;

	@@ -392,7 +435,8 @@
	}

	private function logMessage($type, $message, $context = null) {
	- $this->overseer->logMessage($type, $message, $context);
	+ $this->getDaemonPool()->logMessage($type, $message, $context);
	+
	$this->dispatchEvent(
	self::EVENT_DID_LOG,
	array(
	@@ -402,8 +446,13 @@
	));
	}

	- public function didRemoveDaemon() {
	- $this->dispatchEvent(self::EVENT_WILL_EXIT);
	+ public function toDictionary() {
	+ return array(
	+ 'pid' => $this->getPID(),
	+ 'id' => $this->getDaemonID(),
	+ 'config' => $this->properties,
	+ );
	}

	+
	}
	diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php
	--- a/src/daemon/PhutilDaemonOverseer.php
	+++ b/src/daemon/PhutilDaemonOverseer.php
	@@ -2,17 +2,16 @@

	/**
	* Oversees a daemon and restarts it if it fails.
	+ *
	+ * @task signals Signal Handling
	*/
	final class PhutilDaemonOverseer extends Phobject {

	private $argv;
	- private $moreArgs;
	- private $inAbruptShutdown;
	- private $inGracefulShutdown;
	private static $instance;

	private $config;
	- private $daemons = array();
	+ private $pools = array();
	private $traceMode;
	private $traceMemory;
	private $daemonize;
	@@ -21,12 +20,20 @@
	private $libraries = array();
	private $modules = array();
	private $verbose;
	- private $err = 0;
	private $lastPidfile;
	private $startEpoch;
	private $autoscale = array();
	private $autoscaleConfig = array();

	+ const SIGNAL_NOTIFY = 'signal/notify';
	+ const SIGNAL_RELOAD = 'signal/reload';
	+ const SIGNAL_GRACEFUL = 'signal/graceful';
	+ const SIGNAL_TERMINATE = 'signal/terminate';
	+
	+ private $err = 0;
	+ private $inAbruptShutdown;
	+ private $inGracefulShutdown;
	+
	public function __construct(array $argv) {
	PhutilServiceProfiler::getInstance()->enableDiscardMode();

	@@ -149,11 +156,7 @@

	$this->modules = PhutilDaemonOverseerModule::getAllModules();

	- pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal'));
	-
	- pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal'));
	- pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal'));
	- pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal'));
	+ $this->installSignalHandlers();
	}

	public function addLibrary($library) {
	@@ -162,273 +165,80 @@
	}

	public function run() {
	- $this->daemons = array();
	-
	- foreach ($this->config['daemons'] as $config) {
	- $config += array(
	- 'argv' => array(),
	- 'autoscale' => array(),
	- );
	-
	- $daemon = new PhutilDaemonHandle(
	- $this,
	- $config['class'],
	- $this->argv,
	- array(
	- 'log' => $this->log,
	- 'argv' => $config['argv'],
	- 'load' => $this->libraries,
	- 'autoscale' => $config['autoscale'],
	- ));
	-
	- $daemon->setTraceMemory($this->traceMemory);
	-
	- $this->addDaemon($daemon, $config);
	-
	- $group = idx($config['autoscale'], 'group');
	- if (strlen($group)) {
	- if (isset($this->autoscaleConfig[$group])) {
	- throw new Exception(
	- pht(
	- 'Two daemons are part of the same autoscale group ("%s"). '.
	- 'Each daemon autoscale group must be unique.',
	- $group));
	- }
	- $this->autoscaleConfig[$group] = $config;
	- }
	- }
	-
	- $should_reload = false;
	+ $this->createDaemonPools();

	while (true) {
	- foreach ($this->modules as $module) {
	- try {
	- if ($module->shouldReloadDaemons()) {
	- $this->logMessage(
	- 'RELO',
	- pht(
	- 'Reloading daemons (triggered by overseer module "%s").',
	- get_class($module)));
	- $should_reload = true;
	- }
	- } catch (Exception $ex) {
	- phlog($ex);
	- }
	- }
	-
	- if ($should_reload) {
	- $this->didReceiveReloadSignal(SIGHUP);
	- $should_reload = false;
	+ if ($this->shouldReloadDaemons()) {
	+ $this->didReceiveSignal(SIGHUP);
	}

	$futures = array();
	- foreach ($this->getDaemonHandles() as $daemon) {
	- $daemon->update();
	- if ($daemon->isRunning()) {
	- $futures[] = $daemon->getFuture();
	- }
	+ foreach ($this->getDaemonPools() as $pool) {
	+ $pool->updatePool();

	- if ($daemon->isDone()) {
	- $this->removeDaemon($daemon);
	+ foreach ($pool->getFutures() as $future) {
	+ $futures[] = $future;
	}
	}

	$this->updatePidfile();
	- $this->updateAutoscale();
	+ $this->updateMemory();

	- if ($futures) {
	- $iter = id(new FutureIterator($futures))
	- ->setUpdateInterval(1);
	- foreach ($iter as $future) {
	- break;
	- }
	- } else {
	+ $this->waitForDaemonFutures($futures);
	+
	+ if (!$futures) {
	if ($this->inGracefulShutdown) {
	break;
	}
	- sleep(1);
	}
	}

	exit($this->err);
	}

	- private function addDaemon(PhutilDaemonHandle $daemon, array $config) {
	- $id = $daemon->getDaemonID();
	- $this->daemons[$id] = array(
	- 'handle' => $daemon,
	- 'config' => $config,
	- );

	- $autoscale_group = $this->getAutoscaleGroup($daemon);
	- if ($autoscale_group) {
	- $this->autoscale[$autoscale_group][$id] = true;
	- }
	-
	- return $this;
	- }
	-
	- private function removeDaemon(PhutilDaemonHandle $daemon) {
	- $id = $daemon->getDaemonID();
	-
	- $autoscale_group = $this->getAutoscaleGroup($daemon);
	- if ($autoscale_group) {
	- unset($this->autoscale[$autoscale_group][$id]);
	- }
	+ private function waitForDaemonFutures(array $futures) {
	+ assert_instances_of($futures, 'ExecFuture');

	- unset($this->daemons[$id]);
	-
	- $daemon->didRemoveDaemon();
	-
	- return $this;
	- }
	-
	- private function getAutoscaleGroup(PhutilDaemonHandle $daemon) {
	- $id = $daemon->getDaemonID();
	- $autoscale = $this->daemons[$id]['config']['autoscale'];
	- return idx($autoscale, 'group');
	- }
	-
	- private function getAutoscaleProperty($group_key, $key, $default = null) {
	- $config = $this->autoscaleConfig[$group_key]['autoscale'];
	- return idx($config, $key, $default);
	- }
	-
	- public function didBeginWork(PhutilDaemonHandle $daemon) {
	- $id = $daemon->getDaemonID();
	- $busy = idx($this->daemons[$daemon->getDaemonID()], 'busy');
	- if (!$busy) {
	- $this->daemons[$id]['busy'] = time();
	- }
	- }
	-
	- public function didBeginIdle(PhutilDaemonHandle $daemon) {
	- $id = $daemon->getDaemonID();
	- unset($this->daemons[$id]['busy']);
	- }
	-
	- public function updateAutoscale() {
	- if ($this->inGracefulShutdown) {
	- return;
	- }
	-
	- foreach ($this->autoscale as $group => $daemons) {
	- $scaleup_duration = $this->getAutoscaleProperty($group, 'up', 2);
	- $max_pool_size = $this->getAutoscaleProperty($group, 'pool', 8);
	- $reserve = $this->getAutoscaleProperty($group, 'reserve', 0);
	-
	- // Don't scale a group if it is already at the maximum pool size.
	- if (count($daemons) >= $max_pool_size) {
	- continue;
	+ if ($futures) {
	+ // TODO: This only wakes if any daemons actually exit. It would be a bit
	+ // cleaner to wait on any I/O with Channels.
	+ $iter = id(new FutureIterator($futures))
	+ ->setUpdateInterval(1);
	+ foreach ($iter as $future) {
	+ break;
	}
	-
	- $should_scale = true;
	- foreach ($daemons as $daemon_id => $ignored) {
	- $busy = idx($this->daemons[$daemon_id], 'busy');
	- if (!$busy) {
	- // At least one daemon in the group hasn't reported that it has
	- // started work.
	- $should_scale = false;
	- break;
	- }
	-
	- if ((time() - $busy) < $scaleup_duration) {
	- // At least one daemon in the group was idle recently, so we have
	- // not fullly
	- $should_scale = false;
	- break;
	- }
	- }
	-
	- // If we have a configured memory reserve for this pool, it tells us that
	- // we should not scale up unless there's at least that much memory left
	- // on the system (for example, a reserve of 0.25 means that 25% of system
	- // memory must be free to autoscale).
	- if ($should_scale && $reserve) {
	- // On some systems this may be slightly more expensive than other
	- // checks, so only do it once we're prepared to scale up.
	- $memory = PhutilSystem::getSystemMemoryInformation();
	- $free_ratio = ($memory['free'] / $memory['total']);
	-
	- // If we don't have enough free memory, don't scale.
	- if ($free_ratio <= $reserve) {
	- continue;
	- }
	- }
	-
	- if ($should_scale) {
	- $config = $this->autoscaleConfig[$group];
	-
	- $config['autoscale']['clone'] = true;
	-
	- $clone = new PhutilDaemonHandle(
	- $this,
	- $config['class'],
	- $this->argv,
	- array(
	- 'log' => $this->log,
	- 'argv' => $config['argv'],
	- 'load' => $this->libraries,
	- 'autoscale' => $config['autoscale'],
	- ));
	-
	- $this->logMessage(
	- 'AUTO',
	- pht(
	- 'Scaling pool "%s" up to %s daemon(s).',
	- $group,
	- new PhutilNumber(count($daemons) + 1)));
	-
	- $this->addDaemon($clone, $config);
	-
	- // Don't scale more than one pool up per iteration. Otherwise, we could
	- // break the memory barrier if we have a lot of pools and scale them
	- // all up at once.
	- return;
	+ } else {
	+ if (!$this->inGracefulShutdown) {
	+ sleep(1);
	}
	}
	}

	- public function didReceiveNotifySignal($signo) {
	- foreach ($this->getDaemonHandles() as $daemon) {
	- $daemon->didReceiveNotifySignal($signo);
	- }
	- }
	+ private function createDaemonPools() {
	+ $configs = $this->config['daemons'];

	- public function didReceiveReloadSignal($signo) {
	- foreach ($this->getDaemonHandles() as $daemon) {
	- $daemon->didReceiveReloadSignal($signo);
	- }
	- }
	+ $forced_options = array(
	+ 'load' => $this->libraries,
	+ 'log' => $this->log,
	+ );

	- public function didReceiveGracefulSignal($signo) {
	- // If we receive SIGINT more than once, interpret it like SIGTERM.
	- if ($this->inGracefulShutdown) {
	- return $this->didReceiveTerminalSignal($signo);
	- }
	- $this->inGracefulShutdown = true;
	+ foreach ($configs as $config) {
	+ $config = $forced_options + $config;

	- foreach ($this->getDaemonHandles() as $daemon) {
	- $daemon->didReceiveGracefulSignal($signo);
	- }
	- }
	+ $pool = PhutilDaemonPool::newFromConfig($config)
	+ ->setOverseer($this)
	+ ->setCommandLineArguments($this->argv);

	- public function didReceiveTerminalSignal($signo) {
	- $this->err = 128 + $signo;
	- if ($this->inAbruptShutdown) {
	- exit($this->err);
	- }
	- $this->inAbruptShutdown = true;
	-
	- foreach ($this->getDaemonHandles() as $daemon) {
	- $daemon->didReceiveTerminalSignal($signo);
	+ $this->pools[] = $pool;
	}
	}

	- private function getDaemonHandles() {
	- return ipull($this->daemons, 'handle');
	+ private function getDaemonPools() {
	+ return $this->pools;
	}

	+
	/**
	* Identify running daemons by examining the process table. This isn't
	* completely reliable, but can be used as a fallback if the pid files fail
	@@ -496,35 +306,45 @@
	return;
	}

	- $daemons = array();
	+ $pidfile = $this->toDictionary();

	- foreach ($this->daemons as $daemon) {
	- $handle = $daemon['handle'];
	- $config = $daemon['config'];
	+ if ($pidfile !== $this->lastPidfile) {
	+ $this->lastPidfile = $pidfile;
	+ $pidfile_path = $this->piddir.'/daemon.'.getmypid();
	+ Filesystem::writeFile($pidfile_path, phutil_json_encode($pidfile));
	+ }
	+ }

	- if (!$handle->isRunning()) {
	- continue;
	- }
	+ public function toDictionary() {
	+ $daemons = array();
	+ foreach ($this->getDaemonPools() as $pool) {
	+ foreach ($pool->getDaemons() as $daemon) {
	+ if (!$daemon->isRunning()) {
	+ continue;
	+ }

	- $daemons[] = array(
	- 'pid' => $handle->getPID(),
	- 'id' => $handle->getDaemonID(),
	- 'config' => $config,
	- );
	+ $daemons[] = $daemon->toDictionary();
	+ }
	}

	- $pidfile = array(
	+ return array(
	'pid' => getmypid(),
	'start' => $this->startEpoch,
	'config' => $this->config,
	'daemons' => $daemons,
	);
	+ }

	- if ($pidfile !== $this->lastPidfile) {
	- $this->lastPidfile = $pidfile;
	- $pidfile_path = $this->piddir.'/daemon.'.getmypid();
	- Filesystem::writeFile($pidfile_path, json_encode($pidfile));
	+ private function updateMemory() {
	+ if (!$this->traceMemory) {
	+ return;
	}
	+
	+ $this->logMessage(
	+ 'RAMS',
	+ pht(
	+ 'Overseer Memory Usage: %s KB',
	+ new PhutilNumber(memory_get_usage() / 1024, 1)));
	}

	public function logMessage($type, $message, $context = null) {
	@@ -533,4 +353,101 @@
	}
	}

	+
	+/* -( Signal Handling )---------------------------------------------------- */
	+
	+
	+ /**
	+ * @task signals
	+ */
	+ private function installSignalHandlers() {
	+ $signals = array(
	+ SIGUSR2,
	+ SIGHUP,
	+ SIGINT,
	+ SIGTERM,
	+ );
	+
	+ foreach ($signals as $signal) {
	+ pcntl_signal($signal, array($this, 'didReceiveSignal'));
	+ }
	+ }
	+
	+
	+ /**
	+ * @task signals
	+ */
	+ public function didReceiveSignal($signo) {
	+ switch ($signo) {
	+ case SIGUSR2:
	+ $signal_type = self::SIGNAL_NOTIFY;
	+ break;
	+ case SIGHUP:
	+ $signal_type = self::SIGNAL_RELOAD;
	+ break;
	+ case SIGINT:
	+ // If we receive SIGINT more than once, interpret it like SIGTERM.
	+ if ($this->inGracefulShutdown) {
	+ return $this->didReceiveSignal(SIGTERM);
	+ }
	+
	+ $this->inGracefulShutdown = true;
	+ $signal_type = self::SIGNAL_GRACEFUL;
	+ break;
	+ case SIGTERM:
	+ // If we receive SIGTERM more than once, terminate abruptly.
	+ $this->err = 128 + $signo;
	+ if ($this->inAbruptShutdown) {
	+ exit($this->err);
	+ }
	+
	+ $this->inAbruptShutdown = true;
	+ $signal_type = self::SIGNAL_TERMINATE;
	+ break;
	+ default:
	+ throw new Exception(
	+ pht(
	+ 'Signal handler called with unknown signal type ("%d")!',
	+ $signo));
	+ }
	+
	+ foreach ($this->getDaemonPools() as $pool) {
	+ $pool->didReceiveSignal($signal_type, $signo);
	+ }
	+ }
	+
	+
	+/* -( Daemon Modules )----------------------------------------------------- */
	+
	+
	+ private function getModules() {
	+ return $this->modules;
	+ }
	+
	+ private function shouldReloadDaemons() {
	+ $modules = $this->getModules();
	+
	+ $should_reload = false;
	+ foreach ($modules as $module) {
	+ try {
	+ // NOTE: Even if one module tells us to reload, we call the method on
	+ // each module anyway to make calls a little more predictable.
	+
	+ if ($module->shouldReloadDaemons()) {
	+ $this->logMessage(
	+ 'RELO',
	+ pht(
	+ 'Reloading daemons (triggered by overseer module "%s").',
	+ get_class($module)));
	+ $should_reload = true;
	+ }
	+ } catch (Exception $ex) {
	+ phlog($ex);
	+ }
	+ }
	+
	+ return $should_reload;
	+ }
	+
	+
	}
	diff --git a/src/daemon/PhutilDaemonPool.php b/src/daemon/PhutilDaemonPool.php
	new file mode 100644
	--- /dev/null
	+++ b/src/daemon/PhutilDaemonPool.php
	@@ -0,0 +1,245 @@
	+<?php
	+
	+final class PhutilDaemonPool extends Phobject {
	+
	+ private $properties = array();
	+ private $commandLineArguments;
	+
	+ private $overseer;
	+ private $daemons = array();
	+ private $argv;
	+
	+ private function __construct() {
	+ // <empty>
	+ }
	+
	+ public static function newFromConfig(array $config) {
	+ PhutilTypeSpec::checkMap(
	+ $config,
	+ array(
	+ 'class' => 'string',
	+ 'label' => 'string',
	+ 'argv' => 'optional list<string>',
	+ 'load' => 'optional list<string>',
	+ 'log' => 'optional string\|null',
	+ 'pool' => 'optional int',
	+ 'up' => 'optional int',
	+ 'down' => 'optional int',
	+ 'reserve' => 'optional int\|float',
	+ ));
	+
	+ $config = $config + array(
	+ 'argv' => array(),
	+ 'load' => array(),
	+ 'log' => null,
	+ 'pool' => 1,
	+ 'up' => 2,
	+ 'down' => 15,
	+ 'reserve' => 0,
	+ );
	+
	+ $pool = new self();
	+ $pool->properties = $config;
	+
	+ return $pool;
	+ }
	+
	+ public function setOverseer(PhutilDaemonOverseer $overseer) {
	+ $this->overseer = $overseer;
	+ return $this;
	+ }
	+
	+ public function getOverseer() {
	+ return $this->overseer;
	+ }
	+
	+ public function setCommandLineArguments(array $arguments) {
	+ $this->commandLineArguments = $arguments;
	+ return $this;
	+ }
	+
	+ public function getCommandLineArguments() {
	+ return $this->commandLineArguments;
	+ }
	+
	+ private function newDaemon() {
	+ $config = $this->properties;
	+
	+ if (count($this->daemons)) {
	+ $down_duration = $this->getPoolScaledownDuration();
	+ } else {
	+ // TODO: For now, never scale pools down to 0.
	+ $down_duration = 0;
	+ }
	+
	+ $forced_config = array(
	+ 'down' => $down_duration,
	+ );
	+
	+ $config = $forced_config + $config;
	+
	+ $config = array_select_keys(
	+ $config,
	+ array(
	+ 'class',
	+ 'log',
	+ 'load',
	+ 'argv',
	+ 'down',
	+ ));
	+
	+ $daemon = PhutilDaemonHandle::newFromConfig($config)
	+ ->setDaemonPool($this)
	+ ->setCommandLineArguments($this->getCommandLineArguments());
	+
	+ $daemon_id = $daemon->getDaemonID();
	+ $this->daemons[$daemon_id] = $daemon;
	+
	+ $daemon->didLaunch();
	+
	+ return $daemon;
	+ }
	+
	+ public function getDaemons() {
	+ return $this->daemons;
	+ }
	+
	+ public function getFutures() {
	+ $futures = array();
	+ foreach ($this->getDaemons() as $daemon) {
	+ $future = $daemon->getFuture();
	+ if ($future) {
	+ $futures[] = $future;
	+ }
	+ }
	+
	+ return $futures;
	+ }
	+
	+ public function didReceiveSignal($signal, $signo) {
	+ foreach ($this->getDaemons() as $daemon) {
	+ switch ($signal) {
	+ case PhutilDaemonOverseer::SIGNAL_NOTIFY:
	+ $daemon->didReceiveNotifySignal($signo);
	+ break;
	+ case PhutilDaemonOverseer::SIGNAL_RELOAD:
	+ $daemon->didReceiveReloadSignal($signo);
	+ break;
	+ case PhutilDaemonOverseer::SIGNAL_GRACEFUL:
	+ $daemon->didReceiveGracefulSignal($signo);
	+ break;
	+ case PhutilDaemonOverseer::SIGNAL_TERMINATE:
	+ $daemon->didReceiveTerminateSignal($signo);
	+ break;
	+ default:
	+ throw new Exception(
	+ pht(
	+ 'Unknown signal "%s" ("%d").',
	+ $signal,
	+ $signo));
	+ }
	+ }
	+ }
	+
	+ public function getPoolLabel() {
	+ return $this->getPoolProperty('label');
	+ }
	+
	+ public function getPoolMaximumSize() {
	+ return $this->getPoolProperty('pool');
	+ }
	+
	+ public function getPoolScaleupDuration() {
	+ return $this->getPoolProperty('up');
	+ }
	+
	+ public function getPoolScaledownDuration() {
	+ return $this->getPoolProperty('down');
	+ }
	+
	+ public function getPoolMemoryReserve() {
	+ return $this->getPoolProperty('reserve');
	+ }
	+
	+ public function getPoolDaemonClass() {
	+ return $this->getPoolProperty('class');
	+ }
	+
	+ private function getPoolProperty($key) {
	+ return idx($this->properties, $key);
	+ }
	+
	+ public function updatePool() {
	+ $daemons = $this->getDaemons();
	+
	+ foreach ($daemons as $key => $daemon) {
	+ $daemon->update();
	+
	+ if ($daemon->isDone()) {
	+ unset($daemons[$key]);
	+ }
	+ }
	+
	+ $this->updateAutoscale();
	+ }
	+
	+ private function updateAutoscale() {
	+ $daemons = $this->getDaemons();
	+
	+ // If this pool is already at the maximum size, we can't launch any new
	+ // daemons.
	+ $max_size = $this->getPoolMaximumSize();
	+ if (count($daemons) >= $max_size) {
	+ return;
	+ }
	+
	+ $now = time();
	+ $scaleup_duration = $this->getPoolScaleupDuration();
	+
	+ foreach ($daemons as $daemon) {
	+ $busy_epoch = $daemon->getBusyEpoch();
	+ // If any daemons haven't started work yet, don't scale the pool up.
	+ if (!$busy_epoch) {
	+ return;
	+ }
	+
	+ // If any daemons started work very recently, wait a little while
	+ // to scale the pool up.
	+ $busy_for = ($now - $busy_epoch);
	+ if ($busy_for < $scaleup_duration) {
	+ return;
	+ }
	+ }
	+
	+ // If we have a configured memory reserve for this pool, it tells us that
	+ // we should not scale up unless there's at least that much memory left
	+ // on the system (for example, a reserve of 0.25 means that 25% of system
	+ // memory must be free to autoscale).
	+ $reserve = $this->getPoolMemoryReserve();
	+ if ($reserve) {
	+ // On some systems this may be slightly more expensive than other checks,
	+ // so we only do it once we're prepared to scale up.
	+ $memory = PhutilSystem::getSystemMemoryInformation();
	+ $free_ratio = ($memory['free'] / $memory['total']);
	+
	+ // If we don't have enough free memory, don't scale.
	+ if ($free_ratio <= $reserve) {
	+ return;
	+ }
	+ }
	+
	+ $this->logMessage(
	+ 'AUTO',
	+ pht(
	+ 'Scaling pool "%s" up to %s daemon(s).',
	+ $this->getPoolLabel(),
	+ new PhutilNumber(count($daemons) + 1)));
	+
	+ $this->newDaemon();
	+ }
	+
	+ public function logMessage($type, $message, $context = null) {
	+ return $this->getOverseer()->logMessage($type, $message, $context);
	+ }
	+
	+}

File Metadata

Mime Type: text/plain
Expires: Sep 5 2025, 9:12 PM (6 w, 4 d ago)
Storage Engine: amazon-s3
Storage Format: Encrypted (AES-256-CBC)
Storage Handle: phabricator/secure/hb/23/c67afimixvvj42m6
Default Alt Text: D17389.diff (33 KB)

D17389.diffNo OneTemporaryActions

D17389.diffView Options

File Metadata

Event Timeline

D17389.diff
No OneTemporary
Actions

D17389.diff
View Options