diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php index aceff0f..c48d3de 100644 --- a/src/daemon/PhutilDaemon.php +++ b/src/daemon/PhutilDaemon.php @@ -1,398 +1,396 @@ shouldExit()) { * if (work_available()) { * $this->willBeginWork(); * do_work(); * $this->sleep(0); * } else { * $this->willBeginIdle(); * $this->sleep(1); * } * } * * In particular, call @{method:willBeginWork} before becoming busy, and * @{method:willBeginIdle} when no work is available. If the daemon is launched * into an autoscale pool, this will cause the pool to automatically scale up * when busy and down when idle. * * See @{class:PhutilHighIntensityIntervalDaemon} for an example of a simple * autoscaling daemon. * * Launching a daemon which does not make these callbacks into an autoscale * pool will have no effect. * * @task overseer Communicating With the Overseer * @task autoscale Autoscaling Daemon Pools - * - * @stable */ abstract class PhutilDaemon { const MESSAGETYPE_STDOUT = 'stdout'; const MESSAGETYPE_HEARTBEAT = 'heartbeat'; const MESSAGETYPE_BUSY = 'busy'; const MESSAGETYPE_IDLE = 'idle'; const MESSAGETYPE_DOWN = 'down'; const WORKSTATE_BUSY = 'busy'; const WORKSTATE_IDLE = 'idle'; private $argv; private $traceMode; private $traceMemory; private $verbose; private $notifyReceived; private $inGracefulShutdown; private $workState = null; private $idleSince = null; private $autoscaleProperties = array(); final public function setVerbose($verbose) { $this->verbose = $verbose; return $this; } final public function getVerbose() { return $this->verbose; } private static $sighandlerInstalled; final public function __construct(array $argv) { declare(ticks = 1); $this->argv = $argv; if (!self::$sighandlerInstalled) { self::$sighandlerInstalled = true; pcntl_signal(SIGTERM, __CLASS__.'::exitOnSignal'); } pcntl_signal(SIGINT, array($this, 'onGracefulSignal')); pcntl_signal(SIGUSR2, array($this, 'onNotifySignal')); // Without discard mode, this consumes unbounded amounts of memory. Keep // memory bounded. PhutilServiceProfiler::getInstance()->enableDiscardMode(); $this->beginStdoutCapture(); } final public function __destruct() { $this->endStdoutCapture(); } final public function stillWorking() { $this->emitOverseerMessage(self::MESSAGETYPE_HEARTBEAT, null); if ($this->traceMemory) { $daemon = get_class($this); fprintf( STDERR, "<%s> %s %s\n", '', $daemon, pht( 'Memory Usage: %s KB', new PhutilNumber(memory_get_usage() / 1024, 1))); } } final public function shouldExit() { return $this->inGracefulShutdown; } final protected function sleep($duration) { $this->notifyReceived = false; $this->willSleep($duration); $this->stillWorking(); $is_autoscale = $this->isClonedAutoscaleDaemon(); $scale_down = $this->getAutoscaleDownDuration(); $max_sleep = 60; if ($is_autoscale) { $max_sleep = min($max_sleep, $scale_down); } if ($is_autoscale) { if ($this->workState == self::WORKSTATE_IDLE) { $dur = (time() - $this->idleSince); $this->log(pht('Idle for %s seconds.', $dur)); } } while ($duration > 0 && !$this->notifyReceived && !$this->shouldExit()) { // If this is an autoscaling clone and we've been idle for too long, // we're going to scale the pool down by exiting and not restarting. The // DOWN message tells the overseer that we don't want to be restarted. if ($is_autoscale) { if ($this->workState == self::WORKSTATE_IDLE) { if ($this->idleSince && ($this->idleSince + $scale_down < time())) { $this->inGracefulShutdown = true; $this->emitOverseerMessage(self::MESSAGETYPE_DOWN, null); $this->log( pht( 'Daemon was idle for more than %s seconds, scaling pool '. 'down.', $scale_down)); break; } } } sleep(min($duration, $max_sleep)); $duration -= $max_sleep; $this->stillWorking(); } } protected function willSleep($duration) { return; } public static function exitOnSignal($signo) { // Normally, PHP doesn't invoke destructors when existing in response to // a signal. This forces it to do so, so we have a fighting chance of // releasing any locks, leases or resources on our way out. exit(128 + $signo); } final protected function getArgv() { return $this->argv; } final public function execute() { $this->willRun(); $this->run(); } abstract protected function run(); final public function setTraceMemory() { $this->traceMemory = true; return $this; } final public function getTraceMemory() { return $this->traceMemory; } final public function setTraceMode() { $this->traceMode = true; PhutilServiceProfiler::installEchoListener(); PhutilConsole::getConsole()->getServer()->setEnableLog(true); $this->didSetTraceMode(); return $this; } final public function getTraceMode() { return $this->traceMode; } final public function onGracefulSignal($signo) { $this->inGracefulShutdown = true; } final public function onNotifySignal($signo) { $this->notifyReceived = true; $this->onNotify($signo); } protected function onNotify($signo) { // This is a hook for subclasses. } protected function willRun() { // This is a hook for subclasses. } protected function didSetTraceMode() { // This is a hook for subclasses. } final protected function log($message) { if ($this->verbose) { $daemon = get_class($this); fprintf(STDERR, "<%s> %s %s\n", '', $daemon, $message); } } /* -( Communicating With the Overseer )------------------------------------ */ private function beginStdoutCapture() { ob_start(array($this, 'didReceiveStdout'), 2); } private function endStdoutCapture() { ob_end_flush(); } public function didReceiveStdout($data) { if (!strlen($data)) { return ''; } return $this->encodeOverseerMessage(self::MESSAGETYPE_STDOUT, $data); } private function encodeOverseerMessage($type, $data) { $structure = array($type); if ($data !== null) { $structure[] = $data; } return json_encode($structure)."\n"; } private function emitOverseerMessage($type, $data) { $this->endStdoutCapture(); echo $this->encodeOverseerMessage($type, $data); $this->beginStdoutCapture(); } public static function errorListener($event, $value, array $metadata) { // If the caller has redirected the error log to a file, PHP won't output // messages to stderr, so the overseer can't capture them. Install a // listener which just echoes errors to stderr, so the overseer is always // aware of errors. $console = PhutilConsole::getConsole(); $message = idx($metadata, 'default_message'); if ($message) { $console->writeErr("%s\n", $message); } if (idx($metadata, 'trace')) { $trace = PhutilErrorHandler::formatStacktrace($metadata['trace']); $console->writeErr("%s\n", $trace); } } /* -( Autoscaling )-------------------------------------------------------- */ /** * Prepare to become busy. This may autoscale the pool up. * * This notifies the overseer that the daemon has become busy. If daemons * that are part of an autoscale pool are continuously busy for a prolonged * period of time, the overseer may scale up the pool. * * @return this * @task autoscale */ protected function willBeginWork() { if ($this->workState != self::WORKSTATE_BUSY) { $this->workState = self::WORKSTATE_BUSY; $this->idleSince = null; $this->emitOverseerMessage(self::MESSAGETYPE_BUSY, null); } return $this; } /** * Prepare to idle. This may autoscale the pool down. * * This notifies the overseer that the daemon is no longer busy. If daemons * that are part of an autoscale pool are idle for a prolonged period of time, * they may exit to scale the pool down. * * @return this * @task autoscale */ protected function willBeginIdle() { if ($this->workState != self::WORKSTATE_IDLE) { $this->workState = self::WORKSTATE_IDLE; $this->idleSince = time(); $this->emitOverseerMessage(self::MESSAGETYPE_IDLE, null); } return $this; } /** * Determine if this is a clone or the original daemon. * * @return bool True if this is an cloned autoscaling daemon. * @task autoscale */ private function isClonedAutoscaleDaemon() { return (bool)$this->getAutoscaleProperty('clone', false); } /** * Get the duration (in seconds) which a daemon must be continuously idle * for before it should exit to scale the pool down. * * @return int Duration, in seconds. * @task autoscale */ private function getAutoscaleDownDuration() { return $this->getAutoscaleProperty('down', 15); } /** * Configure autoscaling for this daemon. * * @param map Map of autoscale properties. * @return this * @task autoscale */ public function setAutoscaleProperties(array $autoscale_properties) { PhutilTypeSpec::checkMap( $autoscale_properties, array( 'group' => 'optional string', 'up' => 'optional int', 'down' => 'optional int', 'pool' => 'optional int', 'clone' => 'optional bool', 'reserve' => 'optional int|float', )); $this->autoscaleProperties = $autoscale_properties; return $this; } /** * Read autoscaling configuration for this daemon. * * @param string Property to read. * @param wild Default value to return if the property is not set. * @return wild Property value, or `$default` if one is not set. * @task autoscale */ private function getAutoscaleProperty($key, $default = null) { return idx($this->autoscaleProperties, $key, $default); } } diff --git a/src/events/PhutilEventListener.php b/src/events/PhutilEventListener.php index 1392a08..0be9f1a 100644 --- a/src/events/PhutilEventListener.php +++ b/src/events/PhutilEventListener.php @@ -1,40 +1,37 @@ } abstract public function register(); abstract public function handleEvent(PhutilEvent $event); final public function listen($type) { $engine = PhutilEventEngine::getInstance(); $engine->addListener($this, $type); } /** * Return a scalar ID unique to this listener. This is used to deduplicate * listeners which match events on multiple rules, so they are invoked only * once. * * @return int A scalar unique to this object instance. */ final public function getListenerID() { if (!$this->listenerID) { $this->listenerID = self::$nextListenerID; self::$nextListenerID++; } return $this->listenerID; } } diff --git a/src/future/Future.php b/src/future/Future.php index d5fbf73..d3e7fbf 100644 --- a/src/future/Future.php +++ b/src/future/Future.php @@ -1,191 +1,189 @@ getDefaultWait(); do { $this->checkException(); if ($this->isReady()) { break; } $read = $this->getReadSockets(); $write = $this->getWriteSockets(); if ($timeout !== null) { $elapsed = microtime(true) - $start; if ($elapsed > $timeout) { $this->checkException(); return null; } else { $wait = $timeout - $elapsed; } } if ($read || $write) { self::waitForSockets($read, $write, $wait); } } while (true); $this->checkException(); return $this->getResult(); } public function setException(Exception $ex) { $this->exception = $ex; return $this; } public function getException() { return $this->exception; } /** * If an exception was set by setException(), throw it. */ private function checkException() { if ($this->exception) { throw $this->exception; } } /** * Retrieve a list of sockets which we can wait to become readable while * a future is resolving. If your future has sockets which can be * `select()`ed, return them here (or in @{method:getWriteSockets}) to make * the resolve loop do a `select()`. If you do not return sockets in either * case, you'll get a busy wait. * * @return list A list of sockets which we expect to become readable. */ public function getReadSockets() { return array(); } /** * Retrieve a list of sockets which we can wait to become writable while a * future is resolving. See @{method:getReadSockets}. * * @return list A list of sockets which we expect to become writable. */ public function getWriteSockets() { return array(); } /** * Wait for activity on one of several sockets. * * @param list List of sockets expected to become readable. * @param list List of sockets expected to become writable. * @param float Timeout, in seconds. * @return void */ public static function waitForSockets( array $read_list, array $write_list, $timeout = 1) { if (!self::$handlerInstalled) { // If we're spawning child processes, we need to install a signal handler // here to catch cases like execing '(sleep 60 &) &' where the child // exits but a socket is kept open. But we don't actually need to do // anything because the SIGCHLD will interrupt the stream_select(), as // long as we have a handler registered. if (function_exists('pcntl_signal')) { if (!pcntl_signal(SIGCHLD, array(__CLASS__, 'handleSIGCHLD'))) { throw new Exception(pht('Failed to install signal handler!')); } } self::$handlerInstalled = true; } $timeout_sec = (int)$timeout; $timeout_usec = (int)(1000000 * ($timeout - $timeout_sec)); $exceptfds = array(); $ok = @stream_select( $read_list, $write_list, $exceptfds, $timeout_sec, $timeout_usec); if ($ok === false) { // Hopefully, means we received a SIGCHLD. In the worst case, we degrade // to a busy wait. } } public static function handleSIGCHLD($signo) { // This function is a dummy, we just need to have some handler registered // so that PHP will get interrupted during stream_select(). If we don't // register a handler, stream_select() won't fail. } /** * Retrieve the final result of the future. This method will be called after * the future is ready (as per @{method:isReady}) but before results are * passed back to the caller. The major use of this function is that you can * override it in subclasses to do postprocessing or error checking, which is * particularly useful if building application-specific futures on top of * primitive transport futures (like @{class:CurlFuture} and * @{class:ExecFuture}) which can make it tricky to hook this logic into the * main pipeline. * * @return mixed Final resolution of this future. */ protected function getResult() { return $this->result; } /** * Default amount of time to wait on stream select for this future. Normally * 1 second is fine, but if the future has a timeout sooner than that it * should return the amount of time left before the timeout. */ public function getDefaultWait() { return 1; } public function start() { $this->isReady(); return $this; } } diff --git a/src/future/FutureProxy.php b/src/future/FutureProxy.php index 3af12ae..0cb6fda 100644 --- a/src/future/FutureProxy.php +++ b/src/future/FutureProxy.php @@ -1,73 +1,71 @@ setProxiedFuture($proxied); } } public function setProxiedFuture(Future $proxied) { $this->proxied = $proxied; return $this; } protected function getProxiedFuture() { if (!$this->proxied) { throw new Exception(pht('The proxied future has not been provided yet.')); } return $this->proxied; } public function isReady() { return $this->getProxiedFuture()->isReady(); } public function resolve($timeout = null) { $this->getProxiedFuture()->resolve($timeout); return $this->getResult(); } public function setException(Exception $ex) { $this->getProxiedFuture()->setException($ex); return $this; } public function getException() { return $this->getProxiedFuture()->getException(); } public function getReadSockets() { return $this->getProxiedFuture()->getReadSockets(); } public function getWriteSockets() { return $this->getProxiedFuture()->getWriteSockets(); } protected function getResult() { if ($this->result === null) { $result = $this->getProxiedFuture()->resolve(); $result = $this->didReceiveResult($result); $this->result = $result; } return $this->result; } public function start() { $this->getProxiedFuture()->start(); return $this; } abstract protected function didReceiveResult($result); } diff --git a/src/markup/PhutilMarkupEngine.php b/src/markup/PhutilMarkupEngine.php index 7b08b43..65effa2 100644 --- a/src/markup/PhutilMarkupEngine.php +++ b/src/markup/PhutilMarkupEngine.php @@ -1,35 +1,32 @@ engine = $engine; return $this; } final public function getEngine() { return $this->engine; } /** * @return string */ abstract public function getInterpreterName(); abstract public function markupContent($content, array $argv); protected function markupError($string) { if ($this->getEngine()->isTextMode()) { return '('.$string.')'; } else { return phutil_tag( 'div', array( 'class' => 'remarkup-interpreter-error', ), $string); } } } diff --git a/src/markup/engine/remarkup/blockrule/PhutilRemarkupBlockRule.php b/src/markup/engine/remarkup/blockrule/PhutilRemarkupBlockRule.php index 077826c..4e44b99 100644 --- a/src/markup/engine/remarkup/blockrule/PhutilRemarkupBlockRule.php +++ b/src/markup/engine/remarkup/blockrule/PhutilRemarkupBlockRule.php @@ -1,166 +1,163 @@ engine = $engine; $this->updateRules(); return $this; } final protected function getEngine() { return $this->engine; } public function setMarkupRules(array $rules) { assert_instances_of($rules, 'PhutilRemarkupRule'); $this->rules = $rules; $this->updateRules(); return $this; } private function updateRules() { $engine = $this->getEngine(); if ($engine) { $this->rules = msort($this->rules, 'getPriority'); foreach ($this->rules as $rule) { $rule->setEngine($engine); } } return $this; } final public function getMarkupRules() { return $this->rules; } final public function postprocess() { $this->didMarkupText(); } final protected function applyRules($text) { foreach ($this->getMarkupRules() as $rule) { $text = $rule->apply($text); } return $text; } public function supportsChildBlocks() { return false; } public function extractChildText($text) { throw new PhutilMethodNotImplementedException(); } protected function renderRemarkupTable(array $out_rows) { assert_instances_of($out_rows, 'array'); if ($this->getEngine()->isTextMode()) { $lengths = array(); foreach ($out_rows as $r => $row) { foreach ($row['content'] as $c => $cell) { $text = $this->getEngine()->restoreText($cell['content']); $lengths[$c][$r] = phutil_utf8_strlen($text); } } $max_lengths = array_map('max', $lengths); $out = array(); foreach ($out_rows as $r => $row) { $headings = false; foreach ($row['content'] as $c => $cell) { $length = $max_lengths[$c] - $lengths[$c][$r]; $out[] = '| '.$cell['content'].str_repeat(' ', $length).' '; if ($cell['type'] == 'th') { $headings = true; } } $out[] = "|\n"; if ($headings) { foreach ($row['content'] as $c => $cell) { $char = ($cell['type'] == 'th' ? '-' : ' '); $out[] = '| '.str_repeat($char, $max_lengths[$c]).' '; } $out[] = "|\n"; } } return rtrim(implode('', $out), "\n"); } if ($this->getEngine()->isHTMLMailMode()) { $table_attributes = array( 'style' => 'border-collapse: separate; border-spacing: 1px; background: #d3d3d3; margin: 12px 0;', ); $cell_attributes = array ( 'style' => 'background: #ffffff; padding: 3px 6px;', ); } else { $table_attributes = array( 'class' => 'remarkup-table', ); $cell_attributes = array(); } $out = array(); $out[] = "\n"; foreach ($out_rows as $row) { $cells = array(); foreach ($row['content'] as $cell) { $cells[] = phutil_tag( $cell['type'], $cell_attributes, $cell['content']); } $out[] = phutil_tag($row['type'], array(), $cells); $out[] = "\n"; } return phutil_tag('table', $table_attributes, $out); } } diff --git a/src/markup/engine/remarkup/markuprule/PhutilRemarkupRule.php b/src/markup/engine/remarkup/markuprule/PhutilRemarkupRule.php index 292bfe6..29c1d45 100644 --- a/src/markup/engine/remarkup/markuprule/PhutilRemarkupRule.php +++ b/src/markup/engine/remarkup/markuprule/PhutilRemarkupRule.php @@ -1,112 +1,109 @@ engine = $engine; return $this; } public function getEngine() { return $this->engine; } public function getPriority() { return 500.0; } abstract public function apply($text); public function getPostprocessKey() { return spl_object_hash($this); } public function didMarkupText() { return; } protected function replaceHTML($pattern, $callback, $text) { $this->replaceCallback = $callback; return phutil_safe_html(preg_replace_callback( $pattern, array($this, 'replaceHTMLCallback'), phutil_escape_html($text))); } private function replaceHTMLCallback(array $match) { return phutil_escape_html(call_user_func( $this->replaceCallback, array_map('phutil_safe_html', $match))); } /** * Safely generate a tag. * * In Remarkup contexts, it's not safe to use arbitrary text in tag * attributes: even though it will be escaped, it may contain replacement * tokens which are then replaced with markup. * * This method acts as @{function:phutil_tag}, but checks attributes before * using them. * * @param string Tag name. * @param dict Tag attributes. * @param wild Tag content. * @return PhutilSafeHTML Tag object. */ protected function newTag($name, array $attrs, $content = null) { foreach ($attrs as $key => $attr) { if ($attr !== null) { $attrs[$key] = $this->assertFlatText($attr); } } return phutil_tag($name, $attrs, $content); } /** * Assert that a text token is flat (it contains no replacement tokens). * * Because tokens can be replaced with markup, it is dangerous to use * arbitrary input text in tag attributes. Normally, rule precedence should * prevent this. Asserting that text is flat before using it as an attribute * provides an extra layer of security. * * Normally, you can call @{method:newTag} rather than calling this method * directly. @{method:newTag} will check attributes for you. * * @param wild Ostensibly flat text. * @return string Flat text. */ protected function assertFlatText($text) { $text = (string)hsprintf('%s', phutil_safe_html($text)); $rich = (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) !== false); if ($rich) { throw new Exception( pht( 'Remarkup rule precedence is dangerous: rendering text with tokens '. 'as flat text!')); } return $text; } /** * Check whether text is flat (contains no replacement tokens) or not. * * @param wild Ostensibly flat text. * @return bool True if the text is flat. */ protected function isFlatText($text) { $text = (string)hsprintf('%s', phutil_safe_html($text)); return (strpos($text, PhutilRemarkupBlockStorage::MAGIC_BYTE) === false); } } diff --git a/src/parser/argument/workflow/PhutilArgumentWorkflow.php b/src/parser/argument/workflow/PhutilArgumentWorkflow.php index 25ef2bf..2c6cb73 100644 --- a/src/parser/argument/workflow/PhutilArgumentWorkflow.php +++ b/src/parser/argument/workflow/PhutilArgumentWorkflow.php @@ -1,182 +1,181 @@ setTagline('simple calculator example'); * $args->setSynopsis(<<setName('add') * ->setExamples('**add** __n__ ...') * ->setSynopsis('Compute the sum of a list of numbers.') * ->setArguments( * array( * array( * 'name' => 'numbers', * 'wildcard' => true, * ), * )); * * $mul_workflow = id(new PhutilArgumentWorkflow()) * ->setName('mul') * ->setExamples('**mul** __n__ ...') * ->setSynopsis('Compute the product of a list of numbers.') * ->setArguments( * array( * array( * 'name' => 'numbers', * 'wildcard' => true, * ), * )); * * $flow = $args->parseWorkflows( * array( * $add_workflow, * $mul_workflow, * new PhutilHelpArgumentWorkflow(), * )); * * $nums = $args->getArg('numbers'); * if (empty($nums)) { * echo "You must provide one or more numbers!\n"; * exit(1); * } * * foreach ($nums as $num) { * if (!is_numeric($num)) { * echo "Number '{$num}' is not numeric!\n"; * exit(1); * } * } * * switch ($flow->getName()) { * case 'add': * echo array_sum($nums)."\n"; * break; * case 'mul': * echo array_product($nums)."\n"; * break; * } * * You can also subclass this class and return `true` from * @{method:isExecutable}. In this case, the parser will automatically select * your workflow when the user invokes it. * - * @stable * @concrete-extensible */ class PhutilArgumentWorkflow { private $name; private $synopsis; private $specs = array(); private $examples; private $help; final public function __construct() { $this->didConstruct(); } public function setName($name) { $this->name = $name; return $this; } public function getName() { return $this->name; } /** * Provide brief usage examples of common calling conventions, like: * * $workflow->setExamples("**delete** __file__ [__options__]"); * * This text is shown in both brief and detailed help, and should give the * user a quick reference for common uses. You can separate several common * uses with newlines, but usually should not provide more than 2-3 examples. */ final public function setExamples($examples) { $this->examples = $examples; return $this; } final public function getExamples() { if (!$this->examples) { return '**'.$this->name.'**'; } return $this->examples; } /** * Provide a brief description of the command, like "Delete a file.". * * This text is shown in both brief and detailed help, and should give the * user a general idea of what the workflow does. */ final public function setSynopsis($synopsis) { $this->synopsis = $synopsis; return $this; } final public function getSynopsis() { return $this->synopsis; } /** * Provide a full explanation of the command. This text is shown only in * detailed help. */ final public function getHelp() { return $this->help; } final public function setHelp($help) { $this->help = $help; return $this; } final public function setArguments(array $specs) { $specs = PhutilArgumentSpecification::newSpecsFromList($specs); $this->specs = $specs; return $this; } final public function getArguments() { return $this->specs; } protected function didConstruct() { return null; } public function isExecutable() { return false; } public function execute(PhutilArgumentParser $args) { throw new Exception(pht("This workflow isn't executable!")); } /** * Normally, workflow arguments are parsed fully, so unexpected arguments will * raise an error. You can return `true` from this method to parse workflow * arguments only partially. This will allow you to manually parse remaining * arguments or delegate to a second level of workflows. * * @return bool True to partially parse workflow arguments (default false). */ public function shouldParsePartial() { return false; } }