diff --git a/src/conduit/ConduitClient.php b/src/conduit/ConduitClient.php index 0c729a5..81d6b91 100644 --- a/src/conduit/ConduitClient.php +++ b/src/conduit/ConduitClient.php @@ -1,353 +1,353 @@ connectionID; } public function __construct($uri) { $this->uri = new PhutilURI($uri); if (!strlen($this->uri->getDomain())) { throw new Exception( pht("Conduit URI '%s' must include a valid host.", $uri)); } $this->host = $this->uri->getDomain(); } /** * Override the domain specified in the service URI and provide a specific * host identity. * * This can be used to connect to a specific node in a cluster environment. */ public function setHost($host) { $this->host = $host; return $this; } public function getHost() { return $this->host; } public function setConduitToken($conduit_token) { $this->conduitToken = $conduit_token; return $this; } public function getConduitToken() { return $this->conduitToken; } public function callMethodSynchronous($method, array $params) { return $this->callMethod($method, $params)->resolve(); } public function didReceiveResponse($method, $data) { if ($method == 'conduit.connect') { $this->sessionKey = idx($data, 'sessionKey'); $this->connectionID = idx($data, 'connectionID'); } return $data; } public function setTimeout($timeout) { $this->timeout = $timeout; return $this; } public function setSigningKeys( $public_key, PhutilOpaqueEnvelope $private_key) { $this->publicKey = $public_key; $this->privateKey = $private_key; return $this; } public function callMethod($method, array $params) { $meta = array(); if ($this->sessionKey) { $meta['sessionKey'] = $this->sessionKey; } if ($this->connectionID) { $meta['connectionID'] = $this->connectionID; } if ($method == 'conduit.connect') { $certificate = idx($params, 'certificate'); if ($certificate) { $token = time(); $params['authToken'] = $token; $params['authSignature'] = sha1($token.$certificate); } unset($params['certificate']); } if ($this->privateKey && $this->publicKey) { $meta['auth.type'] = self::AUTH_ASYMMETRIC; $meta['auth.key'] = $this->publicKey; $meta['auth.host'] = $this->getHostString(); $signature = $this->signRequest($method, $params, $meta); $meta['auth.signature'] = $signature; } if ($this->conduitToken) { $meta['token'] = $this->conduitToken; } if ($meta) { $params['__conduit__'] = $meta; } $uri = id(clone $this->uri)->setPath('/api/'.$method); $data = array( 'params' => json_encode($params), 'output' => 'json', // This is a hint to Phabricator that the client expects a Conduit // response. It is not necessary, but provides better error messages in // some cases. '__conduit__' => true, ); // Always use the cURL-based HTTPSFuture, for proxy support and other // protocol edge cases that HTTPFuture does not support. $core_future = new HTTPSFuture($uri, $data); $core_future->addHeader('Host', $this->getHost()); $core_future->setMethod('POST'); $core_future->setTimeout($this->timeout); if ($this->username !== null) { $core_future->setHTTPBasicAuthCredentials( $this->username, $this->password); } $conduit_future = new ConduitFuture($core_future); $conduit_future->setClient($this, $method); $conduit_future->beginProfile($data); $conduit_future->isReady(); return $conduit_future; } public function setBasicAuthCredentials($username, $password) { $this->username = $username; $this->password = new PhutilOpaqueEnvelope($password); return $this; } private function getHostString() { $host = $this->getHost(); $uri = new PhutilURI($this->uri); $port = $uri->getPort(); if (!$port) { switch ($uri->getProtocol()) { case 'https': $port = 443; break; default: $port = 80; break; } } return $host.':'.$port; } private function signRequest( $method, array $params, array $meta) { $input = self::encodeRequestDataForSignature( $method, $params, $meta); $signature = null; $result = openssl_sign( $input, $signature, $this->privateKey->openEnvelope()); if (!$result) { throw new Exception( pht('Unable to sign Conduit request with signing key.')); } return self::SIGNATURE_CONSIGN_1.base64_encode($signature); } public static function verifySignature( $method, array $params, array $meta, $openssl_public_key) { $auth_type = idx($meta, 'auth.type'); switch ($auth_type) { case self::AUTH_ASYMMETRIC: break; default: throw new Exception( pht( 'Unable to verify request signature, specified "%s" '. '("%s") is unknown.', 'auth.type', $auth_type)); } $public_key = idx($meta, 'auth.key'); if (!strlen($public_key)) { throw new Exception( pht( 'Unable to verify request signature, no "%s" present in '. 'request protocol information.', 'auth.key')); } $signature = idx($meta, 'auth.signature'); if (!strlen($signature)) { throw new Exception( pht( 'Unable to verify request signature, no "%s" present '. 'in request protocol information.', 'auth.signature')); } $prefix = self::SIGNATURE_CONSIGN_1; if (strncmp($signature, $prefix, strlen($prefix)) !== 0) { throw new Exception( pht( 'Unable to verify request signature, signature format is not '. 'known.')); } $signature = substr($signature, strlen($prefix)); $input = self::encodeRequestDataForSignature( $method, $params, $meta); $signature = base64_decode($signature); $trap = new PhutilErrorTrap(); $result = @openssl_verify( $input, $signature, $openssl_public_key); $err = $trap->getErrorsAsString(); $trap->destroy(); if ($result === 1) { // Signature is good. return true; } else if ($result === 0) { // Signature is bad. throw new Exception( pht( 'Request signature verification failed: signature is not correct.')); } else { // Some kind of error. if (strlen($err)) { throw new Exception( pht( 'OpenSSL encountered an error verifying the request signature: %s', $err)); } else { throw new Exception( pht( 'OpenSSL encountered an unknown error verifying the request: %s', $err)); } } } private static function encodeRequestDataForSignature( $method, array $params, array $meta) { unset($meta['auth.signature']); $structure = array( 'method' => $method, 'protocol' => $meta, 'parameters' => $params, ); return self::encodeRawDataForSignature($structure); } public static function encodeRawDataForSignature($data) { $out = array(); if (is_array($data)) { if (!$data || (array_keys($data) == range(0, count($data) - 1))) { $out[] = 'A'; $out[] = count($data); $out[] = ':'; foreach ($data as $value) { $out[] = self::encodeRawDataForSignature($value); } } else { ksort($data); $out[] = 'O'; $out[] = count($data); $out[] = ':'; foreach ($data as $key => $value) { $out[] = self::encodeRawDataForSignature($key); $out[] = self::encodeRawDataForSignature($value); } } } else if (is_string($data)) { $out[] = 'S'; $out[] = strlen($data); $out[] = ':'; $out[] = $data; - } else if (is_integer($data)) { + } else if (is_int($data)) { $out[] = 'I'; $out[] = strlen((string)$data); $out[] = ':'; $out[] = (string)$data; } else if (is_null($data)) { $out[] = 'N'; $out[] = ':'; } else if ($data === true) { $out[] = 'B1:'; } else if ($data === false) { $out[] = 'B0:'; } else { throw new Exception( pht( 'Unexpected data type in request data: %s.', gettype($data))); } return implode('', $out); } } diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php index 5b276aa..824ab90 100644 --- a/src/daemon/PhutilDaemonOverseer.php +++ b/src/daemon/PhutilDaemonOverseer.php @@ -1,485 +1,485 @@ enableDiscardMode(); $args = new PhutilArgumentParser($argv); $args->setTagline(pht('daemon overseer')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'trace-memory', 'help' => pht('Enable debug memory tracing.'), ), array( 'name' => 'verbose', 'help' => pht('Enable verbose activity logging.'), ), array( 'name' => 'label', 'short' => 'l', 'param' => 'label', 'help' => pht( 'Optional process label. Makes "%s" nicer, no behavioral effects.', 'ps'), ), )); $argv = array(); if ($args->getArg('trace')) { $this->traceMode = true; $argv[] = '--trace'; } if ($args->getArg('trace-memory')) { $this->traceMode = true; $this->traceMemory = true; $argv[] = '--trace-memory'; } $verbose = $args->getArg('verbose'); if ($verbose) { $this->verbose = true; $argv[] = '--verbose'; } $label = $args->getArg('label'); if ($label) { $argv[] = '-l'; $argv[] = $label; } $this->argv = $argv; if (function_exists('posix_isatty') && posix_isatty(STDIN)) { fprintf(STDERR, pht('Reading daemon configuration from stdin...')."\n"); } $config = @file_get_contents('php://stdin'); $config = id(new PhutilJSONParser())->parse($config); $this->libraries = idx($config, 'load'); $this->log = idx($config, 'log'); $this->daemonize = idx($config, 'daemonize'); $this->piddir = idx($config, 'piddir'); $this->config = $config; if (self::$instance) { throw new Exception( pht('You may not instantiate more than one Overseer per process.')); } self::$instance = $this; $this->startEpoch = time(); // Check this before we daemonize, since if it's an issue the child will // exit immediately. if ($this->piddir) { $dir = $this->piddir; try { Filesystem::assertWritable($dir); } catch (Exception $ex) { throw new Exception( pht( "Specified daemon PID directory ('%s') does not exist or is ". "not writable by the daemon user!", $dir)); } } if (!idx($config, 'daemons')) { throw new PhutilArgumentUsageException( pht('You must specify at least one daemon to start!')); } if ($this->log) { // NOTE: Now that we're committed to daemonizing, redirect the error // log if we have a `--log` parameter. Do this at the last moment // so as many setup issues as possible are surfaced. ini_set('error_log', $this->log); } if ($this->daemonize) { // We need to get rid of these or the daemon will hang when we TERM it // waiting for something to read the buffers. TODO: Learn how unix works. fclose(STDOUT); fclose(STDERR); ob_start(); $pid = pcntl_fork(); if ($pid === -1) { throw new Exception(pht('Unable to fork!')); } else if ($pid) { exit(0); } } declare(ticks = 1); pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal')); pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal')); pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal')); pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); } public function addLibrary($library) { $this->libraries[] = $library; return $this; } public function run() { $this->daemons = array(); foreach ($this->config['daemons'] as $config) { $config += array( 'argv' => array(), 'autoscale' => array(), ); $daemon = new PhutilDaemonHandle( $this, $config['class'], $this->argv, array( 'log' => $this->log, 'argv' => $config['argv'], 'load' => $this->libraries, 'autoscale' => $config['autoscale'], )); $daemon->setSilent((!$this->traceMode && !$this->verbose)); $daemon->setTraceMemory($this->traceMemory); $this->addDaemon($daemon, $config); } while (true) { $futures = array(); foreach ($this->getDaemonHandles() as $daemon) { $daemon->update(); if ($daemon->isRunning()) { $futures[] = $daemon->getFuture(); } if ($daemon->isDone()) { $this->removeDaemon($daemon); } } $this->updatePidfile(); $this->updateAutoscale(); if ($futures) { $iter = id(new FutureIterator($futures)) ->setUpdateInterval(1); foreach ($iter as $future) { break; } } else { if ($this->inGracefulShutdown) { break; } sleep(1); } } exit($this->err); } private function addDaemon(PhutilDaemonHandle $daemon, array $config) { $id = $daemon->getDaemonID(); $this->daemons[$id] = array( 'handle' => $daemon, 'config' => $config, ); $autoscale_group = $this->getAutoscaleGroup($daemon); if ($autoscale_group) { $this->autoscale[$autoscale_group][$id] = true; } return $this; } private function removeDaemon(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); $autoscale_group = $this->getAutoscaleGroup($daemon); if ($autoscale_group) { unset($this->autoscale[$autoscale_group][$id]); } unset($this->daemons[$id]); return $this; } private function getAutoscaleGroup(PhutilDaemonHandle $daemon) { return $this->getAutoscaleProperty($daemon, 'group'); } private function getAutoscaleProperty( PhutilDaemonHandle $daemon, $key, $default = null) { $id = $daemon->getDaemonID(); $autoscale = $this->daemons[$id]['config']['autoscale']; return idx($autoscale, $key, $default); } public function didBeginWork(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); $busy = idx($this->daemons[$daemon->getDaemonID()], 'busy'); if (!$busy) { $this->daemons[$id]['busy'] = time(); } } public function didBeginIdle(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); unset($this->daemons[$id]['busy']); } public function updateAutoscale() { foreach ($this->autoscale as $group => $daemons) { $daemon = $this->daemons[head_key($daemons)]['handle']; $scaleup_duration = $this->getAutoscaleProperty($daemon, 'up', 2); $max_pool_size = $this->getAutoscaleProperty($daemon, 'pool', 8); $reserve = $this->getAutoscaleProperty($daemon, 'reserve', 0); // Don't scale a group if it is already at the maximum pool size. if (count($daemons) >= $max_pool_size) { continue; } $should_scale = true; foreach ($daemons as $daemon_id => $ignored) { $busy = idx($this->daemons[$daemon_id], 'busy'); if (!$busy) { // At least one daemon in the group hasn't reported that it has // started work. $should_scale = false; break; } if ((time() - $busy) < $scaleup_duration) { // At least one daemon in the group was idle recently, so we have // not fullly $should_scale = false; break; } } // If we have a configured memory reserve for this pool, it tells us that // we should not scale up unless there's at least that much memory left // on the system (for example, a reserve of 0.25 means that 25% of system // memory must be free to autoscale). if ($should_scale && $reserve) { // On some systems this may be slightly more expensive than other // checks, so only do it once we're prepared to scale up. $memory = PhutilSystem::getSystemMemoryInformation(); $free_ratio = ($memory['free'] / $memory['total']); // If we don't have enough free memory, don't scale. if ($free_ratio <= $reserve) { continue; } } if ($should_scale) { $config = $this->daemons[$daemon_id]['config']; $config['autoscale']['clone'] = true; $clone = new PhutilDaemonHandle( $this, $config['class'], $this->argv, array( 'log' => $this->log, 'argv' => $config['argv'], 'load' => $this->libraries, 'autoscale' => $config['autoscale'], )); $this->addDaemon($clone, $config); // Don't scale more than one pool up per iteration. Otherwise, we could // break the memory barrier if we have a lot of pools and scale them // all up at once. return; } } } public function didReceiveNotifySignal($signo) { foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveNotifySignal($signo); } } public function didReceiveReloadSignal($signo) { foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveReloadSignal($signo); } } public function didReceiveGracefulSignal($signo) { // If we receive SIGINT more than once, interpret it like SIGTERM. if ($this->inGracefulShutdown) { return $this->didReceiveTerminalSignal($signo); } $this->inGracefulShutdown = true; foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveGracefulSignal($signo); } } public function didReceiveTerminalSignal($signo) { $this->err = 128 + $signo; if ($this->inAbruptShutdown) { exit($this->err); } $this->inAbruptShutdown = true; foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveTerminalSignal($signo); } } private function getDaemonHandles() { return ipull($this->daemons, 'handle'); } /** * Identify running daemons by examining the process table. This isn't * completely reliable, but can be used as a fallback if the pid files fail * or we end up with stray daemons by other means. * * Example output (array keys are process IDs): * * array( * 12345 => array( * 'type' => 'overseer', * 'command' => 'php launch_daemon.php --daemonize ...', * 'pid' => 12345, * ), * 12346 => array( * 'type' => 'daemon', * 'command' => 'php exec_daemon.php ...', * 'pid' => 12346, * ), * ); * * @return dict Map of PIDs to process information, identifying running * daemon processes. */ public static function findRunningDaemons() { $results = array(); list($err, $processes) = exec_manual('ps -o pid,command -a -x -w -w -w'); if ($err) { return $results; } $processes = array_filter(explode("\n", trim($processes))); foreach ($processes as $process) { list($pid, $command) = preg_split('/\s+/', trim($process), 2); $pattern = '/((launch|exec)_daemon.php|phd-daemon)/'; $matches = null; if (!preg_match($pattern, $command, $matches)) { continue; } switch ($matches[1]) { case 'exec_daemon.php': $type = 'daemon'; break; case 'launch_daemon.php': case 'phd-daemon': default: $type = 'overseer'; break; } $results[(int)$pid] = array( 'type' => $type, 'command' => $command, - 'pid' => (int) $pid, + 'pid' => (int)$pid, ); } return $results; } private function updatePidfile() { if (!$this->piddir) { return; } $daemons = array(); foreach ($this->daemons as $daemon) { $handle = $daemon['handle']; $config = $daemon['config']; if (!$handle->isRunning()) { continue; } $daemons[] = array( 'pid' => $handle->getPID(), 'id' => $handle->getDaemonID(), 'config' => $config, ); } $pidfile = array( 'pid' => getmypid(), 'start' => $this->startEpoch, 'config' => $this->config, 'daemons' => $daemons, ); if ($pidfile !== $this->lastPidfile) { $this->lastPidfile = $pidfile; $pidfile_path = $this->piddir.'/daemon.'.getmypid(); Filesystem::writeFile($pidfile_path, json_encode($pidfile)); } } } diff --git a/src/internationalization/PhutilTranslator.php b/src/internationalization/PhutilTranslator.php index 35cffad..d854733 100644 --- a/src/internationalization/PhutilTranslator.php +++ b/src/internationalization/PhutilTranslator.php @@ -1,231 +1,231 @@ locale = $locale; $this->localeCode = $locale->getLocaleCode(); $this->shouldPostProcess = $locale->shouldPostProcessTranslations(); return $this; } /** * Add translations which will be later used by @{method:translate}. * The parameter is an array of strings (for simple translations) or arrays * (for translastions with variants). The number of items in the array is * language specific. It is `array($singular, $plural)` for English. * * array( * 'color' => 'colour', * '%d beer(s)' => array('%d beer', '%d beers'), * ); * * The arrays can be nested for strings with more variant parts: * * array( * '%d char(s) on %d row(s)' => array( * array('%d char on %d row', '%d char on %d rows'), * array('%d chars on %d row', '%d chars on %d rows'), * ), * ); * * The translation should have the same placeholders as originals. Swapping * parameter order is possible: * * array( * '%s owns %s.' => '%2$s is owned by %1$s.', * ); * * @param array Identifier in key, translation in value. * @return PhutilTranslator Provides fluent interface. */ public function setTranslations(array $translations) { $this->translations = $translations; return $this; } public function translate($text /* , ... */) { $translation = idx($this->translations, $text, $text); $args = func_get_args(); while (is_array($translation)) { $translation = $this->chooseVariant($translation, next($args)); } array_shift($args); foreach ($args as $k => $arg) { if ($arg instanceof PhutilNumber) { $args[$k] = $this->formatNumber($arg->getNumber(), $arg->getDecimals()); } } // Check if any arguments are PhutilSafeHTML. If they are, we will apply // any escaping necessary and output HTML. $is_html = false; foreach ($args as $arg) { if ($arg instanceof PhutilSafeHTML) { $is_html = true; break; } } if ($is_html) { foreach ($args as $k => $arg) { $args[$k] = (string)phutil_escape_html($arg); } } $result = vsprintf($translation, $args); if ($result === false) { // If vsprintf() fails (often because the translated string references // too many parameters), show the bad template with a note instead of // returning an empty string. This makes it easier to figure out what // went wrong and fix it. $result = pht('[Invalid Translation!] %s', $translation); } if ($this->shouldPostProcess) { $result = $this->locale->didTranslateString( $text, $translation, $args, $result); } if ($is_html) { $result = phutil_safe_html($result); } return $result; } private function chooseVariant(array $translations, $variant) { if (count($translations) == 1) { // If we only have one variant, we can select it directly. return reset($translations); } if ($variant instanceof PhutilNumber) { $variant = $variant->getNumber(); } // TODO: Move these into PhutilLocale if benchmarks show we aren't // eating too much of a performance cost. switch ($this->localeCode) { case 'en_US': case 'en_GB': case 'en_W*': case 'en_R*': case 'en_A*': list($singular, $plural) = $translations; if ($variant == 1) { return $singular; } return $plural; case 'cs_CZ': if ($variant instanceof PhutilPerson) { list($male, $female) = $translations; if ($variant->getSex() == PhutilPerson::SEX_FEMALE) { return $female; } return $male; } list($singular, $paucal, $plural) = $translations; if ($variant == 1) { return $singular; } if ($variant >= 2 && $variant <= 4) { return $paucal; } return $plural; default: throw new Exception(pht("Unknown language '%s'.", $this->language)); } } /** * Translate date formatted by `$date->format()`. * * @param string Format accepted by `DateTime::format()`. * @param DateTime * @return string Formatted and translated date. */ public function translateDate($format, DateTime $date) { static $format_cache = array(); if (!isset($format_cache[$format])) { $translatable = 'DlSFMaA'; preg_match_all( '/['.$translatable.']|(\\\\.|[^'.$translatable.'])+/', $format, $format_cache[$format], PREG_SET_ORDER); } $parts = array(); foreach ($format_cache[$format] as $match) { $part = $date->format($match[0]); if (!isset($match[1])) { $part = $this->translate($part); } $parts[] = $part; } return implode('', $parts); } /** * Format number with grouped thousands and optional decimal part. Requires * translations of '.' (decimal point) and ',' (thousands separator). Both * these translations must be 1 byte long with PHP < 5.4.0. * * @param float * @param int * @return string */ public function formatNumber($number, $decimals = 0) { return number_format( $number, $decimals, $this->translate('.'), $this->translate(',')); } public function validateTranslation($original, $translation) { $pattern = '/<(\S[^>]*>?)?|&(\S[^;]*;?)?/i'; $original_matches = null; $translation_matches = null; preg_match_all($pattern, $original, $original_matches); preg_match_all($pattern, $translation, $translation_matches); sort($original_matches[0]); sort($translation_matches[0]); if ($original_matches[0] !== $translation_matches[0]) { return false; } return true; } } diff --git a/src/parser/__tests__/PhutilTypeSpecTestCase.php b/src/parser/__tests__/PhutilTypeSpecTestCase.php index e4ec484..9c833ae 100644 --- a/src/parser/__tests__/PhutilTypeSpecTestCase.php +++ b/src/parser/__tests__/PhutilTypeSpecTestCase.php @@ -1,291 +1,291 @@ ', 'int | null', 'list < string >', 'int (must be even)', 'optional int', 'int?', 'int|null?', 'optional int? (minimum 300)', 'list', 'list>>> (easy)', ); $bad = array( '', 'list<>', 'list', 'map|map', 'int optional', '(derp)', 'list', 'int?|string', ); $good = array_fill_keys($good, true); $bad = array_fill_keys($bad, false); foreach ($good + $bad as $input => $expect) { $caught = null; try { PhutilTypeSpec::newFromString($input); } catch (Exception $ex) { $caught = $ex; } $this->assertEqual( $expect, ($caught === null), $input); } } public function testTypeSpecStringify() { $types = array( 'int', 'list', 'map', 'list>', 'map>', 'int|null', 'int|string|null', 'list', 'list', 'optional int', 'int (even)', ); foreach ($types as $type) { $this->assertEqual( $type, PhutilTypeSpec::newFromString($type)->toString()); } } public function testCanonicalize() { $tests = array( 'int?' => 'optional int', 'int | null' => 'int|null', 'list < map < int , string > > ?' => 'optional list>', 'int ( x )' => 'int ( x )', ); foreach ($tests as $input => $expect) { $this->assertEqual( $expect, PhutilTypeSpec::newFromString($input)->toString(), $input); } } public function testGetCommonParentClass() { $map = array( 'stdClass' => array( array('stdClass', 'stdClass'), ), false => array( array('Exception', 'stdClass'), ), 'Exception' => array( array('Exception', 'RuntimeException'), array('LogicException', 'RuntimeException'), array('BadMethodCallException', 'OutOfBoundsException'), ), ); foreach ($map as $expect => $tests) { if (is_int($expect)) { - $expect = (bool) $expect; + $expect = (bool)$expect; } foreach ($tests as $input) { list($class_a, $class_b) = $input; $this->assertEqual( $expect, PhutilTypeSpec::getCommonParentClass($class_a, $class_b), print_r($input, true)); } } } public function testGetTypeOf() { $map = array( 'int' => 1, 'string' => 'asdf', 'float' => 1.5, 'bool' => true, 'null' => null, 'map' => array(), 'list' => array('a', 'b'), 'list' => array(1, 2, 3), 'map' => array('x' => 3), 'map>' => array(1 => array('x', 'y')), 'stdClass' => new stdClass(), 'list' => array( new Exception(), new LogicException(), new RuntimeException(), ), 'map' => array('x' => new stdClass()), ); foreach ($map as $expect => $input) { $this->assertEqual( $expect, PhutilTypeSpec::getTypeOf($input), print_r($input, true)); PhutilTypeSpec::newFromString($expect)->check($input); } } public function testTypeCheckFailures() { $map = array( 'int' => 'string', 'string' => 32, 'null' => true, 'bool' => null, 'map' => 16, 'list' => array('y' => 'z'), 'int|null' => 'ducks', 'stdClass' => new Exception(), 'list' => array(new Exception()), ); foreach ($map as $type => $value) { $caught = null; try { PhutilTypeSpec::newFromString($type)->check($value); } catch (PhutilTypeCheckException $ex) { $caught = $ex; } $this->assertTrue($ex instanceof PhutilTypeCheckException); } } public function testCheckMap() { $spec = array( 'count' => 'int', 'color' => 'optional string', ); // Valid PhutilTypeSpec::checkMap( array( 'count' => 1, ), $spec); // Valid, with optional parameter. PhutilTypeSpec::checkMap( array( 'count' => 3, 'color' => 'red', ), $spec); // Parameter "count" is required but missing. $caught = null; try { PhutilTypeSpec::checkMap( array(), $spec); } catch (Exception $ex) { $caught = $ex; } $this->assertTrue($ex instanceof PhutilTypeMissingParametersException); // Parameter "size" is specified but does not exist. $caught = null; try { PhutilTypeSpec::checkMap( array( 'count' => 4, 'size' => 'large', ), $spec); } catch (Exception $ex) { $caught = $ex; } $this->assertTrue($ex instanceof PhutilTypeExtraParametersException); } public function testRegexValidation() { PhutilTypeSpec::checkMap( array( 'regex' => '/.*/', ), array( 'regex' => 'regex', )); $caught = null; try { PhutilTypeSpec::checkMap( array( 'regex' => '.*', ), array( 'regex' => 'regex', )); } catch (PhutilTypeCheckException $ex) { $caught = $ex; } $this->assertTrue($ex instanceof PhutilTypeCheckException); } public function testScalarOrListRegexp() { PhutilTypeSpec::checkMap( array( 'regex' => '/.*/', ), array( 'regex' => 'regex | list', )); PhutilTypeSpec::checkMap( array( 'regex' => array('/.*/'), ), array( 'regex' => 'regex | list', )); PhutilTypeSpec::checkMap( array( 'regex' => '/.*/', ), array( 'regex' => 'list | regex', )); PhutilTypeSpec::checkMap( array( 'regex' => array('/.*/'), ), array( 'regex' => 'list | regex', )); $this->assertTrue(true); } } diff --git a/src/parser/aast/api/AASTNode.php b/src/parser/aast/api/AASTNode.php index 86ce97f..7a6a3ee 100644 --- a/src/parser/aast/api/AASTNode.php +++ b/src/parser/aast/api/AASTNode.php @@ -1,326 +1,326 @@ id = $id; $this->typeID = $data[0]; if (isset($data[1])) { $this->l = $data[1]; } else { $this->l = -1; } if (isset($data[2])) { $this->r = $data[2]; } else { $this->r = -1; } $this->tree = $tree; } - public final function getParentNode() { + final public function getParentNode() { return $this->parentNode; } - public final function getID() { + final public function getID() { return $this->id; } - public final function getTypeID() { + final public function getTypeID() { return $this->typeID; } - public final function getTree() { + final public function getTree() { return $this->tree; } - public final function getTypeName() { + final public function getTypeName() { if (empty($this->typeName)) { $this->typeName = $this->tree->getNodeTypeNameFromTypeID($this->getTypeID()); } return $this->typeName; } - public final function getChildren() { + final public function getChildren() { return $this->children; } public function getChildrenOfType($type) { $nodes = array(); foreach ($this->children as $child) { if ($child->getTypeName() == $type) { $nodes[] = $child; } } return $nodes; } public function getChildOfType($index, $type) { $child = $this->getChildByIndex($index); if ($child->getTypeName() != $type) { throw new Exception( pht( "Child in position '%d' is not of type '%s': %s", $index, $type, $this->getDescription())); } return $child; } public function getChildByIndex($index) { // NOTE: Microoptimization to avoid calls like array_values() or idx(). $idx = 0; foreach ($this->children as $child) { if ($idx == $index) { return $child; } ++$idx; } throw new Exception(pht("No child with index '%d'.", $index)); } /** * Build a cache to improve the performance of * @{method:selectDescendantsOfType}. This cache makes a time/memory tradeoff * by aggressively caching node descendants. It may improve the tree's query * performance substantially if you make a large number of queries, but also * requires a significant amount of memory. * * This builds a cache for the entire tree and improves performance of all * @{method:selectDescendantsOfType} calls. */ public function buildSelectCache() { $cache = array(); foreach ($this->getChildren() as $id => $child) { $type_id = $child->getTypeID(); if (empty($cache[$type_id])) { $cache[$type_id] = array(); } $cache[$type_id][$id] = $child; foreach ($child->buildSelectCache() as $type_id => $nodes) { if (empty($cache[$type_id])) { $cache[$type_id] = array(); } $cache[$type_id] += $nodes; } } $this->selectCache = $cache; return $this->selectCache; } /** * Build a cache to improve the performance of @{method:selectTokensOfType}. * This cache makes a time/memory tradeoff by aggressively caching token * types. It may improve the tree's query performance substantially if you * make a large number of queries, but also requires a significant amount of * memory. * * This builds a cache for this node only. */ public function buildTokenCache() { $cache = array(); foreach ($this->getTokens() as $id => $token) { $cache[$token->getTypeName()][$id] = $token; } $this->tokenCache = $cache; return $this->tokenCache; } public function selectTokensOfType($type_name) { return $this->selectTokensOfTypes(array($type_name)); } /** * Select all tokens of any given types. */ public function selectTokensOfTypes(array $type_names) { $tokens = array(); foreach ($type_names as $type_name) { if (isset($this->tokenCache)) { $cached_tokens = idx($this->tokenCache, $type_name, array()); foreach ($cached_tokens as $id => $cached_token) { $tokens[$id] = $cached_token; } } else { foreach ($this->getTokens() as $id => $token) { if ($token->getTypeName() == $type_name) { $tokens[$id] = $token; } } } } return $tokens; } public function selectDescendantsOfType($type_name) { return $this->selectDescendantsOfTypes(array($type_name)); } public function selectDescendantsOfTypes(array $type_names) { $nodes = array(); foreach ($type_names as $type_name) { $type = $this->getTypeIDFromTypeName($type_name); if (isset($this->selectCache)) { if (isset($this->selectCache[$type])) { $nodes = $nodes + $this->selectCache[$type]; } } else { $nodes = $nodes + $this->executeSelectDescendantsOfType($this, $type); } } return AASTNodeList::newFromTreeAndNodes($this->tree, $nodes); } protected function executeSelectDescendantsOfType($node, $type) { $results = array(); foreach ($node->getChildren() as $id => $child) { if ($child->getTypeID() == $type) { $results[$id] = $child; } $results += $this->executeSelectDescendantsOfType($child, $type); } return $results; } public function getTokens() { if ($this->l == -1 || $this->r == -1) { return array(); } $tokens = $this->tree->getRawTokenStream(); $result = array(); foreach (range($this->l, $this->r) as $token_id) { $result[$token_id] = $tokens[$token_id]; } return $result; } public function getConcreteString() { $values = array(); foreach ($this->getTokens() as $token) { $values[] = $token->getValue(); } return implode('', $values); } public function getSemanticString() { $tokens = $this->getTokens(); foreach ($tokens as $id => $token) { if ($token->isComment()) { unset($tokens[$id]); } } return implode('', mpull($tokens, 'getValue')); } public function getIndentation() { $tokens = $this->getTokens(); $left = head($tokens); while ($left && (!$left->isAnyWhitespace() || strpos($left->getValue(), "\n") === false)) { $left = $left->getPrevToken(); } if (!$left) { return null; } return preg_replace("/^.*\n/s", '', $left->getValue()); } public function getDescription() { $concrete = $this->getConcreteString(); if (strlen($concrete) > 75) { $concrete = substr($concrete, 0, 36).'...'.substr($concrete, -36); } $concrete = addcslashes($concrete, "\\\n\""); return pht('a node of type %s: "%s"', $this->getTypeName(), $concrete); } - protected final function getTypeIDFromTypeName($type_name) { + final protected function getTypeIDFromTypeName($type_name) { return $this->tree->getNodeTypeIDFromTypeName($type_name); } - public final function getOffset() { + final public function getOffset() { $stream = $this->tree->getRawTokenStream(); if (empty($stream[$this->l])) { return null; } return $stream[$this->l]->getOffset(); } - public final function getLength() { + final public function getLength() { $stream = $this->tree->getRawTokenStream(); if (empty($stream[$this->r])) { return null; } return $stream[$this->r]->getOffset() - $this->getOffset(); } public function getSurroundingNonsemanticTokens() { $before = array(); $after = array(); $tokens = $this->tree->getRawTokenStream(); if ($this->l != -1) { $before = $tokens[$this->l]->getNonsemanticTokensBefore(); } if ($this->r != -1) { $after = $tokens[$this->r]->getNonsemanticTokensAfter(); } return array($before, $after); } - public final function getLineNumber() { + final public function getLineNumber() { return idx($this->tree->getOffsetToLineNumberMap(), $this->getOffset()); } - public final function getEndLineNumber() { + final public function getEndLineNumber() { return idx( $this->tree->getOffsetToLineNumberMap(), $this->getOffset() + $this->getLength()); } public function dispose() { foreach ($this->getChildren() as $child) { $child->dispose(); } unset($this->selectCache); } } diff --git a/src/parser/aast/api/AASTToken.php b/src/parser/aast/api/AASTToken.php index 4b1fcde..c8ac09c 100644 --- a/src/parser/aast/api/AASTToken.php +++ b/src/parser/aast/api/AASTToken.php @@ -1,82 +1,82 @@ id = $id; $this->typeID = $type; $this->value = $value; $this->offset = $offset; $this->tree = $tree; } - public final function getTokenID() { + final public function getTokenID() { return $this->id; } - public final function getTypeID() { + final public function getTypeID() { return $this->typeID; } public function getTypeName() { if (empty($this->typeName)) { $this->typeName = $this->tree->getTokenTypeNameFromTypeID($this->typeID); } return $this->typeName; } - public final function getValue() { + final public function getValue() { return $this->value; } - public final function getOffset() { + final public function getOffset() { return $this->offset; } abstract public function isComment(); abstract public function isAnyWhitespace(); public function isSemantic() { return !($this->isComment() || $this->isAnyWhitespace()); } public function getPrevToken() { $tokens = $this->tree->getRawTokenStream(); return idx($tokens, $this->id - 1); } public function getNextToken() { $tokens = $this->tree->getRawTokenStream(); return idx($tokens, $this->id + 1); } public function getNonsemanticTokensBefore() { $tokens = $this->tree->getRawTokenStream(); $result = array(); $ii = $this->id - 1; while ($ii >= 0 && !$tokens[$ii]->isSemantic()) { $result[$ii] = $tokens[$ii]; --$ii; } return array_reverse($result); } public function getNonsemanticTokensAfter() { $tokens = $this->tree->getRawTokenStream(); $result = array(); $ii = $this->id + 1; while ($ii < count($tokens) && !$tokens[$ii]->isSemantic()) { $result[$ii] = $tokens[$ii]; ++$ii; } return $result; } } diff --git a/src/parser/aast/api/AASTTree.php b/src/parser/aast/api/AASTTree.php index 7ec2440..7ae6d1c 100644 --- a/src/parser/aast/api/AASTTree.php +++ b/src/parser/aast/api/AASTTree.php @@ -1,180 +1,180 @@ stream[$ii] = $this->newToken( $ii, $token[0], substr($source, $offset, $token[1]), $offset, $this); $offset += $token[1]; ++$ii; } $this->rawSource = $source; $this->buildTree(array($tree)); } - public final function setTreeType($description) { + final public function setTreeType($description) { $this->treeType = $description; return $this; } - public final function getTreeType() { + final public function getTreeType() { return $this->treeType; } - public final function setTokenConstants(array $token_map) { + final public function setTokenConstants(array $token_map) { $this->tokenConstants = $token_map; $this->tokenReverseMap = array_flip($token_map); return $this; } - public final function setNodeConstants(array $node_map) { + final public function setNodeConstants(array $node_map) { $this->nodeConstants = $node_map; $this->nodeReverseMap = array_flip($node_map); return $this; } - public final function getNodeTypeNameFromTypeID($type_id) { + final public function getNodeTypeNameFromTypeID($type_id) { if (empty($this->nodeConstants[$type_id])) { $tree_type = $this->getTreeType(); throw new Exception( pht( "No type name for node type ID '%s' in '%s' AAST.", $type_id, $tree_type)); } return $this->nodeConstants[$type_id]; } - public final function getNodeTypeIDFromTypeName($type_name) { + final public function getNodeTypeIDFromTypeName($type_name) { if (empty($this->nodeReverseMap[$type_name])) { $tree_type = $this->getTreeType(); throw new Exception( pht( "No type ID for node type name '%s' in '%s' AAST.", $type_name, $tree_type)); } return $this->nodeReverseMap[$type_name]; } - public final function getTokenTypeNameFromTypeID($type_id) { + final public function getTokenTypeNameFromTypeID($type_id) { if (empty($this->tokenConstants[$type_id])) { $tree_type = $this->getTreeType(); throw new Exception( pht( "No type name for token type ID '%s' in '%s' AAST.", $type_id, $tree_type)); } return $this->tokenConstants[$type_id]; } - public final function getTokenTypeIDFromTypeName($type_name) { + final public function getTokenTypeIDFromTypeName($type_name) { if (empty($this->tokenReverseMap[$type_name])) { $tree_type = $this->getTreeType(); throw new Exception( pht( "No type ID for token type name '%s' in '%s' AAST.", $type_name, $tree_type)); } return $this->tokenReverseMap[$type_name]; } /** * Unlink internal datastructures so that PHP will garbage collect the tree. * * This renders the object useless. * * @return void */ public function dispose() { $this->getRootNode()->dispose(); unset($this->tree); unset($this->stream); } - public final function getRootNode() { + final public function getRootNode() { return $this->tree[0]; } protected function buildTree(array $tree) { $ii = count($this->tree); $nodes = array(); foreach ($tree as $node) { $this->tree[$ii] = $this->newNode($ii, $node, $this); $nodes[$ii] = $node; ++$ii; } foreach ($nodes as $node_id => $node) { if (isset($node[3])) { $children = $this->buildTree($node[3]); foreach ($children as $child) { $child->parentNode = $this->tree[$node_id]; } $this->tree[$node_id]->children = $children; } } $result = array(); foreach ($nodes as $key => $node) { $result[$key] = $this->tree[$key]; } return $result; } - public final function getRawTokenStream() { + final public function getRawTokenStream() { return $this->stream; } public function getOffsetToLineNumberMap() { if ($this->lineMap === null) { $src = $this->rawSource; $len = strlen($src); $lno = 1; $map = array(); for ($ii = 0; $ii < $len; ++$ii) { $map[$ii] = $lno; if ($src[$ii] == "\n") { ++$lno; } } $this->lineMap = $map; } return $this->lineMap; } }