diff --git a/src/daemon/PhutilDaemon.php b/src/daemon/PhutilDaemon.php index 6e41d51..4d7afe6 100644 --- a/src/daemon/PhutilDaemon.php +++ b/src/daemon/PhutilDaemon.php @@ -1,396 +1,396 @@ shouldExit()) { * if (work_available()) { * $this->willBeginWork(); * do_work(); * $this->sleep(0); * } else { * $this->willBeginIdle(); * $this->sleep(1); * } * } * * In particular, call @{method:willBeginWork} before becoming busy, and * @{method:willBeginIdle} when no work is available. If the daemon is launched * into an autoscale pool, this will cause the pool to automatically scale up * when busy and down when idle. * * See @{class:PhutilHighIntensityIntervalDaemon} for an example of a simple * autoscaling daemon. * * Launching a daemon which does not make these callbacks into an autoscale * pool will have no effect. * * @task overseer Communicating With the Overseer * @task autoscale Autoscaling Daemon Pools */ abstract class PhutilDaemon extends Phobject { const MESSAGETYPE_STDOUT = 'stdout'; const MESSAGETYPE_HEARTBEAT = 'heartbeat'; const MESSAGETYPE_BUSY = 'busy'; const MESSAGETYPE_IDLE = 'idle'; const MESSAGETYPE_DOWN = 'down'; const WORKSTATE_BUSY = 'busy'; const WORKSTATE_IDLE = 'idle'; private $argv; private $traceMode; private $traceMemory; private $verbose; private $notifyReceived; private $inGracefulShutdown; private $workState = null; private $idleSince = null; private $autoscaleProperties = array(); final public function setVerbose($verbose) { $this->verbose = $verbose; return $this; } final public function getVerbose() { return $this->verbose; } private static $sighandlerInstalled; final public function __construct(array $argv) { declare(ticks = 1); $this->argv = $argv; if (!self::$sighandlerInstalled) { self::$sighandlerInstalled = true; pcntl_signal(SIGTERM, __CLASS__.'::exitOnSignal'); } pcntl_signal(SIGINT, array($this, 'onGracefulSignal')); pcntl_signal(SIGUSR2, array($this, 'onNotifySignal')); // Without discard mode, this consumes unbounded amounts of memory. Keep // memory bounded. PhutilServiceProfiler::getInstance()->enableDiscardMode(); $this->beginStdoutCapture(); } final public function __destruct() { $this->endStdoutCapture(); } final public function stillWorking() { $this->emitOverseerMessage(self::MESSAGETYPE_HEARTBEAT, null); if ($this->traceMemory) { $daemon = get_class($this); fprintf( STDERR, "<%s> %s %s\n", '', $daemon, pht( 'Memory Usage: %s KB', new PhutilNumber(memory_get_usage() / 1024, 1))); } } final public function shouldExit() { return $this->inGracefulShutdown; } final protected function sleep($duration) { $this->notifyReceived = false; $this->willSleep($duration); $this->stillWorking(); $is_autoscale = $this->isClonedAutoscaleDaemon(); $scale_down = $this->getAutoscaleDownDuration(); $max_sleep = 60; if ($is_autoscale) { $max_sleep = min($max_sleep, $scale_down); } if ($is_autoscale) { if ($this->workState == self::WORKSTATE_IDLE) { $dur = (time() - $this->idleSince); $this->log(pht('Idle for %s seconds.', $dur)); } } while ($duration > 0 && !$this->notifyReceived && !$this->shouldExit()) { // If this is an autoscaling clone and we've been idle for too long, // we're going to scale the pool down by exiting and not restarting. The // DOWN message tells the overseer that we don't want to be restarted. if ($is_autoscale) { if ($this->workState == self::WORKSTATE_IDLE) { if ($this->idleSince && ($this->idleSince + $scale_down < time())) { $this->inGracefulShutdown = true; $this->emitOverseerMessage(self::MESSAGETYPE_DOWN, null); $this->log( pht( - 'Daemon was idle for more than %s seconds, scaling pool '. - 'down.', - $scale_down)); + 'Daemon was idle for more than %s second(s), '. + 'scaling pool down.', + new PhutilNumber($scale_down))); break; } } } sleep(min($duration, $max_sleep)); $duration -= $max_sleep; $this->stillWorking(); } } protected function willSleep($duration) { return; } public static function exitOnSignal($signo) { // Normally, PHP doesn't invoke destructors when existing in response to // a signal. This forces it to do so, so we have a fighting chance of // releasing any locks, leases or resources on our way out. exit(128 + $signo); } final protected function getArgv() { return $this->argv; } final public function execute() { $this->willRun(); $this->run(); } abstract protected function run(); final public function setTraceMemory() { $this->traceMemory = true; return $this; } final public function getTraceMemory() { return $this->traceMemory; } final public function setTraceMode() { $this->traceMode = true; PhutilServiceProfiler::installEchoListener(); PhutilConsole::getConsole()->getServer()->setEnableLog(true); $this->didSetTraceMode(); return $this; } final public function getTraceMode() { return $this->traceMode; } final public function onGracefulSignal($signo) { $this->inGracefulShutdown = true; } final public function onNotifySignal($signo) { $this->notifyReceived = true; $this->onNotify($signo); } protected function onNotify($signo) { // This is a hook for subclasses. } protected function willRun() { // This is a hook for subclasses. } protected function didSetTraceMode() { // This is a hook for subclasses. } final protected function log($message) { if ($this->verbose) { $daemon = get_class($this); fprintf(STDERR, "<%s> %s %s\n", '', $daemon, $message); } } /* -( Communicating With the Overseer )------------------------------------ */ private function beginStdoutCapture() { ob_start(array($this, 'didReceiveStdout'), 2); } private function endStdoutCapture() { ob_end_flush(); } public function didReceiveStdout($data) { if (!strlen($data)) { return ''; } return $this->encodeOverseerMessage(self::MESSAGETYPE_STDOUT, $data); } private function encodeOverseerMessage($type, $data) { $structure = array($type); if ($data !== null) { $structure[] = $data; } return json_encode($structure)."\n"; } private function emitOverseerMessage($type, $data) { $this->endStdoutCapture(); echo $this->encodeOverseerMessage($type, $data); $this->beginStdoutCapture(); } public static function errorListener($event, $value, array $metadata) { // If the caller has redirected the error log to a file, PHP won't output // messages to stderr, so the overseer can't capture them. Install a // listener which just echoes errors to stderr, so the overseer is always // aware of errors. $console = PhutilConsole::getConsole(); $message = idx($metadata, 'default_message'); if ($message) { $console->writeErr("%s\n", $message); } if (idx($metadata, 'trace')) { $trace = PhutilErrorHandler::formatStacktrace($metadata['trace']); $console->writeErr("%s\n", $trace); } } /* -( Autoscaling )-------------------------------------------------------- */ /** * Prepare to become busy. This may autoscale the pool up. * * This notifies the overseer that the daemon has become busy. If daemons * that are part of an autoscale pool are continuously busy for a prolonged * period of time, the overseer may scale up the pool. * * @return this * @task autoscale */ protected function willBeginWork() { if ($this->workState != self::WORKSTATE_BUSY) { $this->workState = self::WORKSTATE_BUSY; $this->idleSince = null; $this->emitOverseerMessage(self::MESSAGETYPE_BUSY, null); } return $this; } /** * Prepare to idle. This may autoscale the pool down. * * This notifies the overseer that the daemon is no longer busy. If daemons * that are part of an autoscale pool are idle for a prolonged period of time, * they may exit to scale the pool down. * * @return this * @task autoscale */ protected function willBeginIdle() { if ($this->workState != self::WORKSTATE_IDLE) { $this->workState = self::WORKSTATE_IDLE; $this->idleSince = time(); $this->emitOverseerMessage(self::MESSAGETYPE_IDLE, null); } return $this; } /** * Determine if this is a clone or the original daemon. * * @return bool True if this is an cloned autoscaling daemon. * @task autoscale */ private function isClonedAutoscaleDaemon() { return (bool)$this->getAutoscaleProperty('clone', false); } /** * Get the duration (in seconds) which a daemon must be continuously idle * for before it should exit to scale the pool down. * * @return int Duration, in seconds. * @task autoscale */ private function getAutoscaleDownDuration() { return $this->getAutoscaleProperty('down', 15); } /** * Configure autoscaling for this daemon. * * @param map Map of autoscale properties. * @return this * @task autoscale */ public function setAutoscaleProperties(array $autoscale_properties) { PhutilTypeSpec::checkMap( $autoscale_properties, array( 'group' => 'optional string', 'up' => 'optional int', 'down' => 'optional int', 'pool' => 'optional int', 'clone' => 'optional bool', 'reserve' => 'optional int|float', )); $this->autoscaleProperties = $autoscale_properties; return $this; } /** * Read autoscaling configuration for this daemon. * * @param string Property to read. * @param wild Default value to return if the property is not set. * @return wild Property value, or `$default` if one is not set. * @task autoscale */ private function getAutoscaleProperty($key, $default = null) { return idx($this->autoscaleProperties, $key, $default); } } diff --git a/src/filesystem/Filesystem.php b/src/filesystem/Filesystem.php index 759423e..0abd41e 100644 --- a/src/filesystem/Filesystem.php +++ b/src/filesystem/Filesystem.php @@ -1,1104 +1,1104 @@ > 3]; } return $result; } /** * Identify the MIME type of a file. This returns only the MIME type (like * text/plain), not the encoding (like charset=utf-8). * * @param string Path to the file to examine. * @param string Optional default mime type to return if the file's mime * type can not be identified. * @return string File mime type. * * @task file * * @phutil-external-symbol function mime_content_type * @phutil-external-symbol function finfo_open * @phutil-external-symbol function finfo_file */ public static function getMimeType( $path, $default = 'application/octet-stream') { $path = self::resolvePath($path); self::assertExists($path); self::assertIsFile($path); self::assertReadable($path); $mime_type = null; // Fileinfo is the best approach since it doesn't rely on `file`, but // it isn't builtin for older versions of PHP. if (function_exists('finfo_open')) { $finfo = finfo_open(FILEINFO_MIME); if ($finfo) { $result = finfo_file($finfo, $path); if ($result !== false) { $mime_type = $result; } } } // If we failed Fileinfo, try `file`. This works well but not all systems // have the binary. if ($mime_type === null) { list($err, $stdout) = exec_manual( 'file --brief --mime %s', $path); if (!$err) { $mime_type = trim($stdout); } } // If we didn't get anywhere, try the deprecated mime_content_type() // function. if ($mime_type === null) { if (function_exists('mime_content_type')) { $result = mime_content_type($path); if ($result !== false) { $mime_type = $result; } } } // If we come back with an encoding, strip it off. if (strpos($mime_type, ';') !== false) { list($type, $encoding) = explode(';', $mime_type, 2); $mime_type = $type; } if ($mime_type === null) { $mime_type = $default; } return $mime_type; } /* -( Directories )-------------------------------------------------------- */ /** * Create a directory in a manner similar to mkdir(), but throw detailed * exceptions on failure. * * @param string Path to directory. The parent directory must exist and * be writable. * @param int Permission umask. Note that umask is in octal, so you * should specify it as, e.g., `0777', not `777'. * @param boolean Recursively create directories. Default to false. * @return string Path to the created directory. * * @task directory */ public static function createDirectory( $path, $umask = 0755, $recursive = false) { $path = self::resolvePath($path); if (is_dir($path)) { if ($umask) { self::changePermissions($path, $umask); } return $path; } $dir = dirname($path); if ($recursive && !file_exists($dir)) { // Note: We could do this with the recursive third parameter of mkdir(), // but then we loose the helpful FilesystemExceptions we normally get. self::createDirectory($dir, $umask, true); } self::assertIsDirectory($dir); self::assertExists($dir); self::assertWritable($dir); self::assertNotExists($path); if (!mkdir($path, $umask)) { throw new FilesystemException( $path, - pht("Failed to create directory `%s'.", $path)); + pht("Failed to create directory '%s'.", $path)); } // Need to change permissions explicitly because mkdir does something // slightly different. mkdir(2) man page: // 'The parameter mode specifies the permissions to use. It is modified by // the process's umask in the usual way: the permissions of the created // directory are (mode & ~umask & 0777)."' if ($umask) { self::changePermissions($path, $umask); } return $path; } /** * Create a temporary directory and return the path to it. You are * responsible for removing it (e.g., with Filesystem::remove()) * when you are done with it. * * @param string Optional directory prefix. * @param int Permissions to create the directory with. By default, * these permissions are very restrictive (0700). * @return string Path to newly created temporary directory. * * @task directory */ public static function createTemporaryDirectory($prefix = '', $umask = 0700) { $prefix = preg_replace('/[^A-Z0-9._-]+/i', '', $prefix); $tmp = sys_get_temp_dir(); if (!$tmp) { throw new FilesystemException( $tmp, pht('Unable to determine system temporary directory.')); } $base = $tmp.DIRECTORY_SEPARATOR.$prefix; $tries = 3; do { $dir = $base.substr(base_convert(md5(mt_rand()), 16, 36), 0, 16); try { self::createDirectory($dir, $umask); break; } catch (FilesystemException $ex) { // Ignore. } } while (--$tries); if (!$tries) { $df = disk_free_space($tmp); if ($df !== false && $df < 1024 * 1024) { throw new FilesystemException( $dir, pht('Failed to create a temporary directory: the disk is full.')); } throw new FilesystemException( $dir, pht("Failed to create a temporary directory in '%s'.", $tmp)); } return $dir; } /** * List files in a directory. * * @param string Path, absolute or relative to PWD. * @param bool If false, exclude files beginning with a ".". * * @return array List of files and directories in the specified * directory, excluding `.' and `..'. * * @task directory */ public static function listDirectory($path, $include_hidden = true) { $path = self::resolvePath($path); self::assertExists($path); self::assertIsDirectory($path); self::assertReadable($path); $list = @scandir($path); if ($list === false) { throw new FilesystemException( $path, - pht("Unable to list contents of directory `%s'.", $path)); + pht("Unable to list contents of directory '%s'.", $path)); } foreach ($list as $k => $v) { if ($v == '.' || $v == '..' || (!$include_hidden && $v[0] == '.')) { unset($list[$k]); } } return array_values($list); } /** * Return all directories between a path and the specified root directory * (defaulting to "/"). Iterating over them walks from the path to the root. * * @param string Path, absolute or relative to PWD. * @param string The root directory. * @return list List of parent paths, including the provided path. * @task directory */ public static function walkToRoot($path, $root = '/') { $path = self::resolvePath($path); $root = self::resolvePath($root); if (is_link($path)) { $path = realpath($path); } if (is_link($root)) { $root = realpath($root); } // NOTE: We don't use `isDescendant()` here because we don't want to reject // paths which don't exist on disk. $root_list = new FileList(array($root)); if (!$root_list->contains($path)) { return array(); } $walk = array(); $parts = explode(DIRECTORY_SEPARATOR, $path); foreach ($parts as $k => $part) { if (!strlen($part)) { unset($parts[$k]); } } while (true) { if (phutil_is_windows()) { $next = implode(DIRECTORY_SEPARATOR, $parts); } else { $next = DIRECTORY_SEPARATOR.implode(DIRECTORY_SEPARATOR, $parts); } $walk[] = $next; if ($next == $root) { break; } if (!$parts) { break; } array_pop($parts); } return $walk; } /* -( Paths )-------------------------------------------------------------- */ /** * Checks if a path is specified as an absolute path. * * @param string * @return bool */ public static function isAbsolutePath($path) { if (phutil_is_windows()) { return (bool)preg_match('/^[A-Za-z]+:/', $path); } else { return !strncmp($path, DIRECTORY_SEPARATOR, 1); } } /** * Canonicalize a path by resolving it relative to some directory (by * default PWD), following parent symlinks and removing artifacts. If the * path is itself a symlink it is left unresolved. * * @param string Path, absolute or relative to PWD. * @return string Canonical, absolute path. * * @task path */ public static function resolvePath($path, $relative_to = null) { $is_absolute = self::isAbsolutePath($path); if (!$is_absolute) { if (!$relative_to) { $relative_to = getcwd(); } $path = $relative_to.DIRECTORY_SEPARATOR.$path; } if (is_link($path)) { $parent_realpath = realpath(dirname($path)); if ($parent_realpath !== false) { return $parent_realpath.DIRECTORY_SEPARATOR.basename($path); } } $realpath = realpath($path); if ($realpath !== false) { return $realpath; } // This won't work if the file doesn't exist or is on an unreadable mount // or something crazy like that. Try to resolve a parent so we at least // cover the nonexistent file case. $parts = explode(DIRECTORY_SEPARATOR, trim($path, DIRECTORY_SEPARATOR)); while (end($parts) !== false) { array_pop($parts); if (phutil_is_windows()) { $attempt = implode(DIRECTORY_SEPARATOR, $parts); } else { $attempt = DIRECTORY_SEPARATOR.implode(DIRECTORY_SEPARATOR, $parts); } $realpath = realpath($attempt); if ($realpath !== false) { $path = $realpath.substr($path, strlen($attempt)); break; } } return $path; } /** * Test whether a path is descendant from some root path after resolving all * symlinks and removing artifacts. Both paths must exists for the relation * to obtain. A path is always a descendant of itself as long as it exists. * * @param string Child path, absolute or relative to PWD. * @param string Root path, absolute or relative to PWD. * @return bool True if resolved child path is in fact a descendant of * resolved root path and both exist. * @task path */ public static function isDescendant($path, $root) { try { self::assertExists($path); self::assertExists($root); } catch (FilesystemException $e) { return false; } $fs = new FileList(array($root)); return $fs->contains($path); } /** * Convert a canonical path to its most human-readable format. It is * guaranteed that you can use resolvePath() to restore a path to its * canonical format. * * @param string Path, absolute or relative to PWD. * @param string Optionally, working directory to make files readable * relative to. * @return string Human-readable path. * * @task path */ public static function readablePath($path, $pwd = null) { if ($pwd === null) { $pwd = getcwd(); } foreach (array($pwd, self::resolvePath($pwd)) as $parent) { $parent = rtrim($parent, DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR; $len = strlen($parent); if (!strncmp($parent, $path, $len)) { $path = substr($path, $len); return $path; } } return $path; } /** * Determine whether or not a path exists in the filesystem. This differs from * file_exists() in that it returns true for symlinks. This method does not * attempt to resolve paths before testing them. * * @param string Test for the existence of this path. * @return bool True if the path exists in the filesystem. * @task path */ public static function pathExists($path) { return file_exists($path) || is_link($path); } /** * Determine if an executable binary (like `git` or `svn`) exists within * the configured `$PATH`. * * @param string Binary name, like `'git'` or `'svn'`. * @return bool True if the binary exists and is executable. * @task exec */ public static function binaryExists($binary) { return self::resolveBinary($binary) !== null; } /** * Locates the full path that an executable binary (like `git` or `svn`) is at * the configured `$PATH`. * * @param string Binary name, like `'git'` or `'svn'`. * @return string The full binary path if it is present, or null. * @task exec */ public static function resolveBinary($binary) { if (phutil_is_windows()) { list($err, $stdout) = exec_manual('where %s', $binary); $stdout = phutil_split_lines($stdout); // If `where %s` could not find anything, check for relative binary if ($err) { $path = self::resolvePath($binary); if (self::pathExists($path)) { return $path; } return null; } $stdout = head($stdout); } else { list($err, $stdout) = exec_manual('which %s', $binary); } return $err === 0 ? trim($stdout) : null; } /** * Determine if two paths are equivalent by resolving symlinks. This is * different from resolving both paths and comparing them because * resolvePath() only resolves symlinks in parent directories, not the * path itself. * * @param string First path to test for equivalence. * @param string Second path to test for equivalence. * @return bool True if both paths are equivalent, i.e. reference the same * entity in the filesystem. * @task path */ public static function pathsAreEquivalent($u, $v) { $u = self::resolvePath($u); $v = self::resolvePath($v); $real_u = realpath($u); $real_v = realpath($v); if ($real_u) { $u = $real_u; } if ($real_v) { $v = $real_v; } return ($u == $v); } /* -( Assert )------------------------------------------------------------- */ /** * Assert that something (e.g., a file, directory, or symlink) exists at a * specified location. * * @param string Assert that this path exists. * @return void * * @task assert */ public static function assertExists($path) { if (!self::pathExists($path)) { throw new FilesystemException( $path, - pht("File system entity `%s' does not exist.", $path)); + pht("File system entity '%s' does not exist.", $path)); } } /** * Assert that nothing exists at a specified location. * * @param string Assert that this path does not exist. * @return void * * @task assert */ public static function assertNotExists($path) { if (file_exists($path) || is_link($path)) { throw new FilesystemException( $path, - pht("Path `%s' already exists!", $path)); + pht("Path '%s' already exists!", $path)); } } /** * Assert that a path represents a file, strictly (i.e., not a directory). * * @param string Assert that this path is a file. * @return void * * @task assert */ public static function assertIsFile($path) { if (!is_file($path)) { throw new FilesystemException( $path, - pht("Requested path `%s' is not a file.", $path)); + pht("Requested path '%s' is not a file.", $path)); } } /** * Assert that a path represents a directory, strictly (i.e., not a file). * * @param string Assert that this path is a directory. * @return void * * @task assert */ public static function assertIsDirectory($path) { if (!is_dir($path)) { throw new FilesystemException( $path, - pht("Requested path `%s' is not a directory.", $path)); + pht("Requested path '%s' is not a directory.", $path)); } } /** * Assert that a file or directory exists and is writable. * * @param string Assert that this path is writable. * @return void * * @task assert */ public static function assertWritable($path) { if (!is_writable($path)) { throw new FilesystemException( $path, - pht("Requested path `%s' is not writable.", $path)); + pht("Requested path '%s' is not writable.", $path)); } } /** * Assert that a file or directory exists and is readable. * * @param string Assert that this path is readable. * @return void * * @task assert */ public static function assertReadable($path) { if (!is_readable($path)) { throw new FilesystemException( $path, - pht("Path `%s' is not readable.", $path)); + pht("Path '%s' is not readable.", $path)); } } } diff --git a/src/future/exec/CommandException.php b/src/future/exec/CommandException.php index 35094ac..a83b86b 100644 --- a/src/future/exec/CommandException.php +++ b/src/future/exec/CommandException.php @@ -1,78 +1,78 @@ command = $command; $this->error = $error; $this->stdout = $stdout; $this->stderr = $stderr; $summary = array(); $summary[] = $this->summarize($message); $summary[] = 'COMMAND'; $summary[] = $this->summarize($command); $summary[] = null; $summary[] = 'STDOUT'; $summary[] = $this->summarize($stdout); $summary[] = null; $summary[] = 'STDERR'; $summary[] = $this->summarize($stderr); $summary = implode("\n", $summary); parent::__construct($summary); } public function getCommand() { return $this->command; } public function getError() { return $this->error; } public function getStdout() { return $this->stdout; } public function getStderr() { return $this->stderr; } private function summarize($string) { if (!strlen($string)) { return '(empty)'; } $limit = 1000; $len = strlen($string); if ($len > $limit) { $cut = $len - $limit; - $suffix = pht('... (%s more bytes) ...', new PhutilNumber($cut)); + $suffix = pht('... (%s more byte(s)) ...', new PhutilNumber($cut)); if ($cut > strlen($suffix)) { $string = substr($string, 0, $limit).$suffix; } } // Strip out any credentials for the purpose of building a human readable // summary of the exception, since these are rarely-if-ever useful when // debugging, but can expose otherwise sensitive information. $string = phutil_censor_credentials($string); return $string; } } diff --git a/src/future/exec/ExecFuture.php b/src/future/exec/ExecFuture.php index 1361a83..c1c545f 100644 --- a/src/future/exec/ExecFuture.php +++ b/src/future/exec/ExecFuture.php @@ -1,960 +1,960 @@ array('pipe', 'r'), // stdin 1 => array('pipe', 'w'), // stdout 2 => array('pipe', 'w'), // stderr ); /* -( Creating ExecFutures )----------------------------------------------- */ /** * Create a new ExecFuture. * * $future = new ExecFuture('wc -l %s', $file_path); * * @param string `sprintf()`-style command string which will be passed * through @{function:csprintf} with the rest of the arguments. * @param ... Zero or more additional arguments for @{function:csprintf}. * @return ExecFuture ExecFuture for running the specified command. * @task create */ public function __construct($command) { $argv = func_get_args(); $this->command = call_user_func_array('csprintf', $argv); $this->stdin = new PhutilRope(); } /* -( Command Information )------------------------------------------------ */ /** * Retrieve the raw command to be executed. * * @return string Raw command. * @task info */ public function getCommand() { return $this->command; } /** * Retrieve the byte limit for the stderr buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStderrSizeLimit() { return $this->stderrSizeLimit; } /** * Retrieve the byte limit for the stdout buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStdoutSizeLimit() { return $this->stdoutSizeLimit; } /** * Get the process's pid. This only works after execution is initiated, e.g. * by a call to start(). * * @return int Process ID of the executing process. * @task info */ public function getPID() { $status = $this->procGetStatus(); return $status['pid']; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a maximum size for the stdout read buffer. To limit stderr, see * @{method:setStderrSizeLimit}. The major use of these methods is to use less * memory if you are running a command which sometimes produces huge volumes * of output that you don't really care about. * * NOTE: Setting this to 0 means "no buffer", not "unlimited buffer". * * @param int Maximum size of the stdout read buffer. * @return this * @task config */ public function setStdoutSizeLimit($limit) { $this->stdoutSizeLimit = $limit; return $this; } /** * Set a maximum size for the stderr read buffer. * See @{method:setStdoutSizeLimit} for discussion. * * @param int Maximum size of the stderr read buffer. * @return this * @task config */ public function setStderrSizeLimit($limit) { $this->stderrSizeLimit = $limit; return $this; } /** * Set the maximum internal read buffer size this future. The future will * block reads once the internal stdout or stderr buffer exceeds this size. * * NOTE: If you @{method:resolve} a future with a read buffer limit, you may * block forever! * * TODO: We should probably release the read buffer limit during * @{method:resolve}, or otherwise detect this. For now, be careful. * * @param int|null Maximum buffer size, or `null` for unlimited. * @return this */ public function setReadBufferSize($read_buffer_size) { $this->readBufferSize = $read_buffer_size; return $this; } /** * Set the current working directory to use when executing the command. * * @param string Directory to set as CWD before executing the command. * @return this * @task config */ public function setCWD($cwd) { $this->cwd = $cwd; return $this; } /** * Set the environment variables to use when executing the command. * * @param array Environment variables to use when executing the command. * @return this * @task config */ public function setEnv($env, $wipe_process_env = false) { if ($wipe_process_env) { $this->env = $env; } else { $this->env = $env + $_ENV; } return $this; } /** * Set the value of a specific environmental variable for this command. * * @param string Environmental variable name. * @param string|null New value, or null to remove this variable. * @return this * @task config */ public function updateEnv($key, $value) { if (!is_array($this->env)) { $this->env = $_ENV; } if ($value === null) { unset($this->env[$key]); } else { $this->env[$key] = $value; } return $this; } /** * Set whether to use non-blocking streams on Windows. * * @param bool Whether to use non-blocking streams. * @return this * @task config */ public function setUseWindowsFileStreams($use_streams) { if (phutil_is_windows()) { $this->useWindowsFileStreams = $use_streams; } return $this; } /* -( Interacting With Commands )------------------------------------------ */ /** * Read and return output from stdout and stderr, if any is available. This * method keeps a read cursor on each stream, but the entire streams are * still returned when the future resolves. You can call read() again after * resolving the future to retrieve only the parts of the streams you did not * previously read: * * $future = new ExecFuture('...'); * // ... * list($stdout) = $future->read(); // Returns output so far * list($stdout) = $future->read(); // Returns new output since first call * // ... * list($stdout) = $future->resolvex(); // Returns ALL output * list($stdout) = $future->read(); // Returns unread output * * NOTE: If you set a limit with @{method:setStdoutSizeLimit} or * @{method:setStderrSizeLimit}, this method will not be able to read data * past the limit. * * NOTE: If you call @{method:discardBuffers}, all the stdout/stderr data * will be thrown away and the cursors will be reset. * * @return pair <$stdout, $stderr> pair with new output since the last call * to this method. * @task interact */ public function read() { $stdout = $this->readStdout(); $result = array( $stdout, (string)substr($this->stderr, $this->stderrPos), ); $this->stderrPos = strlen($this->stderr); return $result; } public function readStdout() { if ($this->start) { $this->isReady(); // Sync } $result = (string)substr($this->stdout, $this->stdoutPos); $this->stdoutPos = strlen($this->stdout); return $result; } /** * Write data to stdin of the command. * * @param string Data to write. * @param bool If true, keep the pipe open for writing. By default, the pipe * will be closed as soon as possible so that commands which * listen for EOF will execute. If you want to keep the pipe open * past the start of command execution, do an empty write with * `$keep_pipe = true` first. * @return this * @task interact */ public function write($data, $keep_pipe = false) { if (strlen($data)) { if (!$this->stdin) { throw new Exception(pht('Writing to a closed pipe!')); } $this->stdin->append($data); } $this->closePipe = !$keep_pipe; return $this; } /** * Permanently discard the stdout and stderr buffers and reset the read * cursors. This is basically useful only if you are streaming a large amount * of data from some process: * * $future = new ExecFuture('zcat huge_file.gz'); * do { * $done = $future->resolve(0.1); // Every 100ms, * list($stdout) = $future->read(); // read output... * echo $stdout; // send it somewhere... * $future->discardBuffers(); // and then free the buffers. * } while ($done === null); * * Conceivably you might also need to do this if you're writing a client using * @{class:ExecFuture} and `netcat`, but you probably should not do that. * * NOTE: This completely discards the data. It won't be available when the * future resolves. This is almost certainly only useful if you need the * buffer memory for some reason. * * @return this * @task interact */ public function discardBuffers() { $this->discardStdoutBuffer(); $this->stderr = ''; $this->stderrPos = 0; return $this; } public function discardStdoutBuffer() { $this->stdout = ''; $this->stdoutPos = 0; return $this; } /** * Returns true if this future was killed by a timeout configured with * @{method:setTimeout}. * * @return bool True if the future was killed for exceeding its time limit. */ public function getWasKilledByTimeout() { return $this->killedByTimeout; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a hard limit on execution time. If the command runs longer, it will * be killed and the future will resolve with an error code. You can test * if a future was killed by a timeout with @{method:getWasKilledByTimeout}. * * @param int Maximum number of seconds this command may execute for. * @return this * @task config */ public function setTimeout($seconds) { $this->timeout = $seconds; return $this; } /* -( Resolving Execution )------------------------------------------------ */ /** * Resolve a command you expect to exit with return code 0. Works like * @{method:resolve}, but throws if $err is nonempty. Returns only * $stdout and $stderr. See also @{function:execx}. * * list($stdout, $stderr) = $future->resolvex(); * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return pair <$stdout, $stderr> pair. * @task resolve */ public function resolvex($timeout = null) { list($err, $stdout, $stderr) = $this->resolve($timeout); if ($err) { $cmd = $this->command; throw new CommandException( pht('Command failed with error #%d!', $err), $cmd, $err, $stdout, $stderr); } return array($stdout, $stderr); } /** * Resolve a command you expect to return valid JSON. Works like * @{method:resolvex}, but also throws if stderr is nonempty, or stdout is not * valid JSON. Returns a PHP array, decoded from the JSON command output. * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return array PHP array, decoded from JSON command output. * @task resolve */ public function resolveJSON($timeout = null) { list($stdout, $stderr) = $this->resolvex($timeout); if (strlen($stderr)) { $cmd = $this->command; throw new CommandException( pht( "JSON command '%s' emitted text to stderr when none was expected: %d", $cmd, $stderr), $cmd, 0, $stdout, $stderr); } try { return phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { $cmd = $this->command; throw new CommandException( pht( "JSON command '%s' did not produce a valid JSON object on stdout: %s", $cmd, $stdout), $cmd, 0, $stdout, $stderr); } } /** * Resolve the process by abruptly terminating it. * * @return list List of results. * @task resolve */ public function resolveKill() { if (!$this->result) { if (defined('SIGKILL')) { $signal = SIGKILL; } else { $signal = 9; } proc_terminate($this->proc, $signal); $this->result = array( 128 + $signal, $this->stdout, $this->stderr, ); $this->closeProcess(); } return $this->result; } /* -( Internals )---------------------------------------------------------- */ /** * Provides read sockets to the future core. * * @return list List of read sockets. * @task internal */ public function getReadSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdout) && !feof($stdout)) { $sockets[] = $stdout; } if (isset($stderr) && !feof($stderr)) { $sockets[] = $stderr; } return $sockets; } /** * Provides write sockets to the future core. * * @return list List of write sockets. * @task internal */ public function getWriteSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdin) && $this->stdin->getByteLength() && !feof($stdin)) { $sockets[] = $stdin; } return $sockets; } /** * Determine if the read buffer is empty. * * @return bool True if the read buffer is empty. * @task internal */ public function isReadBufferEmpty() { return !strlen($this->stdout); } /** * Determine if the write buffer is empty. * * @return bool True if the write buffer is empty. * @task internal */ public function isWriteBufferEmpty() { return !$this->getWriteBufferSize(); } /** * Determine the number of bytes in the write buffer. * * @return int Number of bytes in the write buffer. * @task internal */ public function getWriteBufferSize() { if (!$this->stdin) { return 0; } return $this->stdin->getByteLength(); } /** * Reads some bytes from a stream, discarding output once a certain amount * has been accumulated. * * @param resource Stream to read from. * @param int Maximum number of bytes to return from $stream. If * additional bytes are available, they will be read and * discarded. * @param string Human-readable description of stream, for exception * message. * @param int Maximum number of bytes to read. * @return string The data read from the stream. * @task internal */ private function readAndDiscard($stream, $limit, $description, $length) { $output = ''; if ($length <= 0) { return ''; } do { $data = fread($stream, min($length, 64 * 1024)); if (false === $data) { throw new Exception(pht('Failed to read from %s', $description)); } $read_bytes = strlen($data); if ($read_bytes > 0 && $limit > 0) { if ($read_bytes > $limit) { $data = substr($data, 0, $limit); } $output .= $data; $limit -= strlen($data); } if (strlen($output) >= $length) { break; } } while ($read_bytes > 0); return $output; } /** * Begin or continue command execution. * * @return bool True if future has resolved. * @task internal */ public function isReady() { // NOTE: We have soft dependencies on PhutilServiceProfiler and // PhutilErrorTrap here. These dependencies are soft to avoid the need to // build them into the Phage agent. Under normal circumstances, these // classes are always available. if (!$this->pipes) { // NOTE: See note above about Phage. if (class_exists('PhutilServiceProfiler')) { $profiler = PhutilServiceProfiler::getInstance(); $this->profilerCallID = $profiler->beginServiceCall( array( 'type' => 'exec', 'command' => (string)$this->command, )); } if (!$this->start) { // We might already have started the timer via initiating resolution. $this->start = microtime(true); } $unmasked_command = $this->command; if ($unmasked_command instanceof PhutilCommandString) { $unmasked_command = $unmasked_command->getUnmaskedString(); } $pipes = array(); if (phutil_is_windows()) { // See T4395. proc_open under Windows uses "cmd /C [cmd]", which will // strip the first and last quote when there aren't exactly two quotes // (and some other conditions as well). This results in a command that // looks like `command" "path to my file" "something something` which is // clearly wrong. By surrounding the command string with quotes we can // be sure this process is harmless. if (strpos($unmasked_command, '"') !== false) { $unmasked_command = '"'.$unmasked_command.'"'; } } // NOTE: Convert all the environmental variables we're going to pass // into strings before we install PhutilErrorTrap. If something in here // is really an object which is going to throw when we try to turn it // into a string, we want the exception to escape here -- not after we // start trapping errors. $env = $this->env; if ($env !== null) { foreach ($env as $key => $value) { $env[$key] = (string)$value; } } // Same for the working directory. if ($this->cwd === null) { $cwd = null; } else { $cwd = (string)$this->cwd; } // NOTE: See note above about Phage. if (class_exists('PhutilErrorTrap')) { $trap = new PhutilErrorTrap(); } else { $trap = null; } $spec = self::$descriptorSpec; if ($this->useWindowsFileStreams) { $this->windowsStdoutTempFile = new TempFile(); $this->windowsStderrTempFile = new TempFile(); $spec = array( 0 => self::$descriptorSpec[0], // stdin 1 => fopen($this->windowsStdoutTempFile, 'wb'), // stdout 2 => fopen($this->windowsStderrTempFile, 'wb'), // stderr ); if (!$spec[1] || !$spec[2]) { throw new Exception(pht( 'Unable to create temporary files for '. 'Windows stdout / stderr streams')); } } $proc = @proc_open( $unmasked_command, $spec, $pipes, $cwd, $env); if ($this->useWindowsFileStreams) { fclose($spec[1]); fclose($spec[2]); $pipes = array( 0 => head($pipes), // stdin 1 => fopen($this->windowsStdoutTempFile, 'rb'), // stdout 2 => fopen($this->windowsStderrTempFile, 'rb'), // stderr ); if (!$pipes[1] || !$pipes[2]) { throw new Exception(pht( 'Unable to open temporary files for '. 'reading Windows stdout / stderr streams')); } } if ($trap) { $err = $trap->getErrorsAsString(); $trap->destroy(); } else { $err = error_get_last(); } if (!is_resource($proc)) { throw new Exception( pht( - 'Failed to %s: %s', + 'Failed to `%s`: %s', 'proc_open()', $err)); } $this->pipes = $pipes; $this->proc = $proc; list($stdin, $stdout, $stderr) = $pipes; if (!phutil_is_windows()) { // On Windows, we redirect process standard output and standard error // through temporary files, and then use stream_select to determine // if there's more data to read. if ((!stream_set_blocking($stdout, false)) || (!stream_set_blocking($stderr, false)) || (!stream_set_blocking($stdin, false))) { $this->__destruct(); throw new Exception(pht('Failed to set streams nonblocking.')); } } $this->tryToCloseStdin(); return false; } if (!$this->proc) { return true; } list($stdin, $stdout, $stderr) = $this->pipes; while (isset($this->stdin) && $this->stdin->getByteLength()) { $write_segment = $this->stdin->getAnyPrefix(); $bytes = fwrite($stdin, $write_segment); if ($bytes === false) { throw new Exception(pht('Unable to write to stdin!')); } else if ($bytes) { $this->stdin->removeBytesFromHead($bytes); } else { // Writes are blocked for now. break; } } $this->tryToCloseStdin(); // Read status before reading pipes so that we can never miss data that // arrives between our last read and the process exiting. $status = $this->procGetStatus(); $read_buffer_size = $this->readBufferSize; $max_stdout_read_bytes = PHP_INT_MAX; $max_stderr_read_bytes = PHP_INT_MAX; if ($read_buffer_size !== null) { $max_stdout_read_bytes = $read_buffer_size - strlen($this->stdout); $max_stderr_read_bytes = $read_buffer_size - strlen($this->stderr); } if ($max_stdout_read_bytes > 0) { $this->stdout .= $this->readAndDiscard( $stdout, $this->getStdoutSizeLimit() - strlen($this->stdout), 'stdout', $max_stdout_read_bytes); } if ($max_stderr_read_bytes > 0) { $this->stderr .= $this->readAndDiscard( $stderr, $this->getStderrSizeLimit() - strlen($this->stderr), 'stderr', $max_stderr_read_bytes); } if (!$status['running']) { if ($this->useWindowsFileStreams) { fclose($stdout); fclose($stderr); } $this->result = array( $status['exitcode'], $this->stdout, $this->stderr, ); $this->closeProcess(); return true; } $elapsed = (microtime(true) - $this->start); if ($this->timeout && ($elapsed >= $this->timeout)) { $this->killedByTimeout = true; $this->resolveKill(); return true; } } /** * @return void * @task internal */ public function __destruct() { if (!$this->proc) { return; } // NOTE: If we try to proc_close() an open process, we hang indefinitely. To // avoid this, kill the process explicitly if it's still running. $status = $this->procGetStatus(); if ($status['running']) { $this->resolveKill(); } else { $this->closeProcess(); } } /** * Close and free resources if necessary. * * @return void * @task internal */ private function closeProcess() { foreach ($this->pipes as $pipe) { if (isset($pipe)) { @fclose($pipe); } } $this->pipes = array(null, null, null); if ($this->proc) { @proc_close($this->proc); $this->proc = null; } $this->stdin = null; if ($this->profilerCallID !== null) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall( $this->profilerCallID, array( 'err' => $this->result ? idx($this->result, 0) : null, )); $this->profilerCallID = null; } } /** * Execute `proc_get_status()`, but avoid pitfalls. * * @return dict Process status. * @task internal */ private function procGetStatus() { // After the process exits, we only get one chance to read proc_get_status() // before it starts returning garbage. Make sure we don't throw away the // last good read. if ($this->procStatus) { if (!$this->procStatus['running']) { return $this->procStatus; } } $this->procStatus = proc_get_status($this->proc); return $this->procStatus; } /** * Try to close stdin, if we're done using it. This keeps us from hanging if * the process on the other end of the pipe is waiting for EOF. * * @return void * @task internal */ private function tryToCloseStdin() { if (!$this->closePipe) { // We've been told to keep the pipe open by a call to write(..., true). return; } if ($this->stdin->getByteLength()) { // We still have bytes to write. return; } list($stdin) = $this->pipes; if (!$stdin) { // We've already closed stdin. return; } // There's nothing stopping us from closing stdin, so close it. @fclose($stdin); $this->pipes[0] = null; } public function getDefaultWait() { $wait = parent::getDefaultWait(); if ($this->timeout) { if (!$this->start) { $this->start = microtime(true); } $elapsed = (microtime(true) - $this->start); $wait = max(0, min($this->timeout - $elapsed, $wait)); } return $wait; } } diff --git a/src/future/oauth/PhutilOAuth1Future.php b/src/future/oauth/PhutilOAuth1Future.php index 4ef1b86..8e91f8f 100644 --- a/src/future/oauth/PhutilOAuth1Future.php +++ b/src/future/oauth/PhutilOAuth1Future.php @@ -1,291 +1,291 @@ callbackURI = $callback_uri; return $this; } public function setTimestamp($timestamp) { $this->timestamp = $timestamp; return $this; } public function setNonce($nonce) { $this->nonce = $nonce; return $this; } public function setTokenSecret($token_secret) { $this->tokenSecret = $token_secret; return $this; } public function setToken($token) { $this->token = $token; return $this; } public function setPrivateKey(PhutilOpaqueEnvelope $private_key) { $this->privateKey = $private_key; return $this; } public function setSignatureMethod($signature_method) { $this->signatureMethod = $signature_method; return $this; } public function setConsumerKey($consumer_key) { $this->consumerKey = $consumer_key; return $this; } public function setConsumerSecret(PhutilOpaqueEnvelope $consumer_secret) { $this->consumerSecret = $consumer_secret; return $this; } public function setMethod($method) { $this->method = $method; return $this; } public function __construct($uri, $data = array()) { $this->uri = new PhutilURI((string)$uri); $this->data = $data; $this->setProxiedFuture(new HTTPSFuture($uri, $data)); } public function getSignature() { $params = array(); // NOTE: The JIRA API uses JSON-encoded request bodies which are not // signed, and OAuth1 provides no real way to sign a nonparameterized // request body. Possibly we should split this apart into flags which // control which data is signed, but for now this rule seems to cover // all the use cases. if (is_array($this->data)) { $params = $this->data; } $params = $params + $this->uri->getQueryParams() + $this->getOAuth1Headers(); return $this->sign($params); } public function addHeader($name, $value) { // If we haven't built the future yet, hold on to the header until after // we do, since there might be more changes coming which will affect the // signature process. if (!$this->hasConstructedFuture) { $this->headers[] = array($name, $value); } else { $this->getProxiedFuture()->addHeader($name, $value); } return $this; } protected function getProxiedFuture() { $future = parent::getProxiedFuture(); if (!$this->hasConstructedFuture) { $future->setMethod($this->method); $oauth_headers = $this->getOAuth1Headers(); $oauth_headers['oauth_signature'] = $this->getSignature(); $full_oauth_header = array(); foreach ($oauth_headers as $header => $value) { $full_oauth_header[] = $header.'="'.urlencode($value).'"'; } $full_oauth_header = 'OAuth '.implode(', ', $full_oauth_header); $future->addHeader('Authorization', $full_oauth_header); foreach ($this->headers as $header) { $future->addHeader($header[0], $header[1]); } $this->headers = array(); $this->hasConstructedFuture = true; } return $future; } protected function didReceiveResult($result) { return $result; } private function getOAuth1Headers() { if (!$this->nonce) { $this->nonce = Filesystem::readRandomCharacters(32); } if (!$this->timestamp) { $this->timestamp = time(); } $oauth_headers = array( 'oauth_consumer_key' => $this->consumerKey, 'oauth_signature_method' => $this->signatureMethod, 'oauth_timestamp' => $this->timestamp, 'oauth_nonce' => $this->nonce, 'oauth_version' => '1.0', ); if ($this->callbackURI) { $oauth_headers['oauth_callback'] = (string)$this->callbackURI; } if ($this->token) { $oauth_headers['oauth_token'] = $this->token; } return $oauth_headers; } private function sign(array $params) { ksort($params); $pstr = array(); foreach ($params as $key => $value) { $pstr[] = rawurlencode($key).'='.rawurlencode($value); } $pstr = implode('&', $pstr); $sign_uri = clone $this->uri; $sign_uri->setFragment(''); $sign_uri->setQueryParams(array()); $sign_uri->setProtocol(phutil_utf8_strtolower($sign_uri->getProtocol())); $protocol = $sign_uri->getProtocol(); switch ($protocol) { case 'http': if ($sign_uri->getPort() == 80) { $sign_uri->setPort(null); } break; case 'https': if ($sign_uri->getPort() == 443) { $sign_uri->setPort(null); } break; } $method = rawurlencode(phutil_utf8_strtoupper($this->method)); $sign_uri = rawurlencode((string)$sign_uri); $pstr = rawurlencode($pstr); $sign_input = "{$method}&{$sign_uri}&{$pstr}"; return $this->signString($sign_input); } private function signString($string) { $consumer_secret = null; if ($this->consumerSecret) { $consumer_secret = $this->consumerSecret->openEnvelope(); } $key = urlencode($consumer_secret).'&'.urlencode($this->tokenSecret); switch ($this->signatureMethod) { case 'HMAC-SHA1': if (!$this->consumerSecret) { throw new Exception( pht( "Signature method '%s' requires %s!", 'HMAC-SHA1', 'setConsumerSecret()')); } $hash = hash_hmac('sha1', $string, $key, true); return base64_encode($hash); case 'RSA-SHA1': if (!$this->privateKey) { throw new Exception( pht( "Signature method '%s' requires %s!", 'RSA-SHA1', 'setPrivateKey()')); } $cert = @openssl_pkey_get_private($this->privateKey->openEnvelope()); if (!$cert) { - throw new Exception(pht('%S failed!', 'openssl_pkey_get_private()')); + throw new Exception(pht('%s failed!', 'openssl_pkey_get_private()')); } $pkey = @openssl_get_privatekey($cert); if (!$pkey) { throw new Exception(pht('%s failed!', 'openssl_get_privatekey()')); } $signature = null; $ok = openssl_sign($string, $signature, $pkey, OPENSSL_ALGO_SHA1); if (!$ok) { throw new Exception(pht('%s failed!', 'openssl_sign()')); } openssl_free_key($pkey); return base64_encode($signature); case 'PLAINTEXT': if (!$this->consumerSecret) { throw new Exception( pht( "Signature method '%s' requires %s!", 'PLAINTEXT', 'setConsumerSecret()')); } return $key; default: throw new Exception(pht("Unknown signature method '%s'!", $string)); } } public function resolvex() { $result = $this->getProxiedFuture()->resolvex(); return $this->didReceiveResult($result); } public function resolveJSON() { $result = $this->getProxiedFuture()->resolvex(); $result = $this->didReceiveResult($result); list($body) = $result; try { return phutil_json_decode($body); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException(pht('Expected JSON.'), $ex); } } } diff --git a/src/moduleutils/PhutilLibraryMapBuilder.php b/src/moduleutils/PhutilLibraryMapBuilder.php index 882e5bc..af63ca3 100644 --- a/src/moduleutils/PhutilLibraryMapBuilder.php +++ b/src/moduleutils/PhutilLibraryMapBuilder.php @@ -1,509 +1,511 @@ root = $root; } /** * Control status output. Use `--quiet` to set this. * * @param bool If true, don't show status output. * @return this * * @task map */ public function setQuiet($quiet) { $this->quiet = $quiet; return $this; } /** * Control subprocess parallelism limit. Use `--limit` to set this. * * @param int Maximum number of subprocesses to run in parallel. * @return this * * @task map */ public function setSubprocessLimit($limit) { $this->subprocessLimit = $limit; return $this; } /** * Get the map of symbols in this library, analyzing the library to build it * if necessary. * * @return map Information about symbols in this library. * * @task map */ public function buildMap() { if ($this->librarySymbolMap === null) { $this->analyzeLibrary(); } return $this->librarySymbolMap; } /** * Get the map of files in this library, analyzing the library to build it * if necessary. * * Returns a map of file paths to information about symbols used and defined * in the file. * * @return map Information about files in this library. * * @task map */ public function buildFileSymbolMap() { if ($this->fileSymbolMap === null) { $this->analyzeLibrary(); } return $this->fileSymbolMap; } /** * Build and update the library map. * * @return void * * @task map */ public function buildAndWriteMap() { $library_map = $this->buildMap(); $this->log(pht('Writing map...')); $this->writeLibraryMap($library_map); } /** * Write a status message to the user, if not running in quiet mode. * * @param string Message to write. * @return this * * @task map */ private function log($message) { if (!$this->quiet) { @fwrite(STDERR, "%s\n", $message); } return $this; } /* -( Path Management )---------------------------------------------------- */ /** * Get the path to some file in the library. * * @param string A library-relative path. If omitted, returns the library * root path. * @return string An absolute path. * * @task path */ private function getPath($path = '') { return $this->root.'/'.$path; } /** * Get the path to the symbol cache file. * * @return string Absolute path to symbol cache. * * @task path */ private function getPathForSymbolCache() { return $this->getPath('.phutil_module_cache'); } /** * Get the path to the map file. * * @return string Absolute path to the library map. * * @task path */ private function getPathForLibraryMap() { return $this->getPath('__phutil_library_map__.php'); } /** * Get the path to the library init file. * * @return string Absolute path to the library init file * * @task path */ private function getPathForLibraryInit() { return $this->getPath('__phutil_library_init__.php'); } /* -( Symbol Analysis and Caching )---------------------------------------- */ /** * Load the library symbol cache, if it exists and is readable and valid. * * @return dict Map of content hashes to cache of output from * `phutil_symbols.php`. * * @task symbol */ private function loadSymbolCache() { $cache_file = $this->getPathForSymbolCache(); try { $cache = Filesystem::readFile($cache_file); } catch (Exception $ex) { $cache = null; } $symbol_cache = array(); if ($cache) { try { $symbol_cache = phutil_json_decode($cache); } catch (PhutilJSONParserException $ex) { $symbol_cache = array(); } } $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY); if ($version != self::SYMBOL_CACHE_VERSION) { // Throw away caches from a different version of the library. $symbol_cache = array(); } unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]); return $symbol_cache; } /** * Write a symbol map to disk cache. * * @param dict Symbol map of relative paths to symbols. * @param dict Source map (like @{method:loadSourceFileMap}). * @return void * * @task symbol */ private function writeSymbolCache(array $symbol_map, array $source_map) { $cache_file = $this->getPathForSymbolCache(); $cache = array( self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION, ); foreach ($symbol_map as $file => $symbols) { $cache[$source_map[$file]] = $symbols; } $json = json_encode($cache); try { Filesystem::writeFile($cache_file, $json); } catch (FilesystemException $ex) { $this->log(pht('Unable to save the cache!')); } } /** * Drop the symbol cache, forcing a clean rebuild. * * @return this * * @task symbol */ public function dropSymbolCache() { $this->log(pht('Dropping symbol cache...')); Filesystem::remove($this->getPathForSymbolCache()); } /** * Build a future which returns a `phutil_symbols.php` analysis of a source * file. * * @param string Relative path to the source file to analyze. * @return Future Analysis future. * * @task symbol */ private function buildSymbolAnalysisFuture($file) { $absolute_file = $this->getPath($file); $bin = dirname(__FILE__).'/../../scripts/phutil_symbols.php'; return new ExecFuture('php %s --ugly -- %s', $bin, $absolute_file); } /* -( Source Management )-------------------------------------------------- */ /** * Build a map of all source files in a library to hashes of their content. * Returns an array like this: * * array( * 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3', * // ... * ); * * @return dict Map of library-relative paths to content hashes. * @task source */ private function loadSourceFileMap() { $root = $this->getPath(); $init = $this->getPathForLibraryInit(); if (!Filesystem::pathExists($init)) { throw new Exception( pht( "Provided path '%s' is not a %s library.", $root, 'phutil')); } $files = id(new FileFinder($root)) ->withType('f') ->withSuffix('php') ->excludePath('*/.*') ->setGenerateChecksums(true) ->find(); $map = array(); foreach ($files as $file => $hash) { $file = Filesystem::readablePath($file, $root); $file = ltrim($file, '/'); if (dirname($file) == '.') { // We don't permit normal source files at the root level, so just ignore // them; they're special library files. continue; } if (dirname($file) == 'extensions') { // Ignore files in the extensions/ directory. continue; } // We include also filename in the hash to handle cases when the file is // moved without modifying its content. $map[$file] = md5($hash.$file); } return $map; } /** * Convert the symbol analysis of all the source files in the library into * a library map. * * @param dict Symbol analysis of all source files. * @return dict Library map. * @task source */ private function buildLibraryMap(array $symbol_map) { $library_map = array( 'class' => array(), 'function' => array(), 'xmap' => array(), ); // Detect duplicate symbols within the library. foreach ($symbol_map as $file => $info) { foreach ($info['have'] as $type => $symbols) { foreach ($symbols as $symbol => $declaration) { $lib_type = ($type == 'interface') ? 'class' : $type; if (!empty($library_map[$lib_type][$symbol])) { $prior = $library_map[$lib_type][$symbol]; throw new Exception( pht( "Definition of %s '%s' in file '%s' duplicates prior ". "definition in file '%s'. You can not declare the ". "same symbol twice.", $type, $symbol, $file, $prior)); } $library_map[$lib_type][$symbol] = $file; } } $library_map['xmap'] += $info['xmap']; } // Simplify the common case (one parent) to make the file a little easier // to deal with. foreach ($library_map['xmap'] as $class => $extends) { if (count($extends) == 1) { $library_map['xmap'][$class] = reset($extends); } } // Sort the map so it is relatively stable across changes. foreach ($library_map as $key => $symbols) { ksort($symbols); $library_map[$key] = $symbols; } ksort($library_map); return $library_map; } /** * Write a finalized library map. * * @param dict Library map structure to write. * @return void * * @task source */ private function writeLibraryMap(array $library_map) { $map_file = $this->getPathForLibraryMap(); $version = self::LIBRARY_MAP_VERSION; $library_map = array( self::LIBRARY_MAP_VERSION_KEY => $version, ) + $library_map; $library_map = phutil_var_export($library_map); $at = '@'; $source_file = <<log(pht('Finding source files...')); $source_map = $this->loadSourceFileMap(); $this->log( pht('Found %s files.', new PhutilNumber(count($source_map)))); // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. $this->log(pht('Loading symbol cache...')); $symbol_cache = $this->loadSymbolCache(); // If the XHPAST binary is not up-to-date, build it now. Otherwise, // `phutil_symbols.php` will attempt to build the binary and will fail // miserably because it will be trying to build the same file multiple // times in parallel. if (!PhutilXHPASTBinary::isAvailable()) { PhutilXHPASTBinary::build(); } // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } $this->log( pht('Found %s files in cache.', new PhutilNumber(count($symbol_map)))); // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; - $count = number_format(count($futures)); $this->log( - pht('Analyzing %d files with %d subprocesses...', $count, $limit)); + pht( + 'Analyzing %s file(s) with %s subprocess(es)...', + phutil_count($futures), + new PhutilNumber($limit))); $progress = new PhutilConsoleProgressBar(); if ($this->quiet) { $progress->setQuiet(true); } $progress->setTotal(count($futures)); $futures = id(new FutureIterator($futures)) ->limit($limit); foreach ($futures as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); throw new XHPASTSyntaxErrorException( $result['line'], $file.': '.$result['error']); } $progress->update(1); } $progress->done(); } $this->fileSymbolMap = $symbol_map; // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); // Our map is up to date, so either show it on stdout or write it to disk. $this->log(pht('Building library map...')); $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); } } diff --git a/src/parser/__tests__/PhutilJSONTestCase.php b/src/parser/__tests__/PhutilJSONTestCase.php index 2e98cc4..295d7b5 100644 --- a/src/parser/__tests__/PhutilJSONTestCase.php +++ b/src/parser/__tests__/PhutilJSONTestCase.php @@ -1,21 +1,21 @@ assertEqual( $expect, $serializer->encodeFormatted(array('x' => array())), - pht('Empty arrays should serialize as %s, not %s.', '[]', '{}')); + pht('Empty arrays should serialize as `%s`, not `%s`.', '[]', '{}')); } } diff --git a/src/parser/xhpast/__tests__/PHPASTParserTestCase.php b/src/parser/xhpast/__tests__/PHPASTParserTestCase.php index 5a544a1..2c919ff 100644 --- a/src/parser/xhpast/__tests__/PHPASTParserTestCase.php +++ b/src/parser/xhpast/__tests__/PHPASTParserTestCase.php @@ -1,145 +1,146 @@ assertSkipped( pht('%s is not built or not up to date.', 'xhpast')); } } $dir = dirname(__FILE__).'/data/'; foreach (Filesystem::listDirectory($dir) as $file) { if (preg_match('/\.test$/', $file)) { $this->executeParserTest($file, Filesystem::readFile($dir.$file)); } } } private function executeParserTest($name, $data) { $data = explode("\n", $data, 2); if (count($data) !== 2) { throw new Exception( pht('Expected multiple lines in parser test file "%s".', $name)); } $head = head($data); $body = last($data); if (!preg_match('/^#/', $head)) { throw new Exception( pht( - 'Expected first line of parser test file "%s" to begin with "#" '. + 'Expected first line of parser test file "%s" to begin with `%s` '. 'and specify test options.', - $name)); + $name, + '#')); } $head = preg_replace('/^#\s*/', '', $head); $options_parser = new PhutilSimpleOptions(); $options = $options_parser->parse($head); $type = null; foreach ($options as $key => $value) { switch ($key) { case 'pass': case 'fail-syntax': case 'fail-parse': if ($type !== null) { throw new Exception( pht( 'Test file "%s" unexpectedly specifies multiple expected '. 'test outcomes.', $name)); } $type = $key; break; case 'comment': // Human readable comment providing test case information. break; case 'rtrim': // Allows construction of tests which rely on EOF without newlines. $body = rtrim($body); break; default: throw new Exception( pht( 'Test file "%s" has unknown option "%s" in its options '. 'string.', $name, $key)); } } if ($type === null) { throw new Exception( pht( 'Test file "%s" does not specify a test result (like "pass") in '. 'its options string.', $name)); } $future = PhutilXHPASTBinary::getParserFuture($body); list($err, $stdout, $stderr) = $future->resolve(); switch ($type) { case 'pass': case 'fail-parse': $this->assertEqual(0, $err, pht('Exit code for "%s".', $name)); $expect_name = preg_replace('/\.test$/', '.expect', $name); $dir = dirname(__FILE__).'/data/'; $expect = Filesystem::readFile($dir.$expect_name); try { $expect = phutil_json_decode($expect); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht( 'Test ".expect" file "%s" for test "%s" is not valid JSON.', $expect_name, $name), $ex); } try { $stdout = phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht( 'Output for test file "%s" is not valid JSON.', $name), $ex); } $json = new PhutilJSON(); $expect_nice = $json->encodeFormatted($expect); $stdout_nice = $json->encodeFormatted($stdout); if ($type == 'pass') { $this->assertEqual( $expect_nice, $stdout_nice, pht('Parser output for "%s".', $name)); } else { $this->assertFalse( ($expect_nice == $stdout_nice), pht('Expected parser to parse "%s" incorrectly.', $name)); } break; case 'fail-syntax': $this->assertEqual(1, $err, pht('Exit code for "%s".', $name)); $this->assertTrue( (bool)preg_match('/syntax error/', $stderr), pht('Expect "syntax error" in stderr or "%s".', $name)); break; } } } diff --git a/src/serviceprofiler/PhutilServiceProfiler.php b/src/serviceprofiler/PhutilServiceProfiler.php index 2eecf4b..352dcd6 100644 --- a/src/serviceprofiler/PhutilServiceProfiler.php +++ b/src/serviceprofiler/PhutilServiceProfiler.php @@ -1,163 +1,165 @@ discardMode = true; } public static function getInstance() { if (empty(self::$instance)) { self::$instance = new PhutilServiceProfiler(); } return self::$instance; } public function beginServiceCall(array $data) { $data['begin'] = microtime(true); $id = $this->logSize++; $this->events[$id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'begin', $id, $data); } return $id; } public function endServiceCall($call_id, array $data) { $data = ($this->events[$call_id] + $data); $data['end'] = microtime(true); $data['duration'] = ($data['end'] - $data['begin']); $this->events[$call_id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'end', $call_id, $data); } if ($this->discardMode) { unset($this->events[$call_id]); } } public function getServiceCallLog() { return $this->events; } public function addListener($callback) { $this->listeners[] = $callback; } public static function installEchoListener() { $instance = self::getInstance(); $instance->addListener(array(__CLASS__, 'echoListener')); } public static function echoListener($type, $id, $data) { $is_begin = false; $is_end = false; switch ($type) { case 'begin': $is_begin = true; $mark = '>>>'; break; case 'end': $is_end = true; $mark = '<<<'; break; default: $mark = null; break; } $type = idx($data, 'type', 'mystery'); $desc = null; if ($is_begin) { switch ($type) { case 'connect': $desc = $data['database']; break; case 'query': $desc = substr($data['query'], 0, 512); break; case 'multi-query': $desc = array(); foreach ($data['queries'] as $query) { $desc[] = substr($query, 0, 256); } $desc = implode('; ', $desc); break; case 'exec': $desc = '$ '.$data['command']; break; case 'conduit': if (isset($data['size'])) { - $desc = $data['method'].'() '.pht('', $data['size']); + $desc = $data['method'].'() '; + $desc .= pht('', new PhutilNumber($data['size'])); } else { $desc = $data['method'].'()'; } break; case 'http': $desc = phutil_censor_credentials($data['uri']); break; case 'lint': $desc = $data['linter']; if (isset($data['paths'])) { - $desc .= ' '.pht('', count($data['paths'])); + $desc .= ' '.pht('', phutil_count($data['paths'])); } break; case 'lock': $desc = $data['name']; break; case 'event': - $desc = $data['kind'].' '.pht('', $data['count']); + $desc = $data['kind'].' '; + $desc .= pht('', new PhutilNumber($data['count'])); break; case 'ldap': $call = idx($data, 'call', '?'); $params = array(); switch ($call) { case 'connect': $params[] = $data['host'].':'.$data['port']; break; case 'start-tls': break; case 'bind': $params[] = $data['user']; break; case 'search': $params[] = $data['dn']; $params[] = $data['query']; break; default: $params[] = '?'; break; } $desc = "{$call} (".implode(', ', $params).")"; break; } } else if ($is_end) { $desc = pht( '%s us', new PhutilNumber((int)(1000000 * $data['duration']))); } $console = PhutilConsole::getConsole(); $console->writeLog( "%s [%s] <%s> %s\n", $mark, $id, $type, $desc); } } diff --git a/src/utils/utils.php b/src/utils/utils.php index 5e5f305..a6ca161 100644 --- a/src/utils/utils.php +++ b/src/utils/utils.php @@ -1,1409 +1,1409 @@ doStuff(); * * ...but this works fine: * * id(new Thing())->doStuff(); * * @param wild Anything. * @return wild Unmodified argument. */ function id($x) { return $x; } /** * Access an array index, retrieving the value stored there if it exists or * a default if it does not. This function allows you to concisely access an * index which may or may not exist without raising a warning. * * @param array Array to access. * @param scalar Index to access in the array. * @param wild Default value to return if the key is not present in the * array. * @return wild If `$array[$key]` exists, that value is returned. If not, * $default is returned without raising a warning. */ function idx(array $array, $key, $default = null) { // isset() is a micro-optimization - it is fast but fails for null values. if (isset($array[$key])) { return $array[$key]; } // Comparing $default is also a micro-optimization. if ($default === null || array_key_exists($key, $array)) { return null; } return $default; } /** * Call a method on a list of objects. Short for "method pull", this function * works just like @{function:ipull}, except that it operates on a list of * objects instead of a list of arrays. This function simplifies a common type * of mapping operation: * * COUNTEREXAMPLE * $names = array(); * foreach ($objects as $key => $object) { * $names[$key] = $object->getName(); * } * * You can express this more concisely with mpull(): * * $names = mpull($objects, 'getName'); * * mpull() takes a third argument, which allows you to do the same but for * the array's keys: * * COUNTEREXAMPLE * $names = array(); * foreach ($objects as $object) { * $names[$object->getID()] = $object->getName(); * } * * This is the mpull version(): * * $names = mpull($objects, 'getName', 'getID'); * * If you pass ##null## as the second argument, the objects will be preserved: * * COUNTEREXAMPLE * $id_map = array(); * foreach ($objects as $object) { * $id_map[$object->getID()] = $object; * } * * With mpull(): * * $id_map = mpull($objects, null, 'getID'); * * See also @{function:ipull}, which works similarly but accesses array indexes * instead of calling methods. * * @param list Some list of objects. * @param string|null Determines which **values** will appear in the result * array. Use a string like 'getName' to store the * value of calling the named method in each value, or * ##null## to preserve the original objects. * @param string|null Determines how **keys** will be assigned in the result * array. Use a string like 'getID' to use the result * of calling the named method as each object's key, or * `null` to preserve the original keys. * @return dict A dictionary with keys and values derived according * to whatever you passed as `$method` and `$key_method`. */ function mpull(array $list, $method, $key_method = null) { $result = array(); foreach ($list as $key => $object) { if ($key_method !== null) { $key = $object->$key_method(); } if ($method !== null) { $value = $object->$method(); } else { $value = $object; } $result[$key] = $value; } return $result; } /** * Access a property on a list of objects. Short for "property pull", this * function works just like @{function:mpull}, except that it accesses object * properties instead of methods. This function simplifies a common type of * mapping operation: * * COUNTEREXAMPLE * $names = array(); * foreach ($objects as $key => $object) { * $names[$key] = $object->name; * } * * You can express this more concisely with ppull(): * * $names = ppull($objects, 'name'); * * ppull() takes a third argument, which allows you to do the same but for * the array's keys: * * COUNTEREXAMPLE * $names = array(); * foreach ($objects as $object) { * $names[$object->id] = $object->name; * } * * This is the ppull version(): * * $names = ppull($objects, 'name', 'id'); * * If you pass ##null## as the second argument, the objects will be preserved: * * COUNTEREXAMPLE * $id_map = array(); * foreach ($objects as $object) { * $id_map[$object->id] = $object; * } * * With ppull(): * * $id_map = ppull($objects, null, 'id'); * * See also @{function:mpull}, which works similarly but calls object methods * instead of accessing object properties. * * @param list Some list of objects. * @param string|null Determines which **values** will appear in the result * array. Use a string like 'name' to store the value of * accessing the named property in each value, or * `null` to preserve the original objects. * @param string|null Determines how **keys** will be assigned in the result * array. Use a string like 'id' to use the result of * accessing the named property as each object's key, or * `null` to preserve the original keys. * @return dict A dictionary with keys and values derived according * to whatever you passed as `$property` and * `$key_property`. */ function ppull(array $list, $property, $key_property = null) { $result = array(); foreach ($list as $key => $object) { if ($key_property !== null) { $key = $object->$key_property; } if ($property !== null) { $value = $object->$property; } else { $value = $object; } $result[$key] = $value; } return $result; } /** * Choose an index from a list of arrays. Short for "index pull", this function * works just like @{function:mpull}, except that it operates on a list of * arrays and selects an index from them instead of operating on a list of * objects and calling a method on them. * * This function simplifies a common type of mapping operation: * * COUNTEREXAMPLE * $names = array(); * foreach ($list as $key => $dict) { * $names[$key] = $dict['name']; * } * * With ipull(): * * $names = ipull($list, 'name'); * * See @{function:mpull} for more usage examples. * * @param list Some list of arrays. * @param scalar|null Determines which **values** will appear in the result * array. Use a scalar to select that index from each * array, or null to preserve the arrays unmodified as * values. * @param scalar|null Determines which **keys** will appear in the result * array. Use a scalar to select that index from each * array, or null to preserve the array keys. * @return dict A dictionary with keys and values derived according * to whatever you passed for `$index` and `$key_index`. */ function ipull(array $list, $index, $key_index = null) { $result = array(); foreach ($list as $key => $array) { if ($key_index !== null) { $key = $array[$key_index]; } if ($index !== null) { $value = $array[$index]; } else { $value = $array; } $result[$key] = $value; } return $result; } /** * Group a list of objects by the result of some method, similar to how * GROUP BY works in an SQL query. This function simplifies grouping objects * by some property: * * COUNTEREXAMPLE * $animals_by_species = array(); * foreach ($animals as $animal) { * $animals_by_species[$animal->getSpecies()][] = $animal; * } * * This can be expressed more tersely with mgroup(): * * $animals_by_species = mgroup($animals, 'getSpecies'); * * In either case, the result is a dictionary which maps species (e.g., like * "dog") to lists of animals with that property, so all the dogs are grouped * together and all the cats are grouped together, or whatever super * businessesey thing is actually happening in your problem domain. * * See also @{function:igroup}, which works the same way but operates on * array indexes. * * @param list List of objects to group by some property. * @param string Name of a method, like 'getType', to call on each object * in order to determine which group it should be placed into. * @param ... Zero or more additional method names, to subgroup the * groups. * @return dict Dictionary mapping distinct method returns to lists of * all objects which returned that value. */ function mgroup(array $list, $by /* , ... */) { $map = mpull($list, $by); $groups = array(); foreach ($map as $group) { // Can't array_fill_keys() here because 'false' gets encoded wrong. $groups[$group] = array(); } foreach ($map as $key => $group) { $groups[$group][$key] = $list[$key]; } $args = func_get_args(); $args = array_slice($args, 2); if ($args) { array_unshift($args, null); foreach ($groups as $group_key => $grouped) { $args[0] = $grouped; $groups[$group_key] = call_user_func_array('mgroup', $args); } } return $groups; } /** * Group a list of arrays by the value of some index. This function is the same * as @{function:mgroup}, except it operates on the values of array indexes * rather than the return values of method calls. * * @param list List of arrays to group by some index value. * @param string Name of an index to select from each array in order to * determine which group it should be placed into. * @param ... Zero or more additional indexes names, to subgroup the * groups. * @return dict Dictionary mapping distinct index values to lists of * all objects which had that value at the index. */ function igroup(array $list, $by /* , ... */) { $map = ipull($list, $by); $groups = array(); foreach ($map as $group) { $groups[$group] = array(); } foreach ($map as $key => $group) { $groups[$group][$key] = $list[$key]; } $args = func_get_args(); $args = array_slice($args, 2); if ($args) { array_unshift($args, null); foreach ($groups as $group_key => $grouped) { $args[0] = $grouped; $groups[$group_key] = call_user_func_array('igroup', $args); } } return $groups; } /** * Sort a list of objects by the return value of some method. In PHP, this is * often vastly more efficient than `usort()` and similar. * * // Sort a list of Duck objects by name. * $sorted = msort($ducks, 'getName'); * * It is usually significantly more efficient to define an ordering method * on objects and call `msort()` than to write a comparator. It is often more * convenient, as well. * * NOTE: This method does not take the list by reference; it returns a new list. * * @param list List of objects to sort by some property. * @param string Name of a method to call on each object; the return values * will be used to sort the list. * @return list Objects ordered by the return values of the method calls. */ function msort(array $list, $method) { $surrogate = mpull($list, $method); asort($surrogate); $result = array(); foreach ($surrogate as $key => $value) { $result[$key] = $list[$key]; } return $result; } /** * Sort a list of arrays by the value of some index. This method is identical to * @{function:msort}, but operates on a list of arrays instead of a list of * objects. * * @param list List of arrays to sort by some index value. * @param string Index to access on each object; the return values * will be used to sort the list. * @return list Arrays ordered by the index values. */ function isort(array $list, $index) { $surrogate = ipull($list, $index); asort($surrogate); $result = array(); foreach ($surrogate as $key => $value) { $result[$key] = $list[$key]; } return $result; } /** * Filter a list of objects by executing a method across all the objects and * filter out the ones with empty() results. this function works just like * @{function:ifilter}, except that it operates on a list of objects instead * of a list of arrays. * * For example, to remove all objects with no children from a list, where * 'hasChildren' is a method name, do this: * * mfilter($list, 'hasChildren'); * * The optional third parameter allows you to negate the operation and filter * out nonempty objects. To remove all objects that DO have children, do this: * * mfilter($list, 'hasChildren', true); * * @param array List of objects to filter. * @param string A method name. * @param bool Optionally, pass true to drop objects which pass the * filter instead of keeping them. * @return array List of objects which pass the filter. */ function mfilter(array $list, $method, $negate = false) { if (!is_string($method)) { throw new InvalidArgumentException(pht('Argument method is not a string.')); } $result = array(); foreach ($list as $key => $object) { $value = $object->$method(); if (!$negate) { if (!empty($value)) { $result[$key] = $object; } } else { if (empty($value)) { $result[$key] = $object; } } } return $result; } /** * Filter a list of arrays by removing the ones with an empty() value for some * index. This function works just like @{function:mfilter}, except that it * operates on a list of arrays instead of a list of objects. * * For example, to remove all arrays without value for key 'username', do this: * * ifilter($list, 'username'); * * The optional third parameter allows you to negate the operation and filter * out nonempty arrays. To remove all arrays that DO have value for key * 'username', do this: * * ifilter($list, 'username', true); * * @param array List of arrays to filter. * @param scalar The index. * @param bool Optionally, pass true to drop arrays which pass the * filter instead of keeping them. * @return array List of arrays which pass the filter. */ function ifilter(array $list, $index, $negate = false) { if (!is_scalar($index)) { throw new InvalidArgumentException(pht('Argument index is not a scalar.')); } $result = array(); if (!$negate) { foreach ($list as $key => $array) { if (!empty($array[$index])) { $result[$key] = $array; } } } else { foreach ($list as $key => $array) { if (empty($array[$index])) { $result[$key] = $array; } } } return $result; } /** * Selects a list of keys from an array, returning a new array with only the * key-value pairs identified by the selected keys, in the specified order. * * Note that since this function orders keys in the result according to the * order they appear in the list of keys, there are effectively two common * uses: either reducing a large dictionary to a smaller one, or changing the * key order on an existing dictionary. * * @param dict Dictionary of key-value pairs to select from. * @param list List of keys to select. * @return dict Dictionary of only those key-value pairs where the key was * present in the list of keys to select. Ordering is * determined by the list order. */ function array_select_keys(array $dict, array $keys) { $result = array(); foreach ($keys as $key) { if (array_key_exists($key, $dict)) { $result[$key] = $dict[$key]; } } return $result; } /** * Checks if all values of array are instances of the passed class. Throws * `InvalidArgumentException` if it isn't true for any value. * * @param array * @param string Name of the class or 'array' to check arrays. * @return array Returns passed array. */ function assert_instances_of(array $arr, $class) { $is_array = !strcasecmp($class, 'array'); foreach ($arr as $key => $object) { if ($is_array) { if (!is_array($object)) { $given = gettype($object); throw new InvalidArgumentException( pht( "Array item with key '%s' must be of type array, %s given.", $key, $given)); } } else if (!($object instanceof $class)) { $given = gettype($object); if (is_object($object)) { $given = pht('instance of %s', get_class($object)); } throw new InvalidArgumentException( pht( "Array item with key '%s' must be an instance of %s, %s given.", $key, $class, $given)); } } return $arr; } /** * Assert that passed data can be converted to string. * * @param string Assert that this data is valid. * @return void * * @task assert */ function assert_stringlike($parameter) { switch (gettype($parameter)) { case 'string': case 'NULL': case 'boolean': case 'double': case 'integer': return; case 'object': if (method_exists($parameter, '__toString')) { return; } break; case 'array': case 'resource': case 'unknown type': default: break; } throw new InvalidArgumentException( pht( 'Argument must be scalar or object which implements %s!', '__toString()')); } /** * Returns the first argument which is not strictly null, or `null` if there * are no such arguments. Identical to the MySQL function of the same name. * * @param ... Zero or more arguments of any type. * @return mixed First non-`null` arg, or null if no such arg exists. */ function coalesce(/* ... */) { $args = func_get_args(); foreach ($args as $arg) { if ($arg !== null) { return $arg; } } return null; } /** * Similar to @{function:coalesce}, but less strict: returns the first * non-`empty()` argument, instead of the first argument that is strictly * non-`null`. If no argument is nonempty, it returns the last argument. This * is useful idiomatically for setting defaults: * * $display_name = nonempty($user_name, $full_name, "Anonymous"); * * @param ... Zero or more arguments of any type. * @return mixed First non-`empty()` arg, or last arg if no such arg * exists, or null if you passed in zero args. */ function nonempty(/* ... */) { $args = func_get_args(); $result = null; foreach ($args as $arg) { $result = $arg; if ($arg) { break; } } return $result; } /** * Invokes the "new" operator with a vector of arguments. There is no way to * `call_user_func_array()` on a class constructor, so you can instead use this * function: * * $obj = newv($class_name, $argv); * * That is, these two statements are equivalent: * * $pancake = new Pancake('Blueberry', 'Maple Syrup', true); * $pancake = newv('Pancake', array('Blueberry', 'Maple Syrup', true)); * * DO NOT solve this problem in other, more creative ways! Three popular * alternatives are: * * - Build a fake serialized object and unserialize it. * - Invoke the constructor twice. * - just use `eval()` lol * * These are really bad solutions to the problem because they can have side * effects (e.g., __wakeup()) and give you an object in an otherwise impossible * state. Please endeavor to keep your objects in possible states. * * If you own the classes you're doing this for, you should consider whether * or not restructuring your code (for instance, by creating static * construction methods) might make it cleaner before using `newv()`. Static * constructors can be invoked with `call_user_func_array()`, and may give your * class a cleaner and more descriptive API. * * @param string The name of a class. * @param list Array of arguments to pass to its constructor. * @return obj A new object of the specified class, constructed by passing * the argument vector to its constructor. */ function newv($class_name, array $argv) { $reflector = new ReflectionClass($class_name); if ($argv) { return $reflector->newInstanceArgs($argv); } else { return $reflector->newInstance(); } } /** * Returns the first element of an array. Exactly like reset(), but doesn't * choke if you pass it some non-referenceable value like the return value of * a function. * * @param array Array to retrieve the first element from. * @return wild The first value of the array. */ function head(array $arr) { return reset($arr); } /** * Returns the last element of an array. This is exactly like `end()` except * that it won't warn you if you pass some non-referencable array to * it -- e.g., the result of some other array operation. * * @param array Array to retrieve the last element from. * @return wild The last value of the array. */ function last(array $arr) { return end($arr); } /** * Returns the first key of an array. * * @param array Array to retrieve the first key from. * @return int|string The first key of the array. */ function head_key(array $arr) { reset($arr); return key($arr); } /** * Returns the last key of an array. * * @param array Array to retrieve the last key from. * @return int|string The last key of the array. */ function last_key(array $arr) { end($arr); return key($arr); } /** * Merge a vector of arrays performantly. This has the same semantics as * array_merge(), so these calls are equivalent: * * array_merge($a, $b, $c); * array_mergev(array($a, $b, $c)); * * However, when you have a vector of arrays, it is vastly more performant to * merge them with this function than by calling array_merge() in a loop, * because using a loop generates an intermediary array on each iteration. * * @param list Vector of arrays to merge. * @return list Arrays, merged with array_merge() semantics. */ function array_mergev(array $arrayv) { if (!$arrayv) { return array(); } foreach ($arrayv as $key => $item) { if (!is_array($item)) { throw new InvalidArgumentException( pht( - 'Expected all items passed to %s to be arrays, but '. + 'Expected all items passed to `%s` to be arrays, but '. 'argument with key "%s" has type "%s".', __FUNCTION__.'()', $key, gettype($item))); } } return call_user_func_array('array_merge', $arrayv); } /** * Split a corpus of text into lines. This function splits on "\n", "\r\n", or * a mixture of any of them. * * NOTE: This function does not treat "\r" on its own as a newline because none * of SVN, Git or Mercurial do on any OS. * * @param string Block of text to be split into lines. * @param bool If true, retain line endings in result strings. * @return list List of lines. */ function phutil_split_lines($corpus, $retain_endings = true) { if (!strlen($corpus)) { return array(''); } // Split on "\r\n" or "\n". if ($retain_endings) { $lines = preg_split('/(?<=\n)/', $corpus); } else { $lines = preg_split('/\r?\n/', $corpus); } // If the text ends with "\n" or similar, we'll end up with an empty string // at the end; discard it. if (end($lines) == '') { array_pop($lines); } if ($corpus instanceof PhutilSafeHTML) { return array_map('phutil_safe_html', $lines); } return $lines; } /** * Simplifies a common use of `array_combine()`. Specifically, this: * * COUNTEREXAMPLE: * if ($list) { * $result = array_combine($list, $list); * } else { * // Prior to PHP 5.4, array_combine() failed if given empty arrays. * $result = array(); * } * * ...is equivalent to this: * * $result = array_fuse($list); * * @param list List of scalars. * @return dict Dictionary with inputs mapped to themselves. */ function array_fuse(array $list) { if ($list) { return array_combine($list, $list); } return array(); } /** * Add an element between every two elements of some array. That is, given a * list `A, B, C, D`, and some element to interleave, `x`, this function returns * `A, x, B, x, C, x, D`. This works like `implode()`, but does not concatenate * the list into a string. In particular: * * implode('', array_interleave($x, $list)); * * ...is equivalent to: * * implode($x, $list); * * This function does not preserve keys. * * @param wild Element to interleave. * @param list List of elements to be interleaved. * @return list Original list with the new element interleaved. */ function array_interleave($interleave, array $array) { $result = array(); foreach ($array as $item) { $result[] = $item; $result[] = $interleave; } array_pop($result); return $result; } function phutil_is_windows() { // We can also use PHP_OS, but that's kind of sketchy because it returns // "WINNT" for Windows 7 and "Darwin" for Mac OS X. Practically, testing for // DIRECTORY_SEPARATOR is more straightforward. return (DIRECTORY_SEPARATOR != '/'); } function phutil_is_hiphop_runtime() { return (array_key_exists('HPHP', $_ENV) && $_ENV['HPHP'] === 1); } /** * Converts a string to a loggable one, with unprintables and newlines escaped. * * @param string Any string. * @return string String with control and newline characters escaped, suitable * for printing on a single log line. */ function phutil_loggable_string($string) { if (preg_match('/^[\x20-\x7E]+$/', $string)) { return $string; } $result = ''; static $c_map = array( '\\' => '\\\\', "\n" => '\\n', "\r" => '\\r', "\t" => '\\t', ); $len = strlen($string); for ($ii = 0; $ii < $len; $ii++) { $c = $string[$ii]; if (isset($c_map[$c])) { $result .= $c_map[$c]; } else { $o = ord($c); if ($o < 0x20 || $o == 0x7F) { $result .= '\\x'.sprintf('%02X', $o); } else { $result .= $c; } } } return $result; } /** * Perform an `fwrite()` which distinguishes between EAGAIN and EPIPE. * * PHP's `fwrite()` is broken, and never returns `false` for writes to broken * nonblocking pipes: it always returns 0, and provides no straightforward * mechanism for distinguishing between EAGAIN (buffer is full, can't write any * more right now) and EPIPE or similar (no write will ever succeed). * * See: https://bugs.php.net/bug.php?id=39598 * * If you call this method instead of `fwrite()`, it will attempt to detect * when a zero-length write is caused by EAGAIN and return `0` only if the * write really should be retried. * * @param resource Socket or pipe stream. * @param string Bytes to write. * @return bool|int Number of bytes written, or `false` on any error (including * errors which `fpipe()` can not detect, like a broken pipe). */ function phutil_fwrite_nonblocking_stream($stream, $bytes) { if (!strlen($bytes)) { return 0; } $result = @fwrite($stream, $bytes); if ($result !== 0) { // In cases where some bytes are witten (`$result > 0`) or // an error occurs (`$result === false`), the behavior of fwrite() is // correct. We can return the value as-is. return $result; } // If we make it here, we performed a 0-length write. Try to distinguish // between EAGAIN and EPIPE. To do this, we're going to `stream_select()` // the stream, write to it again if PHP claims that it's writable, and // consider the pipe broken if the write fails. $read = array(); $write = array($stream); $except = array(); @stream_select($read, $write, $except, 0); if (!$write) { // The stream isn't writable, so we conclude that it probably really is // blocked and the underlying error was EAGAIN. Return 0 to indicate that // no data could be written yet. return 0; } // If we make it here, PHP **just** claimed that this stream is writable, so // perform a write. If the write also fails, conclude that these failures are // EPIPE or some other permanent failure. $result = @fwrite($stream, $bytes); if ($result !== 0) { // The write worked or failed explicitly. This value is fine to return. return $result; } // We performed a 0-length write, were told that the stream was writable, and // then immediately performed another 0-length write. Conclude that the pipe // is broken and return `false`. return false; } /** * Convert a human-readable unit description into a numeric one. This function * allows you to replace this: * * COUNTEREXAMPLE * $ttl = (60 * 60 * 24 * 30); // 30 days * * ...with this: * * $ttl = phutil_units('30 days in seconds'); * * ...which is self-documenting and difficult to make a mistake with. * * @param string Human readable description of a unit quantity. * @return int Quantity of specified unit. */ function phutil_units($description) { $matches = null; if (!preg_match('/^(\d+) (\w+) in (\w+)$/', $description, $matches)) { throw new InvalidArgumentException( pht( 'Unable to parse unit specification (expected a specification in the '. 'form "%s"): %s', '5 days in seconds', $description)); } $quantity = (int)$matches[1]; $src_unit = $matches[2]; $dst_unit = $matches[3]; switch ($dst_unit) { case 'seconds': switch ($src_unit) { case 'second': case 'seconds': $factor = 1; break; case 'minute': case 'minutes': $factor = 60; break; case 'hour': case 'hours': $factor = 60 * 60; break; case 'day': case 'days': $factor = 60 * 60 * 24; break; default: throw new InvalidArgumentException( pht( 'This function can not convert from the unit "%s".', $src_unit)); } break; default: throw new InvalidArgumentException( pht( 'This function can not convert into the unit "%s".', $dst_unit)); } return $quantity * $factor; } /** * Decode a JSON dictionary. * * @param string A string which ostensibly contains a JSON-encoded list or * dictionary. * @return mixed Decoded list/dictionary. */ function phutil_json_decode($string) { $result = @json_decode($string, true); if (!is_array($result)) { // Failed to decode the JSON. Try to use @{class:PhutilJSONParser} instead. // This will probably fail, but will throw a useful exception. $parser = new PhutilJSONParser(); $result = $parser->parse($string); } return $result; } /** * Encode a value in JSON, raising an exception if it can not be encoded. * * @param wild A value to encode. * @return string JSON representation of the value. */ function phutil_json_encode($value) { $result = @json_encode($value); if ($result === false) { $reason = phutil_validate_json($value); if (function_exists('json_last_error')) { $err = json_last_error(); if (function_exists('json_last_error_msg')) { $msg = json_last_error_msg(); $extra = pht('#%d: %s', $err, $msg); } else { $extra = pht('#%d', $err); } } else { $extra = null; } if ($extra) { $message = pht( 'Failed to JSON encode value (%s): %s.', $extra, $reason); } else { $message = pht( 'Failed to JSON encode value: %s.', $reason); } throw new Exception($message); } return $result; } /** * Produce a human-readable explanation why a value can not be JSON-encoded. * * @param wild Value to validate. * @param string Path within the object to provide context. * @return string|null Explanation of why it can't be encoded, or null. */ function phutil_validate_json($value, $path = '') { if ($value === null) { return; } if ($value === true) { return; } if ($value === false) { return; } if (is_int($value)) { return; } if (is_float($value)) { return; } if (is_array($value)) { foreach ($value as $key => $subvalue) { if (strlen($path)) { $full_key = $path.' > '; } else { $full_key = ''; } if (!phutil_is_utf8($key)) { $full_key = $full_key.phutil_utf8ize($key); return pht( - 'Dictionary key "%s" is not valid UTF8, and can not be JSON encoded.', + 'Dictionary key "%s" is not valid UTF8, and cannot be JSON encoded.', $full_key); } $full_key .= $key; $result = phutil_validate_json($subvalue, $full_key); if ($result !== null) { return $result; } } } if (is_string($value)) { if (!phutil_is_utf8($value)) { $display = substr($value, 0, 256); $display = phutil_utf8ize($display); if (!strlen($path)) { return pht( 'String value is not valid UTF8, and can not be JSON encoded: %s', $display); } else { return pht( - 'Dictionary value at key "%s" is not valid UTF8, and can not be '. + 'Dictionary value at key "%s" is not valid UTF8, and cannot be '. 'JSON encoded: %s', $path, $display); } } } return; } /** * Decode an INI string. * * @param string * @return mixed */ function phutil_ini_decode($string) { $results = null; $trap = new PhutilErrorTrap(); try { if (!function_exists('parse_ini_string')) { throw new PhutilMethodNotImplementedException( pht( '%s is not compatible with your version of PHP (%s). This function '. 'is only supported on PHP versions newer than 5.3.0.', __FUNCTION__, phpversion())); } $results = @parse_ini_string($string, true, INI_SCANNER_RAW); if ($results === false) { throw new PhutilINIParserException(trim($trap->getErrorsAsString())); } foreach ($results as $section => $result) { if (!is_array($result)) { // We JSON decode the value in ordering to perform the following // conversions: // // - `'true'` => `true` // - `'false'` => `false` // - `'123'` => `123` // - `'1.234'` => `1.234` // $result = json_decode($result, true); if ($result !== null && !is_array($result)) { $results[$section] = $result; } continue; } foreach ($result as $key => $value) { $value = json_decode($value, true); if ($value !== null && !is_array($value)) { $results[$section][$key] = $value; } } } } catch (Exception $ex) { $trap->destroy(); throw $ex; } $trap->destroy(); return $results; } /** * Attempt to censor any plaintext credentials from a string. * * The major use case here is to censor usernames and passwords from command * output. For example, when `git fetch` fails, the output includes credentials * for authenticated HTTP remotes. * * @param string Some block of text. * @return string A similar block of text, but with credentials that could * be identified censored. */ function phutil_censor_credentials($string) { return preg_replace(',(?<=://)([^/@\s]+)(?=@|$),', 'xxxxx', $string); } /** * Returns a parsable string representation of a variable. * * This function is intended to behave similarly to PHP's `var_export` function, * but the output is intended to follow our style conventions. * * @param wild The variable you want to export. * @return string */ function phutil_var_export($var) { // `var_export(null, true)` returns `"NULL"` (in uppercase). if ($var === null) { return 'null'; } // PHP's `var_export` doesn't format arrays very nicely. In particular: // // - An empty array is split over two lines (`"array (\n)"`). // - A space separates "array" and the first opening brace. // - Non-associative arrays are returned as associative arrays with an // integer key. // if (is_array($var)) { if (count($var) === 0) { return 'array()'; } // Don't show keys for non-associative arrays. $show_keys = (array_keys($var) !== range(0, count($var) - 1)); $output = array(); $output[] = 'array('; foreach ($var as $key => $value) { // Adjust the indentation of the value. $value = str_replace("\n", "\n ", phutil_var_export($value)); $output[] = ' '. ($show_keys ? var_export($key, true).' => ' : ''). $value.','; } $output[] = ')'; return implode("\n", $output); } // Let PHP handle everything else. return var_export($var, true); } /** * An improved version of `fnmatch`. * * @param string A glob pattern. * @param string A path. * @return bool */ function phutil_fnmatch($glob, $path) { // Modify the glob to allow `**/` to match files in the root directory. $glob = preg_replace('@(?:(?