diff --git a/src/configuration/ArcanistConfigurationManager.php b/src/configuration/ArcanistConfigurationManager.php index f1391a8b..68ebde4e 100644 --- a/src/configuration/ArcanistConfigurationManager.php +++ b/src/configuration/ArcanistConfigurationManager.php @@ -1,362 +1,362 @@ workingCopy = $working_copy; return $this; } /* -( Get config )--------------------------------------------------------- */ const CONFIG_SOURCE_RUNTIME = 'runtime'; const CONFIG_SOURCE_LOCAL = 'local'; const CONFIG_SOURCE_PROJECT = 'project'; const CONFIG_SOURCE_USER = 'user'; const CONFIG_SOURCE_SYSTEM = 'system'; const CONFIG_SOURCE_DEFAULT = 'default'; public function getProjectConfig($key) { if ($this->workingCopy) { return $this->workingCopy->getProjectConfig($key); } return null; } public function getLocalConfig($key) { if ($this->workingCopy) { return $this->workingCopy->getLocalConfig($key); } return null; } public function getWorkingCopyIdentity() { return $this->workingCopy; } /** * Read a configuration directive from any available configuration source. * This includes the directive in local, user and system configuration in * addition to project configuration, and configuration provided as command * arguments ("runtime"). * The precedence is runtime > local > project > user > system * * @param key Key to read. * @param wild Default value if key is not found. * @return wild Value, or default value if not found. * * @task config */ public function getConfigFromAnySource($key, $default = null) { $all = $this->getConfigFromAllSources($key); return empty($all) ? $default : head($all); } /** * For the advanced case where you want customized configuration handling. * * Reads the configuration from all available sources, returning a map (array) * of results, with the source as key. Missing values will not be in the map, * so an empty array will be returned if no results are found. * * The map is ordered by the canonical sources precedence, which is: * runtime > local > project > user > system * * @param key Key to read * @return array Mapping of source => value read. Sources with no value are * not in the array. * * @task config */ public function getConfigFromAllSources($key) { $results = array(); $settings = new ArcanistSettings(); $pval = idx($this->runtimeConfig, $key); if ($pval !== null) { $results[self::CONFIG_SOURCE_RUNTIME] = $settings->willReadValue($key, $pval); } $pval = $this->getLocalConfig($key); if ($pval !== null) { $results[self::CONFIG_SOURCE_LOCAL] = $settings->willReadValue($key, $pval); } $pval = $this->getProjectConfig($key); if ($pval !== null) { $results[self::CONFIG_SOURCE_PROJECT] = $settings->willReadValue($key, $pval); } $user_config = $this->readUserArcConfig(); // For "aliases" coming from the user config file specifically, read the // top level "aliases" key instead of the "aliases" key inside the "config" // setting. Aliases were originally user-specific but later became standard // configuration, which is why this works oddly. if ($key === 'aliases') { $pval = idx($this->readUserConfigurationFile(), $key); } else { $pval = idx($user_config, $key); } if ($pval !== null) { $results[self::CONFIG_SOURCE_USER] = $settings->willReadValue($key, $pval); } $system_config = $this->readSystemArcConfig(); $pval = idx($system_config, $key); if ($pval !== null) { $results[self::CONFIG_SOURCE_SYSTEM] = $settings->willReadValue($key, $pval); } $default_config = $this->readDefaultConfig(); if (array_key_exists($key, $default_config)) { $results[self::CONFIG_SOURCE_DEFAULT] = $default_config[$key]; } return $results; } /** * Sets a runtime config value that takes precedence over any static * config values. * * @param key Key to set. * @param value The value of the key. * * @task config */ public function setRuntimeConfig($key, $value) { $this->runtimeConfig[$key] = $value; return $this; } /* -( Read/write config )--------------------------------------------------- */ public function readLocalArcConfig() { if ($this->workingCopy) { return $this->workingCopy->readLocalArcConfig(); } return array(); } public function writeLocalArcConfig(array $config) { if ($this->workingCopy) { return $this->workingCopy->writeLocalArcConfig($config); } throw new Exception(pht('No working copy to write config to!')); } /** * This is probably not the method you're looking for; try * @{method:readUserArcConfig}. */ public function readUserConfigurationFile() { if ($this->userConfigCache === null) { $user_config = array(); $user_config_path = $this->getUserConfigurationFileLocation(); $console = PhutilConsole::getConsole(); if (Filesystem::pathExists($user_config_path)) { $console->writeLog( "%s\n", pht( 'Config: Reading user configuration file "%s"...', $user_config_path)); if (!phutil_is_windows()) { $mode = fileperms($user_config_path); if (!$mode) { throw new Exception( pht( 'Unable to read file permissions for "%s"!', $user_config_path)); } if ($mode & 0177) { // Mode should allow only owner access. $prompt = pht( "File permissions on your %s are too open. ". "Fix them by chmod'ing to 600?", '~/.arcrc'); if (!phutil_console_confirm($prompt, $default_no = false)) { throw new ArcanistUsageException( pht('Set %s to file mode 600.', '~/.arcrc')); } execx('chmod 600 %s', $user_config_path); // Drop the stat cache so we don't read the old permissions if // we end up here again. If we don't do this, we may prompt the user // to fix permissions multiple times. clearstatcache(); } } $user_config_data = Filesystem::readFile($user_config_path); try { $user_config = phutil_json_decode($user_config_data); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht("Your '%s' file is not a valid JSON file.", '~/.arcrc'), $ex); } } else { $console->writeLog( "%s\n", pht( 'Config: Did not find user configuration at "%s".', $user_config_path)); } $this->userConfigCache = $user_config; } return $this->userConfigCache; } /** * This is probably not the method you're looking for; try * @{method:writeUserArcConfig}. */ public function writeUserConfigurationFile($config) { $json_encoder = new PhutilJSON(); $json = $json_encoder->encodeFormatted($config); $path = $this->getUserConfigurationFileLocation(); Filesystem::writeFile($path, $json); if (!phutil_is_windows()) { execx('chmod 600 %s', $path); } } public function setUserConfigurationFileLocation($custom_arcrc) { if (!Filesystem::pathExists($custom_arcrc)) { throw new Exception( pht('Custom %s file was specified, but it was not found!', 'arcrc')); } $this->customArcrcFilename = $custom_arcrc; $this->userConfigCache = null; return $this; } public function getUserConfigurationFileLocation() { - if (strlen($this->customArcrcFilename)) { + if ($this->customArcrcFilename !== null) { return $this->customArcrcFilename; } if (phutil_is_windows()) { return getenv('APPDATA').'/.arcrc'; } else { return getenv('HOME').'/.arcrc'; } } public function readUserArcConfig() { $config = $this->readUserConfigurationFile(); if (isset($config['config'])) { $config = $config['config']; } return $config; } public function writeUserArcConfig(array $options) { $config = $this->readUserConfigurationFile(); $config['config'] = $options; $this->writeUserConfigurationFile($config); } public function getSystemArcConfigLocation() { if (phutil_is_windows()) { return Filesystem::resolvePath( 'Phabricator/Arcanist/config', getenv('ProgramData')); } else { return '/etc/arcconfig'; } } public function readSystemArcConfig() { static $system_config; if ($system_config === null) { $system_config = array(); $system_config_path = $this->getSystemArcConfigLocation(); $console = PhutilConsole::getConsole(); if (Filesystem::pathExists($system_config_path)) { $console->writeLog( "%s\n", pht( 'Config: Reading system configuration file "%s"...', $system_config_path)); $file = Filesystem::readFile($system_config_path); try { $system_config = phutil_json_decode($file); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht( "Your '%s' file is not a valid JSON file.", $system_config_path), $ex); } } else { $console->writeLog( "%s\n", pht( 'Config: Did not find system configuration at "%s".', $system_config_path)); } } return $system_config; } public function applyRuntimeArcConfig($args) { $arcanist_settings = new ArcanistSettings(); $options = $args->getArg('config'); foreach ($options as $opt) { $opt_config = preg_split('/=/', $opt, 2); if (count($opt_config) !== 2) { throw new ArcanistUsageException( pht( "Argument was '%s', but must be '%s'. For example, %s", $opt, 'name=value', 'history.immutable=true')); } list($key, $value) = $opt_config; $value = $arcanist_settings->willWriteValue($key, $value); $this->setRuntimeConfig($key, $value); } return $this->runtimeConfig; } public function readDefaultConfig() { $settings = new ArcanistSettings(); return $settings->getDefaultSettings(); } } diff --git a/src/filesystem/PhutilDirectoryFixture.php b/src/filesystem/PhutilDirectoryFixture.php index 8eb4c14a..5564b7bb 100644 --- a/src/filesystem/PhutilDirectoryFixture.php +++ b/src/filesystem/PhutilDirectoryFixture.php @@ -1,50 +1,52 @@ getPath(), Filesystem::resolvePath($archive)); return $obj; } public static function newEmptyFixture() { $obj = new PhutilDirectoryFixture(); $obj->path = Filesystem::createTemporaryDirectory(); return $obj; } private function __construct() { // } public function __destruct() { Filesystem::remove($this->path); } public function getPath($to_file = null) { + $to_file = phutil_string_cast($to_file); + return $this->path.'/'.ltrim($to_file, '/'); } public function saveToArchive($path) { $tmp = new TempFile(); execx( 'tar -C %s -czvvf %s .', $this->getPath(), $tmp); $ok = rename($tmp, Filesystem::resolvePath($path)); if (!$ok) { throw new FilesystemException($path, pht('Failed to overwrite file.')); } return $this; } } diff --git a/src/future/exec/ExecFuture.php b/src/future/exec/ExecFuture.php index 7aadd61a..ddf9e515 100644 --- a/src/future/exec/ExecFuture.php +++ b/src/future/exec/ExecFuture.php @@ -1,1036 +1,1050 @@ array('pipe', 'r'), // stdin 1 => array('pipe', 'w'), // stdout 2 => array('pipe', 'w'), // stderr ); protected function didConstruct() { $this->stdin = new PhutilRope(); } /* -( Command Information )------------------------------------------------ */ /** * Retrieve the byte limit for the stderr buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStderrSizeLimit() { return $this->stderrSizeLimit; } /** * Retrieve the byte limit for the stdout buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStdoutSizeLimit() { return $this->stdoutSizeLimit; } /** * Get the process's pid. This only works after execution is initiated, e.g. * by a call to start(). * * @return int Process ID of the executing process. * @task info */ public function getPID() { $status = $this->procGetStatus(); return $status['pid']; } public function hasPID() { if ($this->procStatus) { return true; } if ($this->proc) { return true; } return false; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a maximum size for the stdout read buffer. To limit stderr, see * @{method:setStderrSizeLimit}. The major use of these methods is to use less * memory if you are running a command which sometimes produces huge volumes * of output that you don't really care about. * * NOTE: Setting this to 0 means "no buffer", not "unlimited buffer". * * @param int Maximum size of the stdout read buffer. * @return this * @task config */ public function setStdoutSizeLimit($limit) { $this->stdoutSizeLimit = $limit; return $this; } /** * Set a maximum size for the stderr read buffer. * See @{method:setStdoutSizeLimit} for discussion. * * @param int Maximum size of the stderr read buffer. * @return this * @task config */ public function setStderrSizeLimit($limit) { $this->stderrSizeLimit = $limit; return $this; } /** * Set the maximum internal read buffer size this future. The future will * block reads once the internal stdout or stderr buffer exceeds this size. * * NOTE: If you @{method:resolve} a future with a read buffer limit, you may * block forever! * * TODO: We should probably release the read buffer limit during * @{method:resolve}, or otherwise detect this. For now, be careful. * * @param int|null Maximum buffer size, or `null` for unlimited. * @return this */ public function setReadBufferSize($read_buffer_size) { $this->readBufferSize = $read_buffer_size; return $this; } /* -( Interacting With Commands )------------------------------------------ */ /** * Read and return output from stdout and stderr, if any is available. This * method keeps a read cursor on each stream, but the entire streams are * still returned when the future resolves. You can call read() again after * resolving the future to retrieve only the parts of the streams you did not * previously read: * * $future = new ExecFuture('...'); * // ... * list($stdout) = $future->read(); // Returns output so far * list($stdout) = $future->read(); // Returns new output since first call * // ... * list($stdout) = $future->resolvex(); // Returns ALL output * list($stdout) = $future->read(); // Returns unread output * * NOTE: If you set a limit with @{method:setStdoutSizeLimit} or * @{method:setStderrSizeLimit}, this method will not be able to read data * past the limit. * * NOTE: If you call @{method:discardBuffers}, all the stdout/stderr data * will be thrown away and the cursors will be reset. * * @return pair <$stdout, $stderr> pair with new output since the last call * to this method. * @task interact */ public function read() { - $stdout = $this->readStdout(); + $stdout_value = $this->readStdout(); + + $stderr = $this->stderr; + if ($stderr === null) { + $stderr_value = ''; + } else { + $stderr_value = substr($stderr, $this->stderrPos); + } $result = array( - $stdout, - (string)substr($this->stderr, $this->stderrPos), + $stdout_value, + $stderr_value, ); $this->stderrPos = $this->getStderrBufferLength(); return $result; } public function readStdout() { if ($this->start) { $this->updateFuture(); // Sync } - $result = (string)substr($this->stdout, $this->stdoutPos); + $stdout = $this->stdout; + + if ($stdout === null) { + $result = ''; + } else { + $result = substr($stdout, $this->stdoutPos); + } + $this->stdoutPos = $this->getStdoutBufferLength(); return $result; } /** * Write data to stdin of the command. * * @param string Data to write. * @param bool If true, keep the pipe open for writing. By default, the pipe * will be closed as soon as possible so that commands which * listen for EOF will execute. If you want to keep the pipe open * past the start of command execution, do an empty write with * `$keep_pipe = true` first. * @return this * @task interact */ public function write($data, $keep_pipe = false) { if (strlen($data)) { if (!$this->stdin) { throw new Exception(pht('Writing to a closed pipe!')); } $this->stdin->append($data); } $this->closePipe = !$keep_pipe; return $this; } /** * Permanently discard the stdout and stderr buffers and reset the read * cursors. This is basically useful only if you are streaming a large amount * of data from some process. * * Conceivably you might also need to do this if you're writing a client using * @{class:ExecFuture} and `netcat`, but you probably should not do that. * * NOTE: This completely discards the data. It won't be available when the * future resolves. This is almost certainly only useful if you need the * buffer memory for some reason. * * @return this * @task interact */ public function discardBuffers() { $this->discardStdoutBuffer(); $this->stderr = ''; $this->stderrPos = 0; return $this; } public function discardStdoutBuffer() { $this->stdout = ''; $this->stdoutPos = 0; return $this; } /** * Returns true if this future was killed by a timeout configured with * @{method:setTimeout}. * * @return bool True if the future was killed for exceeding its time limit. */ public function getWasKilledByTimeout() { return $this->killedByTimeout; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a hard limit on execution time. If the command runs longer, it will * be terminated and the future will resolve with an error code. You can test * if a future was killed by a timeout with @{method:getWasKilledByTimeout}. * * The subprocess will be sent a `TERM` signal, and then a `KILL` signal a * short while later if it fails to exit. * * @param int Maximum number of seconds this command may execute for before * it is signaled. * @return this * @task config */ public function setTimeout($seconds) { $this->terminateTimeout = $seconds; $this->killTimeout = $seconds + min($seconds, 60); return $this; } /* -( Resolving Execution )------------------------------------------------ */ /** * Resolve a command you expect to exit with return code 0. Works like * @{method:resolve}, but throws if $err is nonempty. Returns only * $stdout and $stderr. See also @{function:execx}. * * list($stdout, $stderr) = $future->resolvex(); * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return pair <$stdout, $stderr> pair. * @task resolve */ public function resolvex() { $result = $this->resolve(); return $this->raiseResultError($result); } /** * Resolve a command you expect to return valid JSON. Works like * @{method:resolvex}, but also throws if stderr is nonempty, or stdout is not * valid JSON. Returns a PHP array, decoded from the JSON command output. * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return array PHP array, decoded from JSON command output. * @task resolve */ public function resolveJSON() { list($stdout, $stderr) = $this->resolvex(); if (strlen($stderr)) { $cmd = $this->getCommand(); throw new CommandException( pht( "JSON command '%s' emitted text to stderr when none was expected: %d", $cmd, $stderr), $cmd, 0, $stdout, $stderr); } try { return phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { $cmd = $this->getCommand(); throw new CommandException( pht( "JSON command '%s' did not produce a valid JSON object on stdout: %s", $cmd, $stdout), $cmd, 0, $stdout, $stderr); } } /** * Resolve the process by abruptly terminating it. * * @return list List of results. * @task resolve */ public function resolveKill() { if (!$this->hasResult()) { $signal = 9; if ($this->proc) { proc_terminate($this->proc, $signal); } $this->closeProcess(); $result = array( 128 + $signal, $this->stdout, $this->stderr, ); $this->recordResult($result); } return $this->getResult(); } private function recordResult(array $result) { $resolve_on_error = $this->getResolveOnError(); if (!$resolve_on_error) { $result = $this->raiseResultError($result); } $this->setResult($result); } private function raiseResultError($result) { list($err, $stdout, $stderr) = $result; if ($err) { $cmd = $this->getCommand(); if ($this->getWasKilledByTimeout()) { // NOTE: The timeout can be a float and PhutilNumber only handles // integers, so just use "%s" to render it. $message = pht( 'Command killed by timeout after running for more than %s seconds.', $this->terminateTimeout); } else { $message = pht('Command failed with error #%d!', $err); } throw new CommandException( $message, $cmd, $err, $stdout, $stderr); } return array($stdout, $stderr); } /* -( Internals )---------------------------------------------------------- */ /** * Provides read sockets to the future core. * * @return list List of read sockets. * @task internal */ public function getReadSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdout) && !feof($stdout)) { $sockets[] = $stdout; } if (isset($stderr) && !feof($stderr)) { $sockets[] = $stderr; } return $sockets; } /** * Provides write sockets to the future core. * * @return list List of write sockets. * @task internal */ public function getWriteSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdin) && $this->stdin->getByteLength() && !feof($stdin)) { $sockets[] = $stdin; } return $sockets; } /** * Determine if the read buffer is empty. * * @return bool True if the read buffer is empty. * @task internal */ public function isReadBufferEmpty() { return !$this->getStdoutBufferLength(); } /** * Determine if the write buffer is empty. * * @return bool True if the write buffer is empty. * @task internal */ public function isWriteBufferEmpty() { return !$this->getWriteBufferSize(); } /** * Determine the number of bytes in the write buffer. * * @return int Number of bytes in the write buffer. * @task internal */ public function getWriteBufferSize() { if (!$this->stdin) { return 0; } return $this->stdin->getByteLength(); } /** * Reads some bytes from a stream, discarding output once a certain amount * has been accumulated. * * @param resource Stream to read from. * @param int Maximum number of bytes to return from $stream. If * additional bytes are available, they will be read and * discarded. * @param string Human-readable description of stream, for exception * message. * @param int Maximum number of bytes to read. * @return string The data read from the stream. * @task internal */ private function readAndDiscard($stream, $limit, $description, $length) { $output = ''; if ($length <= 0) { return ''; } do { $data = fread($stream, min($length, 64 * 1024)); if (false === $data) { throw new Exception(pht('Failed to read from %s', $description)); } $read_bytes = strlen($data); if ($read_bytes > 0 && $limit > 0) { if ($read_bytes > $limit) { $data = substr($data, 0, $limit); } $output .= $data; $limit -= strlen($data); } if (strlen($output) >= $length) { break; } } while ($read_bytes > 0); return $output; } /** * Begin or continue command execution. * * @return bool True if future has resolved. * @task internal */ public function isReady() { // NOTE: We have a soft dependencies on PhutilErrorTrap here, to avoid // the need to build it into the Phage agent. Under normal circumstances, // this class are always available. if (!$this->pipes) { $is_windows = phutil_is_windows(); if (!$this->start) { // We might already have started the timer via initiating resolution. $this->start = microtime(true); } $unmasked_command = $this->getCommand(); $unmasked_command = $unmasked_command->getUnmaskedString(); $pipes = array(); if ($this->hasEnv()) { $env = $this->getEnv(); } else { $env = null; } $cwd = $this->getCWD(); // NOTE: See note above about Phage. if (class_exists('PhutilErrorTrap')) { $trap = new PhutilErrorTrap(); } else { $trap = null; } $spec = self::$descriptorSpec; if ($is_windows) { $stdout_file = new TempFile(); $stderr_file = new TempFile(); $stdout_handle = fopen($stdout_file, 'wb'); if (!$stdout_handle) { throw new Exception( pht( 'Unable to open stdout temporary file ("%s") for writing.', $stdout_file)); } $stderr_handle = fopen($stderr_file, 'wb'); if (!$stderr_handle) { throw new Exception( pht( 'Unable to open stderr temporary file ("%s") for writing.', $stderr_file)); } $spec = array( 0 => self::$descriptorSpec[0], 1 => $stdout_handle, 2 => $stderr_handle, ); } $proc = @proc_open( $unmasked_command, $spec, $pipes, $cwd, $env, array( 'bypass_shell' => true, )); if ($trap) { $err = $trap->getErrorsAsString(); $trap->destroy(); } else { $err = error_get_last(); if ($err) { $err = $err['message']; } } if ($is_windows) { fclose($stdout_handle); fclose($stderr_handle); } if (!is_resource($proc)) { // When you run an invalid command on a Linux system, the "proc_open()" // works and then the process (really a "/bin/sh -c ...") exits after // it fails to resolve the command. // When you run an invalid command on a Windows system, we bypass the // shell and the "proc_open()" itself fails. See also T13504. Fail the // future immediately, acting as though it exited with an error code // for consistency with Linux. $result = array( 1, '', pht( 'Call to "proc_open()" to open a subprocess failed: %s', $err), ); $this->recordResult($result); return true; } if ($is_windows) { $stdout_handle = fopen($stdout_file, 'rb'); if (!$stdout_handle) { throw new Exception( pht( 'Unable to open stdout temporary file ("%s") for reading.', $stdout_file)); } $stderr_handle = fopen($stderr_file, 'rb'); if (!$stderr_handle) { throw new Exception( pht( 'Unable to open stderr temporary file ("%s") for reading.', $stderr_file)); } $pipes = array( 0 => $pipes[0], 1 => $stdout_handle, 2 => $stderr_handle, ); $this->windowsStdoutTempFile = $stdout_file; $this->windowsStderrTempFile = $stderr_file; } $this->pipes = $pipes; $this->proc = $proc; list($stdin, $stdout, $stderr) = $pipes; if (!$is_windows) { // On Windows, we redirect process standard output and standard error // through temporary files. Files don't block, so we don't need to make // these streams nonblocking. if ((!stream_set_blocking($stdout, false)) || (!stream_set_blocking($stderr, false)) || (!stream_set_blocking($stdin, false))) { $this->__destruct(); throw new Exception(pht('Failed to set streams nonblocking.')); } } $this->tryToCloseStdin(); return false; } if (!$this->proc) { return true; } list($stdin, $stdout, $stderr) = $this->pipes; while (isset($this->stdin) && $this->stdin->getByteLength()) { $write_segment = $this->stdin->getAnyPrefix(); try { $bytes = fwrite($stdin, $write_segment); } catch (RuntimeException $ex) { // If the subprocess has exited, we may get a broken pipe error here // in recent versions of PHP. There does not seem to be any way to // get the actual error code other than reading the exception string. // For now, treat this as if writes are blocked. break; } if ($bytes === false) { throw new Exception(pht('Unable to write to stdin!')); } else if ($bytes) { $this->stdin->removeBytesFromHead($bytes); } else { // Writes are blocked for now. break; } } $this->tryToCloseStdin(); // Read status before reading pipes so that we can never miss data that // arrives between our last read and the process exiting. $status = $this->procGetStatus(); $read_buffer_size = $this->readBufferSize; $max_stdout_read_bytes = PHP_INT_MAX; $max_stderr_read_bytes = PHP_INT_MAX; if ($read_buffer_size !== null) { $stdout_len = $this->getStdoutBufferLength(); $stderr_len = $this->getStderrBufferLength(); $max_stdout_read_bytes = $read_buffer_size - $stdout_len; $max_stderr_read_bytes = $read_buffer_size - $stderr_len; } if ($max_stdout_read_bytes > 0) { $this->stdout .= $this->readAndDiscard( $stdout, $this->getStdoutSizeLimit() - $this->getStdoutBufferLength(), 'stdout', $max_stdout_read_bytes); } if ($max_stderr_read_bytes > 0) { $this->stderr .= $this->readAndDiscard( $stderr, $this->getStderrSizeLimit() - $this->getStderrBufferLength(), 'stderr', $max_stderr_read_bytes); } $is_done = false; if (!$status['running']) { // We may still have unread bytes on stdout or stderr, particularly if // this future is being buffered and streamed. If we do, we don't want to // consider the subprocess to have exited until we've read everything. // See T9724 for context. if (feof($stdout) && feof($stderr)) { $is_done = true; } } if ($is_done) { $signal_info = null; // If the subprocess got nuked with `kill -9`, we get a -1 exitcode. // Upgrade this to a slightly more informative value by examining the // terminating signal code. $err = $status['exitcode']; if ($err == -1) { if ($status['signaled']) { $signo = $status['termsig']; $err = 128 + $signo; $signal_info = pht( "\n\n", phutil_get_signal_name($signo), $signo); } } $result = array( $err, $this->stdout, $signal_info.$this->stderr, ); $this->recordResult($result); $this->closeProcess(); return true; } $elapsed = (microtime(true) - $this->start); if ($this->terminateTimeout && ($elapsed >= $this->terminateTimeout)) { if (!$this->didTerminate) { $this->killedByTimeout = true; $this->sendTerminateSignal(); return false; } } if ($this->killTimeout && ($elapsed >= $this->killTimeout)) { $this->killedByTimeout = true; $this->resolveKill(); return true; } } /** * @return void * @task internal */ public function __destruct() { if (!$this->proc) { return; } // NOTE: If we try to proc_close() an open process, we hang indefinitely. To // avoid this, kill the process explicitly if it's still running. $status = $this->procGetStatus(); if ($status['running']) { $this->sendTerminateSignal(); if (!$this->waitForExit(5)) { $this->resolveKill(); } } else { $this->closeProcess(); } } /** * Close and free resources if necessary. * * @return void * @task internal */ private function closeProcess() { foreach ($this->pipes as $pipe) { if (isset($pipe)) { @fclose($pipe); } } $this->pipes = array(null, null, null); if ($this->proc) { @proc_close($this->proc); $this->proc = null; } $this->stdin = null; unset($this->windowsStdoutTempFile); unset($this->windowsStderrTempFile); } /** * Execute `proc_get_status()`, but avoid pitfalls. * * @return dict Process status. * @task internal */ private function procGetStatus() { // After the process exits, we only get one chance to read proc_get_status() // before it starts returning garbage. Make sure we don't throw away the // last good read. if ($this->procStatus) { if (!$this->procStatus['running']) { return $this->procStatus; } } // See T13555. This may occur if you call "getPID()" on a future which // exited immediately without ever creating a valid subprocess. if (!$this->proc) { throw new Exception( pht( 'Attempting to get subprocess status in "ExecFuture" with no '. 'valid subprocess.')); } $this->procStatus = proc_get_status($this->proc); return $this->procStatus; } /** * Try to close stdin, if we're done using it. This keeps us from hanging if * the process on the other end of the pipe is waiting for EOF. * * @return void * @task internal */ private function tryToCloseStdin() { if (!$this->closePipe) { // We've been told to keep the pipe open by a call to write(..., true). return; } if ($this->stdin->getByteLength()) { // We still have bytes to write. return; } list($stdin) = $this->pipes; if (!$stdin) { // We've already closed stdin. return; } // There's nothing stopping us from closing stdin, so close it. @fclose($stdin); $this->pipes[0] = null; } public function getDefaultWait() { $wait = parent::getDefaultWait(); $next_timeout = $this->getNextTimeout(); if ($next_timeout) { if (!$this->start) { $this->start = microtime(true); } $elapsed = (microtime(true) - $this->start); $wait = max(0, min($next_timeout - $elapsed, $wait)); } return $wait; } private function getNextTimeout() { if ($this->didTerminate) { return $this->killTimeout; } else { return $this->terminateTimeout; } } private function sendTerminateSignal() { $this->didTerminate = true; proc_terminate($this->proc); return $this; } private function waitForExit($duration) { $start = microtime(true); while (true) { $status = $this->procGetStatus(); if (!$status['running']) { return true; } $waited = (microtime(true) - $start); if ($waited > $duration) { return false; } } } protected function getServiceProfilerStartParameters() { return array( 'type' => 'exec', 'command' => phutil_string_cast($this->getCommand()), ); } protected function getServiceProfilerResultParameters() { if ($this->hasResult()) { $result = $this->getResult(); $err = idx($result, 0); } else { $err = null; } return array( 'err' => $err, ); } private function getStdoutBufferLength() { if ($this->stdout === null) { return 0; } return strlen($this->stdout); } private function getStderrBufferLength() { if ($this->stderr === null) { return 0; } return strlen($this->stderr); } } diff --git a/src/lint/linter/ArcanistXMLLinter.php b/src/lint/linter/ArcanistXMLLinter.php index b48b19bc..1d7971ab 100644 --- a/src/lint/linter/ArcanistXMLLinter.php +++ b/src/lint/linter/ArcanistXMLLinter.php @@ -1,74 +1,78 @@ getData($path))) { // XML appears to be valid. return; } foreach (libxml_get_errors() as $error) { $message = id(new ArcanistLintMessage()) ->setPath($path) ->setLine($error->line) ->setChar($error->column ? $error->column : null) ->setCode($this->getLintMessageFullCode($error->code)) ->setName(pht('LibXML Error')) ->setDescription(trim($error->message)); switch ($error->level) { case LIBXML_ERR_NONE: $message->setSeverity(ArcanistLintSeverity::SEVERITY_DISABLED); break; case LIBXML_ERR_WARNING: $message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING); break; case LIBXML_ERR_ERROR: case LIBXML_ERR_FATAL: $message->setSeverity(ArcanistLintSeverity::SEVERITY_ERROR); break; default: $message->setSeverity(ArcanistLintSeverity::SEVERITY_ADVICE); break; } $this->addLintMessage($message); } } } diff --git a/src/lint/linter/__tests__/ArcanistLinterTestCase.php b/src/lint/linter/__tests__/ArcanistLinterTestCase.php index 85144bc2..19d9ce10 100644 --- a/src/lint/linter/__tests__/ArcanistLinterTestCase.php +++ b/src/lint/linter/__tests__/ArcanistLinterTestCase.php @@ -1,372 +1,382 @@ getLinter(); $files = id(new FileFinder($root)) ->withType('f') ->withSuffix('lint-test') ->find(); $test_count = 0; foreach ($files as $file) { $this->lintFile($root.$file, $linter); $test_count++; } $this->assertTrue( ($test_count > 0), pht( 'Expected to find some %s tests in directory %s!', '.lint-test', $root)); } private function lintFile($file, ArcanistLinter $linter) { $linter = clone $linter; + if (!$linter->canRun()) { + $this->assertSkipped( + pht( + 'Linter "%s" can not run.', + get_class($linter))); + } + $contents = Filesystem::readFile($file); $contents = preg_split('/^~{4,}\n/m', $contents); if (count($contents) < 2) { throw new Exception( pht( "Expected '%s' separating test case and results.", '~~~~~~~~~~')); } list($data, $expect, $xform, $config) = array_merge( $contents, array(null, null)); if ($config) { $config = phutil_json_decode($config); } else { $config = array(); } PhutilTypeSpec::checkMap( $config, array( 'config' => 'optional map', 'mode' => 'optional string', 'path' => 'optional string', 'stopped' => 'optional bool', )); $exception = null; $after_lint = null; $messages = null; $exception_message = false; $caught_exception = false; try { $path_name = idx($config, 'path'); if ($path_name !== null) { $basename = basename($path_name); } else { $basename = basename($file); } $tmp = new TempFile($basename); Filesystem::writeFile($tmp, $data); $full_path = (string)$tmp; $mode = idx($config, 'mode'); if ($mode) { Filesystem::changePermissions($tmp, octdec($mode)); } $dir = dirname($full_path); $working_copy = ArcanistWorkingCopyIdentity::newFromRootAndConfigFile( $dir, null, pht('Unit Test')); $configuration_manager = new ArcanistConfigurationManager(); $configuration_manager->setWorkingCopyIdentity($working_copy); $engine = new ArcanistUnitTestableLintEngine(); $engine->setWorkingCopy($working_copy); $engine->setConfigurationManager($configuration_manager); $engine->setPaths(array($basename)); $linter->setEngine($engine); $linter->addPath($basename); $linter->addData($basename, $data); foreach (idx($config, 'config', array()) as $key => $value) { $linter->setLinterConfigurationValue($key, $value); } $engine->addLinter($linter); $engine->addFileData($basename, $data); $results = $engine->run(); $this->assertEqual( 1, count($results), pht('Expect one result returned by linter.')); $assert_stopped = idx($config, 'stopped'); if ($assert_stopped !== null) { $this->assertEqual( $assert_stopped, $linter->didStopAllLinters(), $assert_stopped ? pht('Expect linter to be stopped.') : pht('Expect linter to not be stopped.')); } $result = reset($results); $patcher = ArcanistLintPatcher::newFromArcanistLintResult($result); $after_lint = $patcher->getModifiedFileContent(); } catch (PhutilTestTerminatedException $ex) { throw $ex; } catch (Exception $exception) { $caught_exception = true; if ($exception instanceof PhutilAggregateException) { $caught_exception = false; foreach ($exception->getExceptions() as $ex) { if ($ex instanceof ArcanistUsageException || $ex instanceof ArcanistMissingLinterException) { $this->assertSkipped($ex->getMessage()); } else { $caught_exception = true; } } } else if ($exception instanceof ArcanistUsageException || $exception instanceof ArcanistMissingLinterException) { $this->assertSkipped($exception->getMessage()); } $exception_message = $exception->getMessage()."\n\n". $exception->getTraceAsString(); } $this->assertEqual(false, $caught_exception, $exception_message); $this->compareLint($basename, $expect, $result); $this->compareTransform($xform, $after_lint); } private function compareLint($file, $expect, ArcanistLintResult $results) { $expected_results = new ArcanistLintResult(); $expect = trim($expect); if ($expect) { $expect = explode("\n", $expect); } else { $expect = array(); } foreach ($expect as $result) { $parts = explode(':', $result); $message = new ArcanistLintMessage(); $severity = idx($parts, 0); $line = idx($parts, 1); if ($line === '') { $line = null; } $char = idx($parts, 2); if ($char === '') { $char = null; } $code = idx($parts, 3); if ($code === '') { $code = null; } if ($severity !== null) { $message->setSeverity($severity); } if ($line !== null) { $message->setLine($line); } if ($char !== null) { $message->setChar($char); } if ($code !== null) { $message->setCode($code); } $expected_results->addMessage($message); } $missing = array(); $surprising = $results->getMessages(); // TODO: Make this more efficient. foreach ($expected_results->getMessages() as $expected_message) { $found = false; foreach ($results->getMessages() as $ii => $actual_message) { if (!self::compareLintMessageProperty( $expected_message->getSeverity(), $actual_message->getSeverity())) { continue; } if (!self::compareLintMessageProperty( $expected_message->getLine(), $actual_message->getLine())) { continue; } if (!self::compareLintMessageProperty( $expected_message->getChar(), $actual_message->getChar())) { continue; } if (!self::compareLintMessageProperty( $expected_message->getCode(), $actual_message->getCode())) { continue; } $found = true; unset($surprising[$ii]); } if (!$found) { $missing[] = $expected_message; } } if ($missing || $surprising) { $this->assertFailure( sprintf( "%s\n%s%s", pht( 'Lint emitted an unexpected set of messages for file "%s".', $file), $this->renderMessages(pht('MISSING MESSAGES'), $missing), $this->renderMessages(pht('SURPLUS MESSAGES'), $surprising))); } } private function compareTransform($expected, $actual) { + $expected = phutil_string_cast($expected); + if (!strlen($expected)) { return; } + $this->assertEqual( $expected, $actual, pht('File as patched by lint did not match the expected patched file.')); } /** * Compare properties of @{class:ArcanistLintMessage} instances. * * @param wild * @param wild * @return bool */ private static function compareLintMessageProperty($x, $y) { if ($x === null) { return true; } return ($x === $y); } private function renderMessages($header, array $messages) { if (!$messages) { $display = tsprintf( "%s\n", pht('(No messages.)')); } else { $lines = array(); foreach ($messages as $message) { $line = $message->getLine(); if ($line === null) { $display_line = pht(''); } else { $display_line = $line; } $char = $message->getChar(); if ($char === null) { $display_char = pht(''); } else { $display_char = $char; } $code = $message->getCode(); $name = $message->getName(); if ($code !== null && $name !== null) { $display_code = pht('%s: %s', $code, $name); } else if ($code !== null) { $display_code = pht('%s', $code); } else { $display_code = null; } $severity = $message->getSeverity(); if ($display_code === null) { $display_message = pht( 'Message with severity "%s" at "%s:%s"', $severity, $display_line, $display_char); } else { $display_message = pht( 'Message with severity "%s" at "%s:%s" (%s)', $severity, $display_line, $display_char, $display_code); } $lines[] = tsprintf( " %s\n", $display_message); } $display = implode('', $lines); } return tsprintf( "%s\n%B\n", $header, $display); } } diff --git a/src/lint/linter/xhpast/rules/ArcanistPHPCompatibilityXHPASTLinterRule.php b/src/lint/linter/xhpast/rules/ArcanistPHPCompatibilityXHPASTLinterRule.php index 743d1484..0b52eed4 100644 --- a/src/lint/linter/xhpast/rules/ArcanistPHPCompatibilityXHPASTLinterRule.php +++ b/src/lint/linter/xhpast/rules/ArcanistPHPCompatibilityXHPASTLinterRule.php @@ -1,487 +1,489 @@ version) { return; } if ($compat_info === null) { $target = phutil_get_library_root('arcanist'). '/../resources/php/symbol-information.json'; $compat_info = phutil_json_decode(Filesystem::readFile($target)); } // Create a whitelist for symbols which are being used conditionally. $whitelist = array( 'class' => array(), 'function' => array(), 'constant' => array(), ); $conditionals = $root->selectDescendantsOfType('n_IF'); foreach ($conditionals as $conditional) { $condition = $conditional->getChildOfType(0, 'n_CONTROL_CONDITION'); $function = $condition->getChildByIndex(0); if ($function->getTypeName() != 'n_FUNCTION_CALL') { continue; } $function_token = $function ->getChildByIndex(0); if ($function_token->getTypeName() != 'n_SYMBOL_NAME') { // This may be `Class::method(...)` or `$var(...)`. continue; } $function_name = $function_token->getConcreteString(); switch ($function_name) { case 'class_exists': case 'function_exists': case 'interface_exists': case 'defined': $type = null; switch ($function_name) { case 'class_exists': $type = 'class'; break; case 'function_exists': $type = 'function'; break; case 'interface_exists': $type = 'interface'; break; case 'defined': $type = 'constant'; break; } $params = $function->getChildOfType(1, 'n_CALL_PARAMETER_LIST'); $symbol = $params->getChildByIndex(0); if (!$symbol->isStaticScalar()) { break; } $symbol_name = $symbol->evalStatic(); if (!idx($whitelist[$type], $symbol_name)) { $whitelist[$type][$symbol_name] = array(); } $span = $conditional ->getChildByIndex(1) ->getTokens(); $whitelist[$type][$symbol_name][] = range( head_key($span), last_key($span)); break; } } $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL'); foreach ($calls as $call) { $node = $call->getChildByIndex(0); $name = $node->getConcreteString(); $version = idx($compat_info['functions'], $name, array()); $min = idx($version, 'php.min'); $max = idx($version, 'php.max'); $whitelisted = false; foreach (idx($whitelist['function'], $name, array()) as $range) { if (array_intersect($range, array_keys($node->getTokens()))) { $whitelisted = true; break; } } if ($whitelisted) { continue; } if ($min && version_compare($min, $this->version, '>')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s()` was not '. 'introduced until PHP %s.', $this->version, $name, $min)); } else if ($max && version_compare($max, $this->version, '<')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s()` was '. 'removed in PHP %s.', $this->version, $name, $max)); } else if (array_key_exists($name, $compat_info['params'])) { $params = $call->getChildOfType(1, 'n_CALL_PARAMETER_LIST'); foreach (array_values($params->getChildren()) as $i => $param) { $version = idx($compat_info['params'][$name], $i); if ($version && version_compare($version, $this->version, '>')) { $this->raiseLintAtNode( $param, pht( 'This codebase targets PHP %s, but parameter %d '. 'of `%s()` was not introduced until PHP %s.', $this->version, $i + 1, $name, $version)); } } } if ($this->windowsVersion) { $windows = idx($compat_info['functions_windows'], $name); - if ($windows === false) { + if ($windows === null) { + // This function has no special Windows considerations. + } else if ($windows === false) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s on Windows, '. 'but `%s()` is not available there.', $this->windowsVersion, $name)); } else if (version_compare($windows, $this->windowsVersion, '>')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s on Windows, '. 'but `%s()` is not available there until PHP %s.', $this->windowsVersion, $name, $windows)); } } } $classes = $root->selectDescendantsOfType('n_CLASS_NAME'); foreach ($classes as $node) { $name = $node->getConcreteString(); $version = idx($compat_info['interfaces'], $name, array()); $version = idx($compat_info['classes'], $name, $version); $min = idx($version, 'php.min'); $max = idx($version, 'php.max'); $whitelisted = false; foreach (idx($whitelist['class'], $name, array()) as $range) { if (array_intersect($range, array_keys($node->getTokens()))) { $whitelisted = true; break; } } if ($whitelisted) { continue; } if ($min && version_compare($min, $this->version, '>')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s` was not '. 'introduced until PHP %s.', $this->version, $name, $min)); } else if ($max && version_compare($max, $this->version, '<')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s` was '. 'removed in PHP %s.', $this->version, $name, $max)); } } // TODO: Technically, this will include function names. This is unlikely to // cause any issues (unless, of course, there existed a function that had // the same name as some constant). $constants = $root->selectDescendantsOfTypes(array( 'n_SYMBOL_NAME', 'n_MAGIC_SCALAR', )); foreach ($constants as $node) { $name = $node->getConcreteString(); $version = idx($compat_info['constants'], $name, array()); $min = idx($version, 'php.min'); $max = idx($version, 'php.max'); $whitelisted = false; foreach (idx($whitelist['constant'], $name, array()) as $range) { if (array_intersect($range, array_keys($node->getTokens()))) { $whitelisted = true; break; } } if ($whitelisted) { continue; } if ($min && version_compare($min, $this->version, '>')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s` was not '. 'introduced until PHP %s.', $this->version, $name, $min)); } else if ($max && version_compare($max, $this->version, '<')) { $this->raiseLintAtNode( $node, pht( 'This codebase targets PHP %s, but `%s` was '. 'removed in PHP %s.', $this->version, $name, $max)); } } if (version_compare($this->version, '5.3.0') < 0) { $this->lintPHP53Features($root); } else { $this->lintPHP53Incompatibilities($root); } if (version_compare($this->version, '5.4.0') < 0) { $this->lintPHP54Features($root); } else { $this->lintPHP54Incompatibilities($root); } } private function lintPHP53Features(XHPASTNode $root) { $functions = $root->selectTokensOfType('T_FUNCTION'); foreach ($functions as $function) { $next = $function->getNextToken(); while ($next) { if ($next->isSemantic()) { break; } $next = $next->getNextToken(); } if ($next) { if ($next->getTypeName() === '(') { $this->raiseLintAtToken( $function, pht( 'This codebase targets PHP %s, but anonymous '. 'functions were not introduced until PHP 5.3.', $this->version)); } } } $namespaces = $root->selectTokensOfType('T_NAMESPACE'); foreach ($namespaces as $namespace) { $this->raiseLintAtToken( $namespace, pht( 'This codebase targets PHP %s, but namespaces were not '. 'introduced until PHP 5.3.', $this->version)); } // NOTE: This is only "use x;", in anonymous functions the node type is // n_LEXICAL_VARIABLE_LIST even though both tokens are T_USE. $uses = $root->selectDescendantsOfType('n_USE_LIST'); foreach ($uses as $use) { $this->raiseLintAtNode( $use, pht( 'This codebase targets PHP %s, but namespaces were not '. 'introduced until PHP 5.3.', $this->version)); } $statics = $root->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); foreach ($statics as $static) { $name = $static->getChildByIndex(0); if ($name->getTypeName() != 'n_CLASS_NAME') { continue; } if ($name->getConcreteString() === 'static') { $this->raiseLintAtNode( $name, pht( 'This codebase targets PHP %s, but `%s` was not '. 'introduced until PHP 5.3.', $this->version, 'static::')); } } $ternaries = $root->selectDescendantsOfType('n_TERNARY_EXPRESSION'); foreach ($ternaries as $ternary) { $yes = $ternary->getChildByIndex(2); if ($yes->getTypeName() === 'n_EMPTY') { $this->raiseLintAtNode( $ternary, pht( 'This codebase targets PHP %s, but short ternary was '. 'not introduced until PHP 5.3.', $this->version)); } } $heredocs = $root->selectDescendantsOfType('n_HEREDOC'); foreach ($heredocs as $heredoc) { if (preg_match('/^<<<[\'"]/', $heredoc->getConcreteString())) { $this->raiseLintAtNode( $heredoc, pht( 'This codebase targets PHP %s, but nowdoc was not '. 'introduced until PHP 5.3.', $this->version)); } } } private function lintPHP53Incompatibilities(XHPASTNode $root) {} private function lintPHP54Features(XHPASTNode $root) { $indexes = $root->selectDescendantsOfType('n_INDEX_ACCESS'); foreach ($indexes as $index) { switch ($index->getChildByIndex(0)->getTypeName()) { case 'n_FUNCTION_CALL': case 'n_METHOD_CALL': $this->raiseLintAtNode( $index->getChildByIndex(1), pht( 'The `%s` syntax was not introduced until PHP 5.4, but this '. 'codebase targets an earlier version of PHP. You can rewrite '. 'this expression using `%s`.', 'f()[...]', 'idx()')); break; } } $literals = $root->selectDescendantsOfType('n_ARRAY_LITERAL'); foreach ($literals as $literal) { $open_token = head($literal->getTokens())->getValue(); if ($open_token == '[') { $this->raiseLintAtNode( $literal, pht( 'The short array syntax ("[...]") was not introduced until '. 'PHP 5.4, but this codebase targets an earlier version of PHP. '. 'You can rewrite this expression using `array(...)` instead.')); } } $closures = $this->getAnonymousClosures($root); foreach ($closures as $closure) { $static_accesses = $closure ->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); foreach ($static_accesses as $static_access) { $class = $static_access->getChildByIndex(0); if ($class->getTypeName() != 'n_CLASS_NAME') { continue; } if (strtolower($class->getConcreteString()) != 'self') { continue; } $this->raiseLintAtNode( $class, pht( 'The use of `%s` in an anonymous closure is not '. 'available before PHP 5.4.', 'self')); } $property_accesses = $closure ->selectDescendantsOfType('n_OBJECT_PROPERTY_ACCESS'); foreach ($property_accesses as $property_access) { $variable = $property_access->getChildByIndex(0); if ($variable->getTypeName() != 'n_VARIABLE') { continue; } if ($variable->getConcreteString() != '$this') { continue; } $this->raiseLintAtNode( $variable, pht( 'The use of `%s` in an anonymous closure is not '. 'available before PHP 5.4.', '$this')); } } $numeric_scalars = $root->selectDescendantsOfType('n_NUMERIC_SCALAR'); foreach ($numeric_scalars as $numeric_scalar) { if (preg_match('/^0b[01]+$/i', $numeric_scalar->getConcreteString())) { $this->raiseLintAtNode( $numeric_scalar, pht( 'Binary integer literals are not available before PHP 5.4.')); } } } private function lintPHP54Incompatibilities(XHPASTNode $root) { $breaks = $root->selectDescendantsOfTypes(array('n_BREAK', 'n_CONTINUE')); foreach ($breaks as $break) { $arg = $break->getChildByIndex(0); switch ($arg->getTypeName()) { case 'n_EMPTY': break; case 'n_NUMERIC_SCALAR': if ($arg->getConcreteString() != '0') { break; } default: $this->raiseLintAtNode( $break->getChildByIndex(0), pht( 'The `%s` and `%s` statements no longer accept '. 'variable arguments.', 'break', 'continue')); break; } } } } diff --git a/src/lint/linter/xhpast/rules/ArcanistParentMemberReferenceXHPASTLinterRule.php b/src/lint/linter/xhpast/rules/ArcanistParentMemberReferenceXHPASTLinterRule.php index 0183461b..ab18e683 100644 --- a/src/lint/linter/xhpast/rules/ArcanistParentMemberReferenceXHPASTLinterRule.php +++ b/src/lint/linter/xhpast/rules/ArcanistParentMemberReferenceXHPASTLinterRule.php @@ -1,71 +1,76 @@ selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($class_declarations as $class_declaration) { $extends_list = $class_declaration ->getChildByIndex(2); $parent_class = null; if ($extends_list->getTypeName() == 'n_EXTENDS_LIST') { $parent_class = $extends_list ->getChildOfType(0, 'n_CLASS_NAME') ->getConcreteString(); } if (!$parent_class) { continue; } $class_static_accesses = $class_declaration ->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); $closures = $this->getAnonymousClosures($class_declaration); foreach ($class_static_accesses as $class_static_access) { $double_colons = $class_static_access ->selectTokensOfType('T_PAAMAYIM_NEKUDOTAYIM'); $class_ref = $class_static_access->getChildByIndex(0); if ($class_ref->getTypeName() != 'n_CLASS_NAME') { continue; } $class_ref_name = $class_ref->getConcreteString(); if (strtolower($parent_class) == strtolower($class_ref_name)) { $in_closure = false; foreach ($closures as $closure) { if ($class_ref->isDescendantOf($closure)) { $in_closure = true; break; } } - if (version_compare($this->version, '5.4.0', '>=') || !$in_closure) { + $version_target = $this->version; + if ($version_target === null) { + $version_target = phpversion(); + } + + if (version_compare($version_target, '5.4.0', '>=') || !$in_closure) { $this->raiseLintAtNode( $class_ref, pht( 'Use `%s` to call parent method.', 'parent::'), 'parent'); } } } } } } diff --git a/src/lint/linter/xhpast/rules/ArcanistSelfMemberReferenceXHPASTLinterRule.php b/src/lint/linter/xhpast/rules/ArcanistSelfMemberReferenceXHPASTLinterRule.php index 8db13ca7..8c1ed2e4 100644 --- a/src/lint/linter/xhpast/rules/ArcanistSelfMemberReferenceXHPASTLinterRule.php +++ b/src/lint/linter/xhpast/rules/ArcanistSelfMemberReferenceXHPASTLinterRule.php @@ -1,58 +1,63 @@ selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($class_declarations as $class_declaration) { $class_name = $class_declaration ->getChildOfType(1, 'n_CLASS_NAME') ->getConcreteString(); $class_static_accesses = $class_declaration ->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); $closures = $this->getAnonymousClosures($class_declaration); foreach ($class_static_accesses as $class_static_access) { $class_ref = $class_static_access->getChildByIndex(0); if ($class_ref->getTypeName() != 'n_CLASS_NAME') { continue; } $class_ref_name = $class_ref->getConcreteString(); if (strtolower($class_name) == strtolower($class_ref_name)) { $in_closure = false; foreach ($closures as $closure) { if ($class_ref->isDescendantOf($closure)) { $in_closure = true; break; } } - if (version_compare($this->version, '5.4.0', '>=') || !$in_closure) { + $version_target = $this->version; + if (!$version_target) { + $version_target = phpversion(); + } + + if (version_compare($version_target, '5.4.0', '>=') || !$in_closure) { $this->raiseLintAtNode( $class_ref, pht( 'Use `%s` for local static member references.', 'self::'), 'self'); } } } } } } diff --git a/src/parser/ArcanistBundle.php b/src/parser/ArcanistBundle.php index 6d72047b..4617c9b6 100644 --- a/src/parser/ArcanistBundle.php +++ b/src/parser/ArcanistBundle.php @@ -1,1063 +1,1062 @@ authorEmail = $author_email; return $this; } public function getAuthorEmail() { return $this->authorEmail; } public function setAuthorName($author_name) { $this->authorName = $author_name; return $this; } public function getAuthorName() { return $this->authorName; } public function getFullAuthor() { $author_name = $this->getAuthorName(); if ($author_name === null) { return null; } $author_email = $this->getAuthorEmail(); if ($author_email === null) { return null; } $full_author = sprintf('%s <%s>', $author_name, $author_email); // Because git is very picky about the author being in a valid format, // verify that we can parse it. $address = new PhutilEmailAddress($full_author); if (!$address->getDisplayName() || !$address->getAddress()) { return null; } return $full_author; } public function setConduit(ConduitClient $conduit) { $this->conduit = $conduit; return $this; } public function setBaseRevision($base_revision) { $this->baseRevision = $base_revision; return $this; } public function setEncoding($encoding) { $this->encoding = $encoding; return $this; } public function getEncoding() { return $this->encoding; } public function setByteLimit($byte_limit) { $this->byteLimit = $byte_limit; return $this; } public function getByteLimit() { return $this->byteLimit; } public function getBaseRevision() { return $this->baseRevision; } public function setRevisionID($revision_id) { $this->revisionID = $revision_id; return $this; } public function getRevisionID() { return $this->revisionID; } public static function newFromChanges(array $changes) { $obj = new ArcanistBundle(); $obj->changes = $changes; return $obj; } private function getEOL($patch_type) { // NOTE: Git always generates "\n" line endings, even under Windows, and // can not parse certain patches with "\r\n" line endings. SVN generates // patches with "\n" line endings on Mac or Linux and "\r\n" line endings // on Windows. (This EOL style is used only for patch metadata lines, not // for the actual patch content.) // (On Windows, Mercurial generates \n newlines for `--git` diffs, as it // must, but also \n newlines for unified diffs. We never need to deal with // these as we use Git format for Mercurial, so this case is currently // ignored.) switch ($patch_type) { case 'git': return "\n"; case 'unified': return phutil_is_windows() ? "\r\n" : "\n"; default: throw new Exception( pht("Unknown patch type '%s'!", $patch_type)); } } public static function newFromArcBundle($path) { $path = Filesystem::resolvePath($path); $future = new ExecFuture( 'tar tfO %s', $path); list($stdout, $file_list) = $future->resolvex(); $file_list = explode("\n", trim($file_list)); if (in_array('meta.json', $file_list)) { $future = new ExecFuture( 'tar xfO %s meta.json', $path); $meta_info = $future->resolveJSON(); $version = idx($meta_info, 'version', 0); $base_revision = idx($meta_info, 'baseRevision'); $revision_id = idx($meta_info, 'revisionID'); $encoding = idx($meta_info, 'encoding'); $author_name = idx($meta_info, 'authorName'); $author_email = idx($meta_info, 'authorEmail'); } else { // this arc bundle was probably made before we started storing meta info $version = 0; $base_revision = null; $revision_id = null; $encoding = null; $author = null; } $future = new ExecFuture( 'tar xfO %s changes.json', $path); $changes = $future->resolveJSON(); foreach ($changes as $change_key => $change) { foreach ($change['hunks'] as $key => $hunk) { list($hunk_data) = execx('tar xfO %s hunks/%s', $path, $hunk['corpus']); $changes[$change_key]['hunks'][$key]['corpus'] = $hunk_data; } } foreach ($changes as $change_key => $change) { $changes[$change_key] = ArcanistDiffChange::newFromDictionary($change); } $obj = new ArcanistBundle(); $obj->changes = $changes; $obj->diskPath = $path; $obj->setBaseRevision($base_revision); $obj->setRevisionID($revision_id); $obj->setEncoding($encoding); return $obj; } public static function newFromDiff($data) { $obj = new ArcanistBundle(); $parser = new ArcanistDiffParser(); $obj->changes = $parser->parseDiff($data); return $obj; } private function __construct() {} public function writeToDisk($path) { $changes = $this->getChanges(); $change_list = array(); foreach ($changes as $change) { $change_list[] = $change->toDictionary(); } $hunks = array(); foreach ($change_list as $change_key => $change) { foreach ($change['hunks'] as $key => $hunk) { $hunks[] = $hunk['corpus']; $change_list[$change_key]['hunks'][$key]['corpus'] = count($hunks) - 1; } } $blobs = array(); foreach ($change_list as $change) { if (!empty($change['metadata']['old:binary-phid'])) { $blobs[$change['metadata']['old:binary-phid']] = null; } if (!empty($change['metadata']['new:binary-phid'])) { $blobs[$change['metadata']['new:binary-phid']] = null; } } foreach ($blobs as $phid => $null) { $blobs[$phid] = $this->getBlob($phid); } $meta_info = array( 'version' => 5, 'baseRevision' => $this->getBaseRevision(), 'revisionID' => $this->getRevisionID(), 'encoding' => $this->getEncoding(), 'authorName' => $this->getAuthorName(), 'authorEmail' => $this->getAuthorEmail(), ); $dir = Filesystem::createTemporaryDirectory(); Filesystem::createDirectory($dir.'/hunks'); Filesystem::createDirectory($dir.'/blobs'); Filesystem::writeFile($dir.'/changes.json', json_encode($change_list)); Filesystem::writeFile($dir.'/meta.json', json_encode($meta_info)); foreach ($hunks as $key => $hunk) { Filesystem::writeFile($dir.'/hunks/'.$key, $hunk); } foreach ($blobs as $key => $blob) { Filesystem::writeFile($dir.'/blobs/'.$key, $blob); } execx( '(cd %s; tar -czf %s *)', $dir, Filesystem::resolvePath($path)); Filesystem::remove($dir); } public function toUnifiedDiff() { $this->reservedBytes = 0; $eol = $this->getEOL('unified'); $result = array(); $changes = $this->getChanges(); foreach ($changes as $change) { $hunk_changes = $this->buildHunkChanges($change->getHunks(), $eol); if (!$hunk_changes) { continue; } $old_path = $this->getOldPath($change); $cur_path = $this->getCurrentPath($change); $index_path = $cur_path; if ($index_path === null) { $index_path = $old_path; } $result[] = 'Index: '.$index_path; $result[] = $eol; $result[] = str_repeat('=', 67); $result[] = $eol; if ($old_path === null) { $old_path = '/dev/null'; } if ($cur_path === null) { $cur_path = '/dev/null'; } // When the diff is used by `patch`, `patch` ignores what is listed as the // current path and just makes changes to the file at the old path (unless // the current path is '/dev/null'. // If the old path and the current path aren't the same (and neither is // /dev/null), this indicates the file was moved or copied. By listing // both paths as the new file, `patch` will apply the diff to the new // file. if ($cur_path !== '/dev/null' && $old_path !== '/dev/null') { $old_path = $cur_path; } $result[] = '--- '.$old_path.$eol; $result[] = '+++ '.$cur_path.$eol; $result[] = $hunk_changes; } if (!$result) { return ''; } $diff = implode('', $result); return $this->convertNonUTF8Diff($diff); } public function toGitPatch() { $this->reservedBytes = 0; $eol = $this->getEOL('git'); $result = array(); $changes = $this->getChanges(); $binary_sources = array(); foreach ($changes as $change) { if (!$this->isGitBinaryChange($change)) { continue; } $type = $change->getType(); if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY || $type == ArcanistDiffChangeType::TYPE_COPY_AWAY || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { foreach ($change->getAwayPaths() as $path) { $binary_sources[$path] = $change; } } } foreach (array_keys($changes) as $multicopy_key) { $multicopy_change = $changes[$multicopy_key]; $type = $multicopy_change->getType(); if ($type != ArcanistDiffChangeType::TYPE_MULTICOPY) { continue; } // Decompose MULTICOPY into one MOVE_HERE and several COPY_HERE because // we need more information than we have in order to build a delete patch // and represent it as a bunch of COPY_HERE plus a delete. For details, // see T419. // Basically, MULTICOPY means there are 2 or more corresponding COPY_HERE // changes, so find one of them arbitrarily and turn it into a MOVE_HERE. // TODO: We might be able to do this more cleanly after T230 is resolved. $decompose_okay = false; foreach ($changes as $change_key => $change) { if ($change->getType() != ArcanistDiffChangeType::TYPE_COPY_HERE) { continue; } if ($change->getOldPath() != $multicopy_change->getCurrentPath()) { continue; } $decompose_okay = true; $change = clone $change; $change->setType(ArcanistDiffChangeType::TYPE_MOVE_HERE); $changes[$change_key] = $change; // The multicopy is now fully represented by MOVE_HERE plus one or more // COPY_HERE, so throw it away. unset($changes[$multicopy_key]); break; } if (!$decompose_okay) { throw new Exception( pht( 'Failed to decompose multicopy changeset in '. 'order to generate diff.')); } } foreach ($changes as $change) { $type = $change->getType(); $file_type = $change->getFileType(); if ($file_type == ArcanistDiffChangeType::FILE_DIRECTORY) { // TODO: We should raise a FYI about this, so the user is aware // that we omitted it, if the directory is empty or has permissions // which git can't represent. // Git doesn't support empty directories, so we simply ignore them. If // the directory is nonempty, 'git apply' will create it when processing // the changesets for files inside it. continue; } if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY) { // Git will apply this in the corresponding MOVE_HERE. continue; } $old_mode = idx($change->getOldProperties(), 'unix:filemode', '100644'); $new_mode = idx($change->getNewProperties(), 'unix:filemode', '100644'); $is_binary = $this->isGitBinaryChange($change); if ($is_binary) { $old_binary = idx($binary_sources, $this->getCurrentPath($change)); $change_body = $this->buildBinaryChange($change, $old_binary); } else { $change_body = $this->buildHunkChanges($change->getHunks(), $eol); } if ($type == ArcanistDiffChangeType::TYPE_COPY_AWAY) { // TODO: This is only relevant when patching old Differential diffs // which were created prior to arc pruning TYPE_COPY_AWAY for files // with no modifications. if (!strlen($change_body) && ($old_mode == $new_mode)) { continue; } } $old_path = $this->getOldPath($change); $cur_path = $this->getCurrentPath($change); if ($old_path === null) { $old_index = 'a/'.$cur_path; $old_target = '/dev/null'; } else { $old_index = 'a/'.$old_path; $old_target = 'a/'.$old_path; } if ($cur_path === null) { $cur_index = 'b/'.$old_path; $cur_target = '/dev/null'; } else { $cur_index = 'b/'.$cur_path; $cur_target = 'b/'.$cur_path; } $old_target = $this->encodeGitTargetPath($old_target); $cur_target = $this->encodeGitTargetPath($cur_target); $result[] = "diff --git {$old_index} {$cur_index}".$eol; if ($type == ArcanistDiffChangeType::TYPE_ADD) { $result[] = "new file mode {$new_mode}".$eol; } if ($type == ArcanistDiffChangeType::TYPE_COPY_HERE || $type == ArcanistDiffChangeType::TYPE_MOVE_HERE || $type == ArcanistDiffChangeType::TYPE_COPY_AWAY || $type == ArcanistDiffChangeType::TYPE_CHANGE) { if ($old_mode !== $new_mode) { $result[] = "old mode {$old_mode}".$eol; $result[] = "new mode {$new_mode}".$eol; } } if ($type == ArcanistDiffChangeType::TYPE_COPY_HERE) { $result[] = "copy from {$old_path}".$eol; $result[] = "copy to {$cur_path}".$eol; } else if ($type == ArcanistDiffChangeType::TYPE_MOVE_HERE) { $result[] = "rename from {$old_path}".$eol; $result[] = "rename to {$cur_path}".$eol; } else if ($type == ArcanistDiffChangeType::TYPE_DELETE || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { $old_mode = idx($change->getOldProperties(), 'unix:filemode'); if ($old_mode) { $result[] = "deleted file mode {$old_mode}".$eol; } } if ($change_body) { if (!$is_binary) { $result[] = "--- {$old_target}".$eol; $result[] = "+++ {$cur_target}".$eol; } $result[] = $change_body; } } $diff = implode('', $result).$eol; return $this->convertNonUTF8Diff($diff); } private function isGitBinaryChange(ArcanistDiffChange $change) { $file_type = $change->getFileType(); return ($file_type == ArcanistDiffChangeType::FILE_BINARY || $file_type == ArcanistDiffChangeType::FILE_IMAGE); } private function convertNonUTF8Diff($diff) { if ($this->encoding) { $diff = phutil_utf8_convert($diff, $this->encoding, 'UTF-8'); } return $diff; } public function getChanges() { return $this->changes; } private function breakHunkIntoSmallHunks(ArcanistDiffHunk $base_hunk) { $context = 3; $results = array(); $lines = phutil_split_lines($base_hunk->getCorpus()); $n = count($lines); $old_offset = $base_hunk->getOldOffset(); $new_offset = $base_hunk->getNewOffset(); $ii = 0; $jj = 0; while ($ii < $n) { // Skip lines until we find the next line with changes. Note: this skips // both ' ' (no changes) and '\' (no newline at end of file) lines. If we // don't skip the latter, we may incorrectly generate a terminal hunk // that has no actual change information when a file doesn't have a // terminal newline and not changed near the end of the file. 'patch' will // fail to apply the diff if we generate a hunk that does not actually // contain changes. for ($jj = $ii; $jj < $n; ++$jj) { $char = $lines[$jj][0]; if ($char == '-' || $char == '+') { break; } } if ($jj >= $n) { break; } $hunk_start = max($jj - $context, 0); // NOTE: There are two tricky considerations here. // We can not generate a patch with overlapping hunks, or 'git apply' // rejects it after 1.7.3.4. // We can not generate a patch with too much trailing context, or // 'patch' rejects it. // So we need to ensure that we generate disjoint hunks, but don't // generate any hunks with too much context. $old_lines = 0; $new_lines = 0; $hunk_adjust = 0; $last_change = $jj; $break_here = null; for (; $jj < $n; ++$jj) { if ($lines[$jj][0] == ' ') { if ($jj - $last_change > $context) { if ($break_here === null) { // We haven't seen a change in $context lines, so this is a // potential place to break the hunk. However, we need to keep // looking in case there is another change fewer than $context // lines away, in which case we have to merge the hunks. $break_here = $jj; } } // If the context value is "3" and there are 7 unchanged lines // between the two changes, we could either generate one or two hunks // and end up with the same number of output lines. If we generate // one hunk, the middle line will be a line of source. If we generate // two hunks, the middle line will be an "@@ -1,2 +3,4 @@" header. // We choose to generate two hunks because this is the behavior of // "diff -u". See PHI838. if ($jj - $last_change >= ($context * 2 + 1)) { // We definitely aren't going to merge this with the next hunk, so // break out of the loop. We'll end the hunk at $break_here. break; } } else { $break_here = null; $last_change = $jj; if ($lines[$jj][0] == '\\') { // When we have a "\ No newline at end of file" line, it does not // contribute to either hunk length. ++$hunk_adjust; } else if ($lines[$jj][0] == '-') { ++$old_lines; } else if ($lines[$jj][0] == '+') { ++$new_lines; } } } if ($break_here !== null) { $jj = $break_here; } $hunk_length = min($jj, $n) - $hunk_start; $count_length = ($hunk_length - $hunk_adjust); $hunk = new ArcanistDiffHunk(); $hunk->setOldOffset($old_offset + $hunk_start - $ii); $hunk->setNewOffset($new_offset + $hunk_start - $ii); $hunk->setOldLength($count_length - $new_lines); $hunk->setNewLength($count_length - $old_lines); $corpus = array_slice($lines, $hunk_start, $hunk_length); $corpus = implode('', $corpus); $hunk->setCorpus($corpus); $results[] = $hunk; $old_offset += ($jj - $ii) - $new_lines; $new_offset += ($jj - $ii) - $old_lines; $ii = $jj; } return $results; } private function encodeGitTargetPath($path) { // See T8768. If a target path contains spaces, it must be terminated with // a tab. If we don't do this, Mercurial has the wrong behavior when // applying the patch. This results in a semantic trailing whitespace // character: // // +++ b/X Y.txt\t // // Everyone is at fault here and there are no winners. if (strpos($path, ' ') !== false) { $path = $path."\t"; } return $path; } private function getOldPath(ArcanistDiffChange $change) { $old_path = $change->getOldPath(); $type = $change->getType(); - if (!strlen($old_path) || - $type == ArcanistDiffChangeType::TYPE_ADD) { + if ($old_path === '' || $type == ArcanistDiffChangeType::TYPE_ADD) { $old_path = null; } return $old_path; } private function getCurrentPath(ArcanistDiffChange $change) { $cur_path = $change->getCurrentPath(); $type = $change->getType(); if (!strlen($cur_path) || $type == ArcanistDiffChangeType::TYPE_DELETE || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { $cur_path = null; } return $cur_path; } private function buildHunkChanges(array $hunks, $eol) { assert_instances_of($hunks, 'ArcanistDiffHunk'); $result = array(); foreach ($hunks as $hunk) { $small_hunks = $this->breakHunkIntoSmallHunks($hunk); foreach ($small_hunks as $small_hunk) { $o_off = $small_hunk->getOldOffset(); $o_len = $small_hunk->getOldLength(); $n_off = $small_hunk->getNewOffset(); $n_len = $small_hunk->getNewLength(); $corpus = $small_hunk->getCorpus(); $this->reserveBytes(strlen($corpus)); // NOTE: If the length is 1 it can be omitted. Since git does this, // we also do it so that "arc export --git" diffs are as similar to // real git diffs as possible, which helps debug issues. if ($o_len == 1) { $o_head = "{$o_off}"; } else { $o_head = "{$o_off},{$o_len}"; } if ($n_len == 1) { $n_head = "{$n_off}"; } else { $n_head = "{$n_off},{$n_len}"; } $result[] = "@@ -{$o_head} +{$n_head} @@".$eol; $result[] = $corpus; $last = substr($corpus, -1); if ($last !== false && $last != "\r" && $last != "\n") { $result[] = $eol; } } } return implode('', $result); } public function setLoadFileDataCallback($callback) { $this->loadFileDataCallback = $callback; return $this; } private function getBlob($phid, $name = null) { if ($this->loadFileDataCallback) { return call_user_func($this->loadFileDataCallback, $phid); } if ($this->diskPath) { list($blob_data) = execx('tar xfO %s blobs/%s', $this->diskPath, $phid); return $blob_data; } $console = PhutilConsole::getConsole(); if ($this->conduit) { if ($name) { $console->writeErr( "%s\n", pht("Downloading binary data for '%s'...", $name)); } else { $console->writeErr("%s\n", pht('Downloading binary data...')); } $data_base64 = $this->conduit->callMethodSynchronous( 'file.download', array( 'phid' => $phid, )); return base64_decode($data_base64); } throw new Exception(pht("Nowhere to load blob '%s' from!", $phid)); } private function buildBinaryChange(ArcanistDiffChange $change, $old_binary) { $eol = $this->getEOL('git'); // In Git, when we write out a binary file move or copy, we need the // original binary for the source and the current binary for the // destination. if ($old_binary) { if ($old_binary->getOriginalFileData() !== null) { $old_data = $old_binary->getOriginalFileData(); $old_phid = null; } else { $old_data = null; $old_phid = $old_binary->getMetadata('old:binary-phid'); } } else { $old_data = $change->getOriginalFileData(); $old_phid = $change->getMetadata('old:binary-phid'); } if ($old_data === null && $old_phid) { $name = basename($change->getOldPath()); $old_data = $this->getBlob($old_phid, $name); } $old_length = strlen($old_data); // Here, and below, the binary will be emitted with base85 encoding. This // encoding encodes each 4 bytes of input in 5 bytes of output, so we may // need up to 5/4ths as many bytes to represent it. // We reserve space up front because base85 encoding isn't super cheap. If // the blob is enormous, we'd rather just bail out now before doing a ton // of work and then throwing it away anyway. // However, the data is compressed before it is emitted so we may actually // end up using fewer bytes. For now, the allocator just assumes the worst // case since it isn't important to be precise, but we could do a more // exact job of this. $this->reserveBytes($old_length * 5 / 4); if ($old_data === null) { $old_data = ''; $old_sha1 = str_repeat('0', 40); } else { $old_sha1 = sha1("blob {$old_length}\0{$old_data}"); } $new_phid = $change->getMetadata('new:binary-phid'); $new_data = null; if ($change->getCurrentFileData() !== null) { $new_data = $change->getCurrentFileData(); } else if ($new_phid) { $name = basename($change->getCurrentPath()); $new_data = $this->getBlob($new_phid, $name); } $new_length = strlen($new_data); $this->reserveBytes($new_length * 5 / 4); if ($new_data === null) { $new_data = ''; $new_sha1 = str_repeat('0', 40); } else { $new_sha1 = sha1("blob {$new_length}\0{$new_data}"); } $content = array(); $content[] = "index {$old_sha1}..{$new_sha1}".$eol; $content[] = 'GIT binary patch'.$eol; $content[] = "literal {$new_length}".$eol; $content[] = $this->emitBinaryDiffBody($new_data).$eol; $content[] = "literal {$old_length}".$eol; $content[] = $this->emitBinaryDiffBody($old_data).$eol; return implode('', $content); } private function emitBinaryDiffBody($data) { $eol = $this->getEOL('git'); return self::newBase85Data($data, $eol); } public static function newBase85Data($data, $eol, $mode = null) { // The "32bit" and "64bit" modes are used by unit tests to verify that all // of the encoding pathways here work identically. In these modes, we skip // compression because `gzcompress()` may not be stable and we just want // to test that the output matches some expected result. if ($mode === null) { if (!function_exists('gzcompress')) { throw new Exception( pht( 'This patch has binary data. The PHP zlib extension is required '. 'to apply patches with binary data to git. Install the PHP zlib '. 'extension to continue.')); } $input = gzcompress($data); $is_64bit = (PHP_INT_SIZE >= 8); } else { switch ($mode) { case '32bit': $input = $data; $is_64bit = false; break; case '64bit': $input = $data; $is_64bit = true; break; default: throw new Exception( pht( 'Unsupported base85 encoding mode "%s".', $mode)); } } // See emit_binary_diff_body() in diff.c for git's implementation. // This is implemented awkwardly in order to closely mirror git's // implementation in base85.c // It is also implemented awkwardly to work correctly on 32-bit machines. // Broadly, this algorithm converts the binary input to printable output // by transforming each 4 binary bytes of input to 5 printable bytes of // output, one piece at a time. // // To do this, we convert the 4 bytes into a 32-bit integer, then use // modulus and division by 85 to pick out printable bytes (85^5 is slightly // larger than 2^32). In C, this algorithm is fairly easy to implement // because the accumulator can be made unsigned. // // In PHP, there are no unsigned integers, so values larger than 2^31 break // on 32-bit systems under modulus: // // $ php -r 'print (1 << 31) % 13;' # On a 32-bit machine. // -11 // // However, PHP's float type is an IEEE 754 64-bit double precision float, // so we can safely store integers up to around 2^53 without loss of // precision. To work around the lack of an unsigned type, we just use a // double and perform the modulus with fmod(). // // (Since PHP overflows integer operations into floats, we don't need much // additional casting.) // On 64 bit systems, we skip all this fanfare and just use integers. This // is significantly faster. static $map = array( '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '!', '#', '$', '%', '&', '(', ')', '*', '+', '-', ';', '<', '=', '>', '?', '@', '^', '_', '`', '{', '|', '}', '~', ); $len_map = array(); for ($ii = 0; $ii <= 52; $ii++) { if ($ii <= 26) { $len_map[$ii] = chr($ii + ord('A') - 1); } else { $len_map[$ii] = chr($ii - 26 + ord('a') - 1); } } $buf = ''; $lines = str_split($input, 52); $final = (count($lines) - 1); foreach ($lines as $idx => $line) { if ($idx === $final) { $len = strlen($line); } else { $len = 52; } // The first character encodes the line length. $buf .= $len_map[$len]; $pos = 0; while ($len) { $accum = 0; for ($count = 24; $count >= 0; $count -= 8) { $val = ord($line[$pos++]); $val = $val * (1 << $count); $accum = $accum + $val; if (--$len == 0) { break; } } $slice = ''; // If we're in 64bit mode, we can just use integers. Otherwise, we // need to use floating point math to avoid overflows. if ($is_64bit) { for ($count = 4; $count >= 0; $count--) { $val = $accum % 85; - $accum = $accum / 85; + $accum = (int)($accum / 85); $slice .= $map[$val]; } } else { for ($count = 4; $count >= 0; $count--) { $val = (int)fmod($accum, 85.0); $accum = floor($accum / 85.0); $slice .= $map[$val]; } } $buf .= strrev($slice); } $buf .= $eol; } return $buf; } private function reserveBytes($bytes) { $this->reservedBytes += $bytes; if ($this->byteLimit) { if ($this->reservedBytes > $this->byteLimit) { throw new ArcanistDiffByteSizeException( pht( 'This large diff requires more space than it is allowed to '. 'use (limited to %s bytes; needs more than %s bytes).', new PhutilNumber($this->byteLimit), new PhutilNumber($this->reservedBytes))); } } return $this; } } diff --git a/src/parser/ArcanistDiffParser.php b/src/parser/ArcanistDiffParser.php index 1d213846..9a708fd2 100644 --- a/src/parser/ArcanistDiffParser.php +++ b/src/parser/ArcanistDiffParser.php @@ -1,1398 +1,1401 @@ repositoryAPI = $repository_api; return $this; } public function setDetectBinaryFiles($detect) { $this->detectBinaryFiles = $detect; return $this; } public function setTryEncoding($encoding) { $this->tryEncoding = $encoding; return $this; } public function forcePath($path) { $this->forcePath = $path; return $this; } public function setChanges(array $changes) { assert_instances_of($changes, 'ArcanistDiffChange'); $this->changes = mpull($changes, null, 'getCurrentPath'); return $this; } public function parseSubversionDiff(ArcanistSubversionAPI $api, $paths) { $this->setRepositoryAPI($api); $diffs = array(); foreach ($paths as $path => $status) { if ($status & ArcanistRepositoryAPI::FLAG_UNTRACKED || $status & ArcanistRepositoryAPI::FLAG_CONFLICT || $status & ArcanistRepositoryAPI::FLAG_MISSING) { unset($paths[$path]); } } $root = null; $from = array(); foreach ($paths as $path => $status) { $change = $this->buildChange($path); if ($status & ArcanistRepositoryAPI::FLAG_ADDED) { $change->setType(ArcanistDiffChangeType::TYPE_ADD); } else if ($status & ArcanistRepositoryAPI::FLAG_DELETED) { $change->setType(ArcanistDiffChangeType::TYPE_DELETE); } else { $change->setType(ArcanistDiffChangeType::TYPE_CHANGE); } $is_dir = is_dir($api->getPath($path)); if ($is_dir) { $change->setFileType(ArcanistDiffChangeType::FILE_DIRECTORY); // We have to go hit the diff even for directories because they may // have property changes or moves, etc. } $is_link = is_link($api->getPath($path)); if ($is_link) { $change->setFileType(ArcanistDiffChangeType::FILE_SYMLINK); } $diff = $api->getRawDiffText($path); if ($diff) { $this->parseDiff($diff); } $info = $api->getSVNInfo($path); if (idx($info, 'Copied From URL')) { if (!$root) { $rinfo = $api->getSVNInfo('.'); $root = $rinfo['URL'].'/'; } $cpath = $info['Copied From URL']; $root_len = strlen($root); if (!strncmp($cpath, $root, $root_len)) { $cpath = substr($cpath, $root_len); // The user can "svn cp /path/to/file@12345 x", which pulls a file out // of version history at a specific revision. If we just use the path, // we'll collide with possible changes to that path in the working // copy below. In particular, "svn cp"-ing a path which no longer // exists somewhere in the working copy and then adding that path // gets us to the "origin change type" branches below with a // TYPE_ADD state on the path. To avoid this, append the origin // revision to the path so we'll necessarily generate a new change. // TODO: In theory, you could have an '@' in your path and this could // cause a collision, e.g. two files named 'f' and 'f@12345'. This is // at least somewhat the user's fault, though. if ($info['Copied From Rev']) { if ($info['Copied From Rev'] != $info['Revision']) { $cpath .= '@'.$info['Copied From Rev']; } } $change->setOldPath($cpath); $from[$path] = $cpath; } } $type = $change->getType(); if (($type === ArcanistDiffChangeType::TYPE_MOVE_AWAY || $type === ArcanistDiffChangeType::TYPE_DELETE) && idx($info, 'Node Kind') === 'directory') { $change->setFileType(ArcanistDiffChangeType::FILE_DIRECTORY); } } foreach ($paths as $path => $status) { $change = $this->buildChange($path); if (empty($from[$path])) { continue; } if (empty($this->changes[$from[$path]])) { if ($change->getType() == ArcanistDiffChangeType::TYPE_COPY_HERE) { // If the origin path wasn't changed (or isn't included in this diff) // and we only copied it, don't generate a changeset for it. This // keeps us out of trouble when we go to 'arc commit' and need to // figure out which files should be included in the commit list. continue; } } $origin = $this->buildChange($from[$path]); $origin->addAwayPath($change->getCurrentPath()); $type = $origin->getType(); switch ($type) { case ArcanistDiffChangeType::TYPE_MULTICOPY: case ArcanistDiffChangeType::TYPE_COPY_AWAY: // "Add" is possible if you do some bizarre tricks with svn:ignore and // "svn copy"'ing URLs straight from the repository; you can end up with // a file that is a copy of itself. See T271. case ArcanistDiffChangeType::TYPE_ADD: break; case ArcanistDiffChangeType::TYPE_DELETE: $origin->setType(ArcanistDiffChangeType::TYPE_MOVE_AWAY); break; case ArcanistDiffChangeType::TYPE_MOVE_AWAY: $origin->setType(ArcanistDiffChangeType::TYPE_MULTICOPY); break; case ArcanistDiffChangeType::TYPE_CHANGE: $origin->setType(ArcanistDiffChangeType::TYPE_COPY_AWAY); break; default: throw new Exception(pht('Bad origin state %s.', $type)); } $type = $origin->getType(); switch ($type) { case ArcanistDiffChangeType::TYPE_MULTICOPY: case ArcanistDiffChangeType::TYPE_MOVE_AWAY: $change->setType(ArcanistDiffChangeType::TYPE_MOVE_HERE); break; case ArcanistDiffChangeType::TYPE_ADD: case ArcanistDiffChangeType::TYPE_COPY_AWAY: $change->setType(ArcanistDiffChangeType::TYPE_COPY_HERE); break; default: throw new Exception(pht('Bad origin state %s.', $type)); } } return $this->changes; } public function parseDiff($diff) { if (!strlen(trim($diff))) { throw new Exception(pht("Can't parse an empty diff!")); } // Detect `git-format-patch`, by looking for a "---" line somewhere in // the file and then a footer with Git version number, which looks like // this: // // -- // 1.8.4.2 // // Note that `git-format-patch` adds a space after the "--", but we don't // require it when detecting patches, as trailing whitespace can easily be // lost in transit. $detect_patch = '/^---$.*^-- ?[\s\d.]+\z/ms'; $message = null; if (preg_match($detect_patch, $diff)) { list($message, $diff) = $this->stripGitFormatPatch($diff); } $this->didStartParse($diff); // Strip off header comments. While `patch` allows comments anywhere in the // file, `git apply` is more strict. We get these comments in `hg export` // diffs, and Eclipse can also produce them. $line = $this->getLineTrimmed(); while (preg_match('/^#/', $line)) { $line = $this->nextLine(); } - if (strlen($message)) { + if ($message !== null && strlen($message)) { // If we found a message during pre-parse steps, add it to the resulting // changes here. $change = $this->buildChange(null) ->setType(ArcanistDiffChangeType::TYPE_MESSAGE) ->setMetadata('message', $message); } do { $patterns = array( // This is a normal SVN text change, probably from "svn diff". '(?PIndex): (?P.+)', // This is an SVN text change, probably from "svnlook diff". '(?PModified|Added|Deleted|Copied): (?P.+)', // This is an SVN property change, probably from "svn diff". '(?PProperty changes on): (?P.+)', // This is a git commit message, probably from "git show". '(?Pcommit) (?P[a-f0-9]+)(?: \(.*\))?', // This is a git diff, probably from "git show" or "git diff". // Note that the filenames may appear quoted. '(?Pdiff --git) (?P.*)', // RCS Diff '(?Prcsdiff -u) (?P.*)', // This is a unified diff, probably from "diff -u" or synthetic diffing. '(?P---) (?P.+)\s+\d{4}-\d{2}-\d{2}.*', '(?PBinary files|Files) '. '(?P.+)\s+\d{4}-\d{2}-\d{2} and '. '(?P.+)\s+\d{4}-\d{2}-\d{2} differ.*', // This is a normal Mercurial text change, probably from "hg diff". It // may have two "-r" blocks if it came from "hg diff -r x:y". '(?Pdiff -r) (?P[a-f0-9]+) (?:-r [a-f0-9]+ )?(?P.+)', ); $line = $this->getLineTrimmed(); $match = null; $ok = $this->tryMatchHeader($patterns, $line, $match); $failed_parse = false; if (!$ok && $this->isFirstNonEmptyLine()) { // 'hg export' command creates so called "extended diff" that // contains some meta information and comment at the beginning // (isFirstNonEmptyLine() to check for beginning). Actual mercurial // code detects where comment ends and unified diff starts by // searching for "diff -r" or "diff --git" in the text. $this->saveLine(); $line = $this->nextLineThatLooksLikeDiffStart(); if (!$this->tryMatchHeader($patterns, $line, $match)) { // Restore line before guessing to display correct error. $this->restoreLine(); $failed_parse = true; } } else if (!$ok) { $failed_parse = true; } if ($failed_parse) { $this->didFailParse( pht( "Expected a hunk header, like '%s' (svn), '%s' (svn properties), ". "'%s' (git show), '%s' (git diff), '%s' (unified diff), or ". "'%s' (hg diff or patch).", 'Index: /path/to/file.ext', 'Property changes on: /path/to/file.ext', 'commit 59bcc3ad6775562f845953cf01624225', 'diff --git', '--- filename', 'diff -r')); } if (isset($match['type'])) { if ($match['type'] == 'diff --git') { $filename = self::extractGitCommonFilename($match['oldnew']); if ($filename !== null) { $match['old'] = $filename; $match['cur'] = $filename; } } } $change = $this->buildChange(idx($match, 'cur')); if (isset($match['old'])) { $change->setOldPath($match['old']); } if (isset($match['hash'])) { $change->setCommitHash($match['hash']); } if (isset($match['binary'])) { $change->setFileType(ArcanistDiffChangeType::FILE_BINARY); $line = $this->nextNonemptyLine(); continue; } $line = $this->nextLine(); switch ($match['type']) { case 'Index': case 'Modified': case 'Added': case 'Deleted': case 'Copied': $this->parseIndexHunk($change); break; case 'Property changes on': $this->parsePropertyHunk($change); break; case 'diff --git': $this->setIsGit(true); $this->parseIndexHunk($change); break; case 'commit': $this->setIsGit(true); $this->parseCommitMessage($change); break; case '---': $ok = preg_match( '@^(?:\+\+\+) (.*)\s+\d{4}-\d{2}-\d{2}.*$@', $line, $match); if (!$ok) { $this->didFailParse(pht( "Expected '%s' in unified diff.", '+++ filename')); } $change->setCurrentPath($match[1]); $line = $this->nextLine(); $this->parseChangeset($change); break; case 'diff -r': $this->setIsMercurial(true); $this->parseIndexHunk($change); break; case 'rcsdiff -u': $this->isRCS = true; $this->parseIndexHunk($change); break; default: $this->didFailParse(pht('Unknown diff type.')); break; } } while ($this->getLine() !== null); $this->didFinishParse(); $this->loadSyntheticData(); return $this->changes; } protected function tryMatchHeader($patterns, $line, &$match) { foreach ($patterns as $pattern) { if (preg_match('@^'.$pattern.'$@', $line, $match)) { return true; } } return false; } protected function parseCommitMessage(ArcanistDiffChange $change) { $change->setType(ArcanistDiffChangeType::TYPE_MESSAGE); $message = array(); $line = $this->getLine(); if (preg_match('/^Merge: /', $line)) { $this->nextLine(); } $line = $this->getLine(); if (!preg_match('/^Author: /', $line)) { $this->didFailParse(pht("Expected 'Author:'.")); } $line = $this->nextLine(); if (!preg_match('/^Date: /', $line)) { $this->didFailParse(pht("Expected 'Date:'.")); } while (($line = $this->nextLineTrimmed()) !== null) { if (strlen($line) && $line[0] != ' ') { break; } // Strip leading spaces from Git commit messages. Note that empty lines // are represented as just "\n"; don't touch those. $message[] = preg_replace('/^ /', '', $this->getLine()); } $message = rtrim(implode('', $message), "\r\n"); $change->setMetadata('message', $message); } /** * Parse an SVN property change hunk. These hunks are ambiguous so just sort * of try to get it mostly right. It's entirely possible to foil this parser * (or any other parser) with a carefully constructed property change. */ protected function parsePropertyHunk(ArcanistDiffChange $change) { $line = $this->getLineTrimmed(); if (!preg_match('/^_+$/', $line)) { $this->didFailParse(pht("Expected '%s'.", '______________________')); } $line = $this->nextLine(); while ($line !== null) { $done = preg_match('/^(Index|Property changes on):/', $line); if ($done) { break; } // NOTE: Before 1.5, SVN uses "Name". At 1.5 and later, SVN uses // "Modified", "Added" and "Deleted". $matches = null; $ok = preg_match( '/^(Name|Modified|Added|Deleted): (.*)$/', $line, $matches); if (!$ok) { $this->didFailParse( pht("Expected 'Name', 'Added', 'Deleted', or 'Modified'.")); } $op = $matches[1]; $prop = $matches[2]; list($old, $new) = $this->parseSVNPropertyChange($op, $prop); if ($old !== null) { $change->setOldProperty($prop, $old); } if ($new !== null) { $change->setNewProperty($prop, $new); } $line = $this->getLine(); } } private function parseSVNPropertyChange($op, $prop) { $old = array(); $new = array(); $target = null; $line = $this->nextLine(); $prop_index = 2; while ($line !== null) { $done = preg_match( '/^(Modified|Added|Deleted|Index|Property changes on):/', $line); if ($done) { break; } $trimline = ltrim($line); if ($trimline && $trimline[0] == '#') { // in svn1.7, a line like ## -0,0 +1 ## is put between the Added: line // and the line with the property change. If we have such a line, we'll // just ignore it (: $line = $this->nextLine(); $prop_index = 1; $trimline = ltrim($line); } if ($trimline && $trimline[0] == '+') { if ($op == 'Deleted') { $this->didFailParse(pht( 'Unexpected "%s" section in property deletion.', '+')); } $target = 'new'; $line = substr($trimline, $prop_index); } else if ($trimline && $trimline[0] == '-') { if ($op == 'Added') { $this->didFailParse(pht( 'Unexpected "%s" section in property addition.', '-')); } $target = 'old'; $line = substr($trimline, $prop_index); } else if (!strncmp($trimline, 'Merged', 6)) { if ($op == 'Added') { $target = 'new'; } else { // These can appear on merges. No idea how to interpret this (unclear // what the old / new values are) and it's of dubious usefulness so // just throw it away until someone complains. $target = null; } $line = $trimline; } if ($target == 'new') { $new[] = $line; } else if ($target == 'old') { $old[] = $line; } $line = $this->nextLine(); } $old = rtrim(implode('', $old)); $new = rtrim(implode('', $new)); if (!strlen($old)) { $old = null; } if (!strlen($new)) { $new = null; } return array($old, $new); } protected function setIsGit($git) { if ($this->isGit !== null && $this->isGit != $git) { throw new Exception(pht('Git status has changed!')); } $this->isGit = $git; return $this; } protected function getIsGit() { return $this->isGit; } public function setIsMercurial($is_mercurial) { $this->isMercurial = $is_mercurial; return $this; } public function getIsMercurial() { return $this->isMercurial; } protected function parseIndexHunk(ArcanistDiffChange $change) { $is_git = $this->getIsGit(); $is_mercurial = $this->getIsMercurial(); $is_svn = (!$is_git && !$is_mercurial); $move_source = null; $line = $this->getLine(); if ($is_git) { do { $patterns = array( '(?Pnew) file mode (?P\d+)', '(?Pdeleted) file mode (?P\d+)', // These occur when someone uses `chmod` on a file. 'old mode (?P\d+)', 'new mode (?P\d+)', // These occur when you `mv` a file and git figures it out. 'similarity index ', 'rename from (?P.*)', '(?Prename) to (?P.*)', 'copy from (?P.*)', '(?Pcopy) to (?P.*)', ); $ok = false; $match = null; - foreach ($patterns as $pattern) { - $ok = preg_match('@^'.$pattern.'@', $line, $match); - if ($ok) { - break; + + if ($line !== null) { + foreach ($patterns as $pattern) { + $ok = preg_match('@^'.$pattern.'@', $line, $match); + if ($ok) { + break; + } } } if (!$ok) { if ($line === null || preg_match('/^(diff --git|commit) /', $line)) { // In this case, there are ONLY file mode changes, or this is a // pure move. If it's a move, flag these changesets so we can build // synthetic changes later, enabling us to show file contents in // Differential -- git only gives us a block like this: // // diff --git a/README b/READYOU // similarity index 100% // rename from README // rename to READYOU // // ...i.e., there is no associated diff. // This allows us to distinguish between property changes only // and actual moves. For property changes only, we can't currently // build a synthetic diff correctly, so just skip it. // TODO: Build synthetic diffs for property changes, too. if ($change->getType() != ArcanistDiffChangeType::TYPE_CHANGE) { $change->setNeedsSyntheticGitHunks(true); if ($move_source) { $move_source->setNeedsSyntheticGitHunks(true); } } return; } break; } if (!empty($match['oldmode'])) { $change->setOldProperty('unix:filemode', $match['oldmode']); } if (!empty($match['newmode'])) { $change->setNewProperty('unix:filemode', $match['newmode']); } if (!empty($match['deleted'])) { $change->setType(ArcanistDiffChangeType::TYPE_DELETE); } if (!empty($match['new'])) { // If you replace a symlink with a normal file, git renders the change // as a "delete" of the symlink plus an "add" of the new file. We // prefer to represent this as a change. if ($change->getType() == ArcanistDiffChangeType::TYPE_DELETE) { $change->setType(ArcanistDiffChangeType::TYPE_CHANGE); } else { $change->setType(ArcanistDiffChangeType::TYPE_ADD); } } if (!empty($match['old'])) { $match['old'] = self::unescapeFilename($match['old']); $change->setOldPath($match['old']); } if (!empty($match['cur'])) { $match['cur'] = self::unescapeFilename($match['cur']); $change->setCurrentPath($match['cur']); } if (!empty($match['copy'])) { $change->setType(ArcanistDiffChangeType::TYPE_COPY_HERE); $old = $this->buildChange($change->getOldPath()); $type = $old->getType(); if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY) { $old->setType(ArcanistDiffChangeType::TYPE_MULTICOPY); } else { $old->setType(ArcanistDiffChangeType::TYPE_COPY_AWAY); } $old->addAwayPath($change->getCurrentPath()); } if (!empty($match['move'])) { $change->setType(ArcanistDiffChangeType::TYPE_MOVE_HERE); $old = $this->buildChange($change->getOldPath()); $type = $old->getType(); if ($type == ArcanistDiffChangeType::TYPE_MULTICOPY) { // Great, no change. } else if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY) { $old->setType(ArcanistDiffChangeType::TYPE_MULTICOPY); } else if ($type == ArcanistDiffChangeType::TYPE_COPY_AWAY) { $old->setType(ArcanistDiffChangeType::TYPE_MULTICOPY); } else { $old->setType(ArcanistDiffChangeType::TYPE_MOVE_AWAY); } // We'll reference this above. $move_source = $old; $old->addAwayPath($change->getCurrentPath()); } $line = $this->nextNonemptyLine(); } while (true); } $line = $this->getLine(); if ($is_svn) { $ok = preg_match('/^=+\s*$/', $line); if (!$ok) { $this->didFailParse(pht( "Expected '%s' divider line.", '=======================')); } else { // Adding an empty file in SVN can produce an empty line here. $line = $this->nextNonemptyLine(); } } else if ($is_git) { $ok = preg_match('/^index .*$/', $line); if (!$ok) { // TODO: "hg diff -g" diffs ("mercurial git-style diffs") do not include // this line, so we can't parse them if we fail on it. Maybe introduce // a flag saying "parse this diff using relaxed git-style diff rules"? // $this->didFailParse("Expected 'index af23f...a98bc' header line."); } else { // NOTE: In the git case, where this patch is the last change in the // file, we may have a final terminal newline. Skip over it so that // we'll hit the '$line === null' block below. This is covered by the // 'git-empty-file.gitdiff' test case. $line = $this->nextNonemptyLine(); } } // If there are files with only whitespace changes and -b or -w are // supplied as command-line flags to `diff', svn and git both produce // changes without any body. if ($line === null || preg_match( '/^(Index:|Property changes on:|diff --git|commit) /', $line)) { return; } $is_binary_add = preg_match( '/^Cannot display: file marked as a binary type\.$/', rtrim($line)); if ($is_binary_add) { $this->nextLine(); // Cannot display: file marked as a binary type. $this->nextNonemptyLine(); // svn:mime-type = application/octet-stream $this->markBinary($change); return; } // We can get this in git, or in SVN when a file exists in the repository // WITHOUT a binary mime-type and is changed and given a binary mime-type. $is_binary_diff = preg_match( '/^(Binary files|Files) .* and .* differ$/', rtrim($line)); if ($is_binary_diff) { $this->nextNonemptyLine(); // Binary files x and y differ $this->markBinary($change); return; } // This occurs under "hg diff --git" when a binary file is removed. See // test case "hg-binary-delete.hgdiff". (I believe it never occurs under // git, which reports the "files X and /dev/null differ" string above. Git // can not apply these patches.) $is_hg_binary_delete = preg_match( '/^Binary file .* has changed$/', rtrim($line)); if ($is_hg_binary_delete) { $this->nextNonemptyLine(); $this->markBinary($change); return; } // With "git diff --binary" (not a normal mode, but one users may explicitly // invoke and then, e.g., copy-paste into the web console) or "hg diff // --git" (normal under hg workflows), we may encounter a literal binary // patch. $is_git_binary_patch = preg_match( '/^GIT binary patch$/', rtrim($line)); if ($is_git_binary_patch) { $this->nextLine(); $this->parseGitBinaryPatch(); $line = $this->getLine(); - if (preg_match('/^literal/', $line)) { + if ($line !== null && preg_match('/^literal/', $line)) { // We may have old/new binaries (change) or just a new binary (hg add). // If there are two blocks, parse both. $this->parseGitBinaryPatch(); } $this->markBinary($change); return; } if ($is_git) { // "git diff -b" ignores whitespace, but has an empty hunk target if (preg_match('@^diff --git .*$@', $line)) { $this->nextLine(); return null; } } if ($this->isRCS) { // Skip the RCS headers. $this->nextLine(); $this->nextLine(); $this->nextLine(); } $old_file = $this->parseHunkTarget(); $new_file = $this->parseHunkTarget(); if ($this->isRCS) { $change->setCurrentPath($new_file); } $change->setOldPath($old_file); $this->parseChangeset($change); } private function parseGitBinaryPatch() { // TODO: We could decode the patches, but it's a giant mess so don't bother // for now. We'll pick up the data from the working copy in the common // case ("arc diff"). $line = $this->getLine(); if (!preg_match('/^literal /', $line)) { $this->didFailParse( pht("Expected '%s' to start git binary patch.", 'literal NNNN')); } do { $line = $this->nextLineTrimmed(); if ($line === '' || $line === null) { // Some versions of Mercurial apparently omit the terminal newline, // although it's unclear if Git will ever do this. In either case, // rely on the base85 check for sanity. $this->nextNonemptyLine(); return; } else if (!preg_match('/^[a-zA-Z]/', $line)) { $this->didFailParse( pht('Expected base85 line length character (a-zA-Z).')); } } while (true); } protected function parseHunkTarget() { $line = $this->getLine(); $matches = null; $remainder = '(?:\s*\(.*\))?'; if ($this->getIsMercurial()) { // Something like "Fri Aug 26 01:20:50 2005 -0700", don't bother trying // to parse it. $remainder = '\t.*'; } else if ($this->isRCS) { $remainder = '\s.*'; } else if ($this->getIsGit()) { // When filenames contain spaces, Git terminates this line with a tab. // Normally, the tab is not present. If there's a tab, ignore it. $remainder = '(?:\t.*)?'; } $ok = preg_match( '@^[-+]{3} (?:[ab]/)?(?P.*?)'.$remainder.'$@', $line, $matches); if (!$ok) { $this->didFailParse( pht( "Expected hunk target '%s'.", '+++ path/to/file.ext (revision N)')); } $this->nextLine(); return $matches['path']; } protected function markBinary(ArcanistDiffChange $change) { $change->setFileType(ArcanistDiffChangeType::FILE_BINARY); return $this; } protected function parseChangeset(ArcanistDiffChange $change) { // If a diff includes two sets of changes to the same file, let the // second one win. In particular, this occurs when adding subdirectories // in Subversion that contain files: the file text will be present in // both the directory diff and the file diff. See T5555. Dropping the // hunks lets whichever one shows up later win instead of showing changes // twice. $change->dropHunks(); $all_changes = array(); do { $hunk = new ArcanistDiffHunk(); $line = $this->getLineTrimmed(); $real = array(); // In the case where only one line is changed, the length is omitted. // The final group is for git, which appends a guess at the function // context to the diff. $matches = null; $ok = preg_match( '/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(?: .*?)?$/U', $line, $matches); if (!$ok) { // It's possible we hit the style of an svn1.7 property change. // This is a 4-line Index block, followed by an empty line, followed // by a "Property changes on:" section similar to svn1.6. if ($line == '') { $line = $this->nextNonemptyLine(); $ok = preg_match('/^Property changes on:/', $line); if (!$ok) { $this->didFailParse(pht('Confused by empty line')); } $line = $this->nextLine(); return $this->parsePropertyHunk($change); } $this->didFailParse(pht( "Expected hunk header '%s'.", '@@ -NN,NN +NN,NN @@')); } $hunk->setOldOffset($matches[1]); $hunk->setNewOffset($matches[3]); // Cover for the cases where length wasn't present (implying one line). - $old_len = idx($matches, 2); + $old_len = idx($matches, 2, ''); if (!strlen($old_len)) { $old_len = 1; } - $new_len = idx($matches, 4); + $new_len = idx($matches, 4, ''); if (!strlen($new_len)) { $new_len = 1; } $hunk->setOldLength($old_len); $hunk->setNewLength($new_len); $add = 0; $del = 0; $hit_next_hunk = false; while ((($line = $this->nextLine()) !== null)) { if (strlen(rtrim($line, "\r\n"))) { $char = $line[0]; } else { // Normally, we do not encouter empty lines in diffs, because // unchanged lines have an initial space. However, in Git, with // the option `diff.suppress-blank-empty` set, unchanged blank lines // emit as completely empty. If we encounter a completely empty line, // treat it as a ' ' (i.e., unchanged empty line) line. $char = ' '; } switch ($char) { case '\\': if (!preg_match('@\\ No newline at end of file@', $line)) { $this->didFailParse( pht("Expected '\ No newline at end of file'.")); } if ($new_len) { $real[] = $line; $hunk->setIsMissingOldNewline(true); } else { $real[] = $line; $hunk->setIsMissingNewNewline(true); } if (!$new_len) { break 2; } break; case '+': ++$add; --$new_len; $real[] = $line; break; case '-': if (!$old_len) { // In this case, we've hit "---" from a new file. So don't // advance the line cursor. $hit_next_hunk = true; break 2; } ++$del; --$old_len; $real[] = $line; break; case ' ': if (!$old_len && !$new_len) { break 2; } --$old_len; --$new_len; $real[] = $line; break; default: // We hit something, likely another hunk. $hit_next_hunk = true; break 2; } } if ($old_len || $new_len) { $this->didFailParse(pht('Found the wrong number of hunk lines.')); } $corpus = implode('', $real); $is_binary = false; if ($this->detectBinaryFiles) { $is_binary = !phutil_is_utf8($corpus); $try_encoding = $this->tryEncoding; if ($is_binary && $try_encoding) { $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus); if (!$is_binary) { $corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding); if (!phutil_is_utf8($corpus)) { throw new Exception( pht( "Failed to convert a hunk from '%s' to UTF-8. ". "Check that the specified encoding is correct.", $try_encoding)); } } } } if ($is_binary) { // SVN happily treats binary files which aren't marked with the right // mime type as text files. Detect that junk here and mark the file // binary. We'll catch stuff with unicode too, but that's verboten // anyway. If there are too many false positives with this we might // need to make it threshold-triggered instead of triggering on any // unprintable byte. $change->setFileType(ArcanistDiffChangeType::FILE_BINARY); } else { $hunk->setCorpus($corpus); $hunk->setAddLines($add); $hunk->setDelLines($del); $change->addHunk($hunk); } if (!$hit_next_hunk) { $line = $this->nextNonemptyLine(); } - } while (preg_match('/^@@ /', $line)); + } while (($line !== null) && preg_match('/^@@ /', $line)); } protected function buildChange($path = null) { $change = null; if ($path !== null) { if (!empty($this->changes[$path])) { return $this->changes[$path]; } } if ($this->forcePath) { return $this->changes[$this->forcePath]; } $change = new ArcanistDiffChange(); if ($path !== null) { $change->setCurrentPath($path); $this->changes[$path] = $change; } else { $this->changes[] = $change; } return $change; } protected function didStartParse($text) { $this->rawDiff = $text; // Eat leading whitespace. This may happen if the first change in the diff // is an SVN property change. $text = ltrim($text); // Try to strip ANSI color codes from colorized diffs. ANSI color codes // might be present in two cases: // // - You piped a colorized diff into 'arc --raw' or similar (normally // we're able to disable colorization on diffs we control the generation // of). // - You're diffing a file which actually contains ANSI color codes. // // The former is vastly more likely, but we try to distinguish between the // two cases by testing for a color code at the beginning of a line. If // we find one, we know it's a colorized diff (since the beginning of the // line should be "+", "-" or " " if the code is in the diff text). // // While it's possible a diff might be colorized and fail this test, it's // unlikely, and it covers hg's color extension which seems to be the most // stubborn about colorizing text despite stdout not being a TTY. // // We might incorrectly strip color codes from a colorized diff of a text // file with color codes inside it, but this case is stupid and pathological // and you've dug your own grave. $ansi_color_pattern = '\x1B\[[\d;]*m'; if (preg_match('/^'.$ansi_color_pattern.'/m', $text)) { $text = preg_replace('/'.$ansi_color_pattern.'/', '', $text); } $this->text = phutil_split_lines($text); $this->line = 0; } protected function getLine() { if ($this->text === null) { throw new Exception(pht('Not parsing!')); } if (isset($this->text[$this->line])) { return $this->text[$this->line]; } return null; } protected function getLineTrimmed() { $line = $this->getLine(); if ($line !== null) { $line = trim($line, "\r\n"); } return $line; } protected function nextLine() { $this->line++; return $this->getLine(); } protected function nextLineTrimmed() { $line = $this->nextLine(); if ($line !== null) { $line = trim($line, "\r\n"); } return $line; } protected function nextNonemptyLine() { while (($line = $this->nextLine()) !== null) { if (strlen(trim($line)) !== 0) { break; } } return $this->getLine(); } protected function nextLineThatLooksLikeDiffStart() { while (($line = $this->nextLine()) !== null) { if (preg_match('/^\s*diff\s+-(?:r|-git)/', $line)) { break; } } return $this->getLine(); } protected function saveLine() { $this->lineSaved = $this->line; } protected function restoreLine() { $this->line = $this->lineSaved; } protected function isFirstNonEmptyLine() { $len = count($this->text); for ($ii = 0; $ii < $len; $ii++) { $line = $this->text[$ii]; if (!strlen(trim($line))) { // This line is empty, skip it. continue; } if (preg_match('/^#/', $line)) { // This line is a comment, skip it. continue; } return ($ii == $this->line); } // Entire file is empty. return false; } protected function didFinishParse() { $this->text = null; } public function setWriteDiffOnFailure($write) { $this->writeDiffOnFailure = $write; return $this; } protected function didFailParse($message) { $context = 5; $min = max(0, $this->line - $context); $max = min($this->line + $context, count($this->text) - 1); $context = ''; for ($ii = $min; $ii <= $max; $ii++) { $context .= sprintf( '%8.8s %6.6s %s', ($ii == $this->line) ? '>>> ' : '', $ii + 1, $this->text[$ii]); } $out = array(); $out[] = pht('Diff Parse Exception: %s', $message); if ($this->writeDiffOnFailure) { $temp = new TempFile(); $temp->setPreserveFile(true); Filesystem::writeFile($temp, $this->rawDiff); $out[] = pht('Raw input file was written to: %s', $temp); } $out[] = $context; $out = implode("\n\n", $out); throw new Exception($out); } /** * Unescape escaped filenames, e.g. from "git diff". */ private static function unescapeFilename($name) { if (preg_match('/^".+"$/', $name)) { return stripcslashes(substr($name, 1, -1)); } else { return $name; } } private function loadSyntheticData() { if (!$this->changes) { return; } $repository_api = $this->repositoryAPI; if (!$repository_api) { return; } $imagechanges = array(); $changes = $this->changes; foreach ($changes as $change) { $path = $change->getCurrentPath(); // Certain types of changes (moves and copies) don't contain change data // when expressed in raw "git diff" form. Augment any such diffs with // textual data. if ($change->getNeedsSyntheticGitHunks() && ($repository_api instanceof ArcanistGitAPI)) { $diff = $repository_api->getRawDiffText($path, $moves = false); // NOTE: We're reusing the parser and it doesn't reset change state // between parses because there's an oddball SVN workflow in Phabricator // which relies on being able to inject changes. // TODO: Fix this. $parser = clone $this; $parser->setChanges(array()); $raw_changes = $parser->parseDiff($diff); foreach ($raw_changes as $raw_change) { if ($raw_change->getCurrentPath() == $path) { $change->setFileType($raw_change->getFileType()); foreach ($raw_change->getHunks() as $hunk) { // Git thinks that this file has been added. But we know that it // has been moved or copied without a change. $hunk->setCorpus( preg_replace('/^\+/m', ' ', $hunk->getCorpus())); $change->addHunk($hunk); } break; } } $change->setNeedsSyntheticGitHunks(false); } if ($change->getFileType() != ArcanistDiffChangeType::FILE_BINARY && $change->getFileType() != ArcanistDiffChangeType::FILE_IMAGE) { continue; } $imagechanges[$path] = $change; } // Fetch the actual file contents in batches so repositories // that have slow random file accesses (i.e. mercurial) can // optimize the retrieval. $paths = array_keys($imagechanges); $filedata = $repository_api->getBulkOriginalFileData($paths); foreach ($filedata as $path => $data) { $imagechanges[$path]->setOriginalFileData($data); } $filedata = $repository_api->getBulkCurrentFileData($paths); foreach ($filedata as $path => $data) { $imagechanges[$path]->setCurrentFileData($data); } $this->changes = $changes; } /** * Extracts the common filename from two strings with differing path * prefixes as found after `diff --git`. These strings may be * quoted; if so, the filename is returned unescaped. The prefixes * default to "a/" and "b/", but may be any string -- or may be * entierly absent. This function may return "null" if the hunk * represents a file move or copy, and with pathological renames may * return an incorrect value. Such cases are expected to be * recovered by later rename detection codepaths. * * @param string Text from a diff line after "diff --git ". * @return string Filename being altered, or null for a rename. */ public static function extractGitCommonFilename($paths) { $matches = null; $paths = rtrim($paths, "\r\n"); // Try the exact same string twice in a row separated by a // space, with an optional prefix. This can hit a false // positive for moves from files like "old file old" to "file", // but such a cases will be caught by the "rename from" / // "rename to" lines. $prefix = '(?:[^/]+/)?'; $pattern = "@^(?P(?P\"?){$prefix}(?P.+)\\k)" ." " ."(?P(?P\"?){$prefix}\\k\\k)$@"; if (!preg_match($pattern, $paths, $matches)) { // A rename or some form; return null for now, and let the // "rename from" / "rename to" lines fix it up. return null; } // Use the common subpart. There may be ambiguity here: "src/file // dst/file" may _either_ be a prefix-less move, or a change with // two custom prefixes. We assume it is the latter; if it is a // rename, diff parsing will update based on the "rename from" / // "rename to" lines. // This re-assembles with the differing prefixes removed, but the // quoting from the original. Necessary so we know if we should // unescape characters from the common string. $new = $matches['newq'].$matches['common'].$matches['newq']; $new = self::unescapeFilename($new); return $new; } /** * Strip the header and footer off a `git-format-patch` diff. * * Returns a parseable normal diff and a textual commit message. */ private function stripGitFormatPatch($diff) { // We can parse this by splitting it into two pieces over and over again // along different section dividers: // // 1. Mail headers. // 2. ("\n\n") // 3. Mail body. // 4. ("---") // 5. Diff stat section. // 6. ("\n\n") // 7. Actual diff body. // 8. ("--") // 9. Patch footer. list($head, $tail) = preg_split('/^---$/m', $diff, 2); list($mail_headers, $mail_body) = explode("\n\n", $head, 2); list($body, $foot) = preg_split('/^-- ?$/m', $tail, 2); list($stat, $diff) = explode("\n\n", $body, 2); // Rebuild the commit message by putting the subject line back on top of it, // if we can find one. $matches = null; $pattern = '/^Subject: (?:\[PATCH\] )?(.*)$/mi'; if (preg_match($pattern, $mail_headers, $matches)) { $mail_body = $matches[1]."\n\n".$mail_body; $mail_body = rtrim($mail_body); } return array($mail_body, $diff); } } diff --git a/src/parser/PhutilBugtraqParser.php b/src/parser/PhutilBugtraqParser.php index 407dcb4c..708f3749 100644 --- a/src/parser/PhutilBugtraqParser.php +++ b/src/parser/PhutilBugtraqParser.php @@ -1,129 +1,129 @@ setBugtraqPattern('http://bugs.com/%BUGID%') * ->setBugtraqCaptureExpression('/[Ii]ssues?:?(\s*,?\s*#\d+)+/') * ->setBugtraqSelectExpression('/(\d+)/') * ->processCorpus($message); * * This will produce: * * Issues: http://bugs.com/123, http://bugs.com/345 * */ final class PhutilBugtraqParser extends Phobject { private $bugtraqPattern; private $bugtraqCaptureExpression; private $bugtraqSelectExpression; public function setBugtraqPattern($pattern) { $this->bugtraqPattern = $pattern; return $this; } public function setBugtraqCaptureExpression($regex) { PhutilTypeSpec::newFromString('regex')->check($regex); $this->bugtraqCaptureExpression = $regex; return $this; } public function setBugtraqSelectExpression($regex) { PhutilTypeSpec::newFromString('regex')->check($regex); $this->bugtraqSelectExpression = $regex; return $this; } public function processCorpus($corpus) { $regexp = $this->bugtraqCaptureExpression; $matches = null; $flags = PREG_SET_ORDER | PREG_OFFSET_CAPTURE; // First, find all the matching text ranges. We do this up front because // when we do the replacement pass at the end, the whole thing needs to go // in reverse order. preg_match_all($regexp, $corpus, $matches, $flags); $captures = array(); foreach ($matches as $match) { list($captured_text, $captured_offset) = $match[0]; $captures[] = array( 'text' => $captured_text, 'at' => $captured_offset, ); } // Find the actual bug IDs. If there's a selection expression, we use that // to pick bug IDs out of a larger context. For example, the syntax may be: // // Issues: 123, 124 // // In this case, "123" and "124" are the issue IDs, and could be selected // with an expression like: // // /(\d+)/ // // If there's no selection expression, we use the entire match. $bug_ids = array(); $select_regexp = $this->bugtraqSelectExpression; foreach ($captures as $capture) { $captured_text = $capture['text']; $captured_offset = $capture['at']; - if (strlen($select_regexp)) { + if ($select_regexp !== null) { $selections = null; preg_match_all( $select_regexp, $captured_text, $selections, PREG_OFFSET_CAPTURE); foreach ($selections[1] as $selection) { $bug_ids[] = array( 'text' => $selection[0], 'at' => $captured_offset + $selection[1], ); } } else { $bug_ids[] = array( 'text' => $captured_text, 'at' => $captured_offset, ); } } // Now that we have all the bug IDs to replace, replace them (in reverse // order, so the offsets don't get messed up). $bug_ids = array_reverse($bug_ids); foreach ($bug_ids as $bug) { $new_text = str_replace( '%BUGID%', $bug['text'], $this->bugtraqPattern); $corpus = substr_replace( $corpus, $new_text, $bug['at'], strlen($bug['text'])); } return $corpus; } } diff --git a/src/parser/PhutilEmailAddress.php b/src/parser/PhutilEmailAddress.php index 6acda756..d02c4bce 100644 --- a/src/parser/PhutilEmailAddress.php +++ b/src/parser/PhutilEmailAddress.php @@ -1,114 +1,115 @@ $/', $email_address, $matches)) { $display_name = trim($matches[1], '\'" '); if (strpos($matches[2], '@') !== false) { list($local_part, $domain_name) = explode('@', $matches[2], 2); } else { $local_part = $matches[2]; $domain_name = null; } } else if (preg_match('/^(.*)@(.*)$/', $email_address, $matches)) { $display_name = null; $local_part = $matches[1]; $domain_name = $matches[2]; } else { $display_name = null; $local_part = $email_address; $domain_name = null; } $this->displayName = $display_name; $this->localPart = $local_part; $this->domainName = $domain_name; } public function __toString() { $address = $this->getAddress(); if (strlen($this->displayName)) { $display_name = $this->encodeDisplayName($this->displayName); return $display_name.' <'.$address.'>'; } else { return $address; } } public function setDisplayName($display_name) { $this->displayName = $display_name; return $this; } public function getDisplayName() { return $this->displayName; } public function setLocalPart($local_part) { $this->localPart = $local_part; return $this; } public function getLocalPart() { return $this->localPart; } public function setDomainName($domain_name) { $this->domainName = $domain_name; return $this; } public function getDomainName() { return $this->domainName; } public function setAddress($address) { $parts = explode('@', $address, 2); $this->localPart = $parts[0]; if (isset($parts[1])) { $this->domainName = $parts[1]; } return $this; } public function getAddress() { $address = $this->localPart; - if (strlen($this->domainName)) { + if ($this->domainName !== null && strlen($this->domainName)) { $address .= '@'.$this->domainName; } return $address; } private function encodeDisplayName($name) { // NOTE: This is a reasonable effort based on a cursory reading of // RFC2822, but may be significantly misguided. // Newlines are not permitted, even when escaped. Discard them. $name = preg_replace("/\s*[\r\n]+\s*/", ' ', $name); // Escape double quotes and backslashes. $name = addcslashes($name, '\\"'); // Quote the string. $name = '"'.$name.'"'; return $name; } } diff --git a/src/parser/PhutilURI.php b/src/parser/PhutilURI.php index 1902cd1f..d9701fb0 100644 --- a/src/parser/PhutilURI.php +++ b/src/parser/PhutilURI.php @@ -1,559 +1,564 @@ protocol = $uri->protocol; $this->user = $uri->user; $this->pass = $uri->pass; $this->domain = $uri->domain; $this->port = $uri->port; $this->path = $uri->path; $this->query = $uri->query; $this->fragment = $uri->fragment; $this->type = $uri->type; $this->initializeQueryParams(phutil_string_cast($uri), $params); return; } $uri = phutil_string_cast($uri); $type = self::TYPE_URI; // Reject ambiguous URIs outright. Different versions of different clients // parse these in different ways. See T12526 for discussion. if (preg_match('(^[^/:]*://[^/]*[#?].*:)', $uri)) { throw new Exception( pht( 'Rejecting ambiguous URI "%s". This URI is not formatted or '. 'encoded properly.', $uri)); } $matches = null; if (preg_match('(^([^/:]*://[^/]*)(\\?.*)\z)', $uri, $matches)) { // If the URI is something like `idea://open?file=/path/to/file`, the // `parse_url()` function will parse `open?file=` as the host. This is // not the expected result. Break the URI into two pieces, stick a slash // in between them, parse that, then remove the path. See T6106. $parts = parse_url($matches[1].'/'.$matches[2]); unset($parts['path']); } else if ($this->isGitURIPattern($uri)) { // Handle Git/SCP URIs in the form "user@domain:relative/path". $user = '(?:(?P[^/@]+)@)?'; $host = '(?P[^/:]+)'; $path = ':(?P.*)'; $ok = preg_match('(^'.$user.$host.$path.'\z)', $uri, $matches); if (!$ok) { throw new Exception( pht( 'Failed to parse URI "%s" as a Git URI.', $uri)); } $parts = $matches; $parts['scheme'] = 'ssh'; $type = self::TYPE_GIT; } else { $parts = parse_url($uri); } // The parse_url() call will accept URIs with leading whitespace, but many // other tools (like git) will not. See T4913 for a specific example. If // the input string has leading whitespace, fail the parse. if ($parts) { if (ltrim($uri) != $uri) { $parts = false; } } // NOTE: `parse_url()` is very liberal about host names; fail the parse if // the host looks like garbage. In particular, we do not allow hosts which // begin with "." or "-". See T12961 for a specific attack which relied on // hosts beginning with "-". if ($parts) { $host = idx($parts, 'host', ''); if (strlen($host)) { if (!preg_match('/^[a-zA-Z0-9]+[a-zA-Z0-9\\.\\-]*\z/', $host)) { $parts = false; } } } if (!$parts) { $parts = array(); } // stringyness is to preserve API compatibility and // allow the tests to continue passing $this->protocol = idx($parts, 'scheme', ''); $this->user = rawurldecode(idx($parts, 'user', '')); $this->pass = rawurldecode(idx($parts, 'pass', '')); $this->domain = idx($parts, 'host', ''); $this->port = (string)idx($parts, 'port', ''); $this->path = idx($parts, 'path', ''); $query = idx($parts, 'query'); if ($query) { $pairs = id(new PhutilQueryStringParser()) ->parseQueryStringToPairList($query); foreach ($pairs as $pair) { list($key, $value) = $pair; $this->appendQueryParam($key, $value); } } $this->fragment = idx($parts, 'fragment', ''); $this->type = $type; $this->initializeQueryParams($uri, $params); } public function __toString() { $prefix = null; if ($this->isGitURI()) { $port = null; } else { $port = $this->port; } $domain = $this->domain; $user = $this->user; $pass = $this->pass; if (strlen($user) && strlen($pass)) { $auth = rawurlencode($user).':'.rawurlencode($pass).'@'; } else if (strlen($user)) { $auth = rawurlencode($user).'@'; } else { $auth = null; } $protocol = $this->protocol; if ($this->isGitURI()) { $protocol = null; } else { if ($auth !== null) { $protocol = nonempty($this->protocol, 'http'); } } - if (strlen($protocol) || strlen($auth) || strlen($domain)) { + $has_protocol = ($protocol !== null) && strlen($protocol); + $has_auth = ($auth !== null); + $has_domain = ($domain !== null) && strlen($domain); + $has_port = ($port !== null) && strlen($port); + + if ($has_protocol || $has_auth || $has_domain) { if ($this->isGitURI()) { $prefix = "{$auth}{$domain}"; } else { $prefix = "{$protocol}://{$auth}{$domain}"; } - if (strlen($port)) { + if ($has_port) { $prefix .= ':'.$port; } } if ($this->query) { $query = '?'.phutil_build_http_querystring_from_pairs($this->query); } else { $query = null; } if (strlen($this->getFragment())) { $fragment = '#'.$this->getFragment(); } else { $fragment = null; } $path = $this->getPath(); if ($this->isGitURI()) { if (strlen($path)) { $path = ':'.$path; } } return $prefix.$path.$query.$fragment; } /** * @deprecated */ public function setQueryParam($key, $value) { // To set, we replace the first matching key with the new value, then // remove all other matching keys. This replaces the old value and retains // the parameter order. $is_null = ($value === null); // Typecheck and cast the key before we compare it to existing keys. This // raises an early exception if the key has a bad type. list($key) = phutil_http_parameter_pair($key, ''); $found = false; foreach ($this->query as $list_key => $pair) { list($k, $v) = $pair; if ($k !== $key) { continue; } if ($found) { unset($this->query[$list_key]); continue; } $found = true; if ($is_null) { unset($this->query[$list_key]); } else { $this->insertQueryParam($key, $value, $list_key); } } $this->query = array_values($this->query); // If we didn't find an existing place to put it, add it to the end. if (!$found) { if (!$is_null) { $this->appendQueryParam($key, $value); } } return $this; } /** * @deprecated */ public function setQueryParams(array $params) { $this->query = array(); foreach ($params as $k => $v) { $this->appendQueryParam($k, $v); } return $this; } /** * @deprecated */ public function getQueryParams() { $map = array(); foreach ($this->query as $pair) { list($k, $v) = $pair; $map[$k] = $v; } return $map; } public function getQueryParamsAsMap() { $map = array(); foreach ($this->query as $pair) { list($k, $v) = $pair; if (isset($map[$k])) { throw new Exception( pht( 'Query parameters include a duplicate key ("%s") and can not be '. 'nondestructively represented as a map.', $k)); } $map[$k] = $v; } return $map; } public function getQueryParamsAsPairList() { return $this->query; } public function appendQueryParam($key, $value) { return $this->insertQueryParam($key, $value); } public function removeAllQueryParams() { $this->query = array(); return $this; } public function removeQueryParam($remove_key) { list($remove_key) = phutil_http_parameter_pair($remove_key, ''); foreach ($this->query as $idx => $pair) { list($key, $value) = $pair; if ($key !== $remove_key) { continue; } unset($this->query[$idx]); } $this->query = array_values($this->query); return $this; } public function replaceQueryParam($replace_key, $replace_value) { if ($replace_value === null) { throw new InvalidArgumentException( pht( 'Value provided to "replaceQueryParam()" for key "%s" is NULL. '. 'Use "removeQueryParam()" to remove a query parameter.', $replace_key)); } $this->removeQueryParam($replace_key); $this->appendQueryParam($replace_key, $replace_value); return $this; } private function insertQueryParam($key, $value, $idx = null) { list($key, $value) = phutil_http_parameter_pair($key, $value); if ($idx === null) { $this->query[] = array($key, $value); } else { $this->query[$idx] = array($key, $value); } return $this; } private function initializeQueryParams($uri, array $params) { $have_params = array(); foreach ($this->query as $pair) { list($key) = $pair; $have_params[$key] = true; } foreach ($params as $key => $value) { if (isset($have_params[$key])) { throw new InvalidArgumentException( pht( 'You are trying to construct an ambiguous URI: query parameter '. '"%s" is present in both the string argument ("%s") and the map '. 'argument.', $key, $uri)); } if ($value === null) { continue; } $this->appendQueryParam($key, $value); } return $this; } public function setProtocol($protocol) { $this->protocol = $protocol; return $this; } public function getProtocol() { return $this->protocol; } public function setDomain($domain) { $this->domain = $domain; return $this; } public function getDomain() { return $this->domain; } public function setPort($port) { $this->port = $port; return $this; } public function getPort() { return $this->port; } public function getPortWithProtocolDefault() { static $default_ports = array( 'http' => '80', 'https' => '443', 'ssh' => '22', ); return nonempty( $this->getPort(), idx($default_ports, $this->getProtocol()), ''); } public function setPath($path) { if ($this->isGitURI()) { // Git URIs use relative paths which do not need to begin with "/". } else { if ($this->domain && strlen($path) && $path[0] !== '/') { $path = '/'.$path; } } $this->path = $path; return $this; } public function appendPath($path) { $first = strlen($path) ? $path[0] : null; $last = strlen($this->path) ? $this->path[strlen($this->path) - 1] : null; if (!$this->path) { return $this->setPath($path); } else if ($first === '/' && $last === '/') { $path = substr($path, 1); } else if ($first !== '/' && $last !== '/') { $path = '/'.$path; } $this->path .= $path; return $this; } public function getPath() { return $this->path; } public function setFragment($fragment) { $this->fragment = $fragment; return $this; } public function getFragment() { return $this->fragment; } public function setUser($user) { $this->user = $user; return $this; } public function getUser() { return $this->user; } public function setPass($pass) { $this->pass = $pass; return $this; } public function getPass() { return $this->pass; } public function alter($key, $value) { $altered = clone $this; $altered->replaceQueryParam($key, $value); return $altered; } public function isGitURI() { return ($this->type == self::TYPE_GIT); } public function setType($type) { if ($type == self::TYPE_URI) { $path = $this->getPath(); if (strlen($path) && ($path[0] !== '/')) { // Try to catch this here because we are not allowed to throw from // inside __toString() so we don't have a reasonable opportunity to // react properly if we catch it later. throw new Exception( pht( 'Unable to convert URI "%s" into a standard URI because the '. 'path is relative. Standard URIs can not represent relative '. 'paths.', $this)); } } $this->type = $type; return $this; } public function getType() { return $this->type; } private function isGitURIPattern($uri) { $matches = null; $ok = preg_match('(^(?P[^/]+):(?P(?!//).*)\z)', $uri, $matches); if (!$ok) { return false; } $head = $matches['head']; $last = $matches['last']; // If any part of this has spaces in it, it's not a Git URI. We fail here // so we fall back and don't fail more abruptly later. if (preg_match('(\s)', $head.$last)) { return false; } // If the second part only contains digits, assume we're looking at // casually specified "domain.com:123" URI, not a Git URI pointed at an // entirely numeric relative path. if (preg_match('(^\d+\z)', $last)) { return false; } // If the first part has a "." or an "@" in it, interpret it as a domain // or a "user@host" string. if (preg_match('([.@])', $head)) { return true; } // Otherwise, interpret the URI conservatively as a "javascript:"-style // URI. This means that "localhost:path" is parsed as a normal URI instead // of a Git URI, but we can't tell which the user intends and it's safer // to treat it as a normal URI. return false; } } diff --git a/src/parser/aast/api/AASTNodeList.php b/src/parser/aast/api/AASTNodeList.php index 92f76f2a..985fda31 100644 --- a/src/parser/aast/api/AASTNodeList.php +++ b/src/parser/aast/api/AASTNodeList.php @@ -1,110 +1,116 @@ tree, $nodes); } public function selectDescendantsOfType($type_name) { return $this->selectDescendantsOfTypes(array($type_name)); } public function selectDescendantsOfTypes(array $type_names) { $results = array(); foreach ($type_names as $type_name) { foreach ($this->list as $id => $node) { $results += $node->selectDescendantsOfType($type_name)->getRawNodes(); } } return $this->newList($results); } public function getChildrenByIndex($index) { $results = array(); foreach ($this->list as $id => $node) { $child = $node->getChildByIndex($index); $results[$child->getID()] = $child; } return $this->newList($results); } public function add(AASTNodeList $list) { foreach ($list->list as $id => $node) { $this->list[$id] = $node; } $this->ids = array_keys($this->list); return $this; } public function getTokens() { $tokens = array(); foreach ($this->list as $node) { $tokens += $node->getTokens(); } return $tokens; } public function getRawNodes() { return $this->list; } public static function newFromTreeAndNodes(AASTTree $tree, array $nodes) { // We could do `assert_instances_of($nodes, 'AASTNode')` here, but doing // so imposes an observable performance penalty for linting. $obj = new AASTNodeList(); $obj->tree = $tree; $obj->list = $nodes; $obj->ids = array_keys($nodes); return $obj; } public static function newFromTree(AASTTree $tree) { $obj = new AASTNodeList(); $obj->tree = $tree; $obj->list = array(0 => $tree->getRootNode()); $obj->ids = array(0 => 0); return $obj; } /* -( Countable )---------------------------------------------------------- */ + #[\ReturnTypeWillChange] public function count() { return count($this->ids); } /* -( Iterator )----------------------------------------------------------- */ + #[\ReturnTypeWillChange] public function current() { return $this->list[$this->key()]; } + #[\ReturnTypeWillChange] public function key() { return $this->ids[$this->pos]; } + #[\ReturnTypeWillChange] public function next() { $this->pos++; } + #[\ReturnTypeWillChange] public function rewind() { $this->pos = 0; } + #[\ReturnTypeWillChange] public function valid() { return $this->pos < count($this->ids); } } diff --git a/src/serviceprofiler/PhutilServiceProfiler.php b/src/serviceprofiler/PhutilServiceProfiler.php index 7ab8a12e..9ca9b9c1 100644 --- a/src/serviceprofiler/PhutilServiceProfiler.php +++ b/src/serviceprofiler/PhutilServiceProfiler.php @@ -1,222 +1,226 @@ discardMode = true; } public function setCollectStackTraces($collect_stack_traces) { $this->collectStackTraces = $collect_stack_traces; return $this; } public function getCollectStackTraces() { return $this->collectStackTraces; } public static function getInstance() { if (empty(self::$instance)) { self::$instance = new PhutilServiceProfiler(); } return self::$instance; } public function beginServiceCall(array $data) { $data['begin'] = microtime(true); if ($this->collectStackTraces) { $trace = debug_backtrace(); $trace = PhutilErrorHandler::formatStacktrace($trace); $data['trace'] = $trace; } $id = $this->logSize++; $this->events[$id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'begin', $id, $data); } return $id; } public function endServiceCall($call_id, array $data) { $data = ($this->events[$call_id] + $data); $data['end'] = microtime(true); $data['duration'] = ($data['end'] - $data['begin']); $this->events[$call_id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'end', $call_id, $data); } if ($this->discardMode) { unset($this->events[$call_id]); } } public function getServiceCallLog() { return $this->events; } public function addListener($callback) { $this->listeners[] = $callback; } public static function installEchoListener() { $instance = self::getInstance(); $instance->addListener(array(__CLASS__, 'echoListener')); } public static function echoListener($type, $id, $data) { $is_begin = false; $is_end = false; switch ($type) { case 'begin': $is_begin = true; $mark = '>>>'; break; case 'end': $is_end = true; $mark = '<<<'; break; default: $mark = null; break; } $type = idx($data, 'type', 'mystery'); $desc = null; if ($is_begin) { switch ($type) { case 'connect': $desc = $data['database']; break; case 'query': $desc = substr($data['query'], 0, 512); break; case 'multi-query': $desc = array(); foreach ($data['queries'] as $query) { $desc[] = substr($query, 0, 256); } $desc = implode('; ', $desc); break; case 'exec': $desc = '$ '.$data['command']; break; case 'conduit': if (isset($data['size'])) { $desc = $data['method'].'() '; $desc .= pht('', new PhutilNumber($data['size'])); } else { $desc = $data['method'].'()'; } break; case 'http': if (isset($data['proxy'])) { $proxy = phutil_censor_credentials($data['proxy']); } else { $proxy = null; } $uri = phutil_censor_credentials($data['uri']); - if (strlen($proxy)) { + if ($proxy !== null) { $desc = "{$proxy} >> {$uri}"; } else { $desc = $uri; } break; case 'lint': $desc = $data['linter']; if (isset($data['paths'])) { $desc .= ' '.pht('', phutil_count($data['paths'])); } break; case 'lock': $desc = $data['name']; break; case 'event': $desc = $data['kind'].' '; $desc .= pht('', new PhutilNumber($data['count'])); break; case 'ldap': $call = idx($data, 'call', '?'); $params = array(); switch ($call) { case 'connect': $params[] = $data['host'].':'.$data['port']; break; case 'start-tls': break; case 'bind': $params[] = $data['user']; break; case 'search': $params[] = $data['dn']; $params[] = $data['query']; break; default: $params[] = '?'; break; } $desc = "{$call} (".implode(', ', $params).")"; break; } } else if ($is_end) { $desc = pht( '%s us', new PhutilNumber((int)(1000000 * $data['duration']))); } $instance = self::getInstance(); if (!$instance->zeroTime) { $instance->zeroTime = microtime(true); } $elapsed = microtime(true) - $instance->zeroTime; $console = PhutilConsole::getConsole(); $console->writeLog( "%s [%s] (+%s) <%s> %s\n", $mark, $id, pht('%s', new PhutilNumber((int)(1000 * $elapsed))), $type, self::escapeProfilerStringForDisplay($desc)); } private static function escapeProfilerStringForDisplay($string) { + if ($string === null) { + return ''; + } + // Convert tabs and newlines to spaces and collapse blocks of whitespace, // most often formatting in queries. $string = preg_replace('/\s{2,}/', ' ', $string); // Replace sequences of binary characters with printable text. We allow // some printable characters to appear in between unprintable characters // to try to collapse the entire run. $string = preg_replace( '/[\x00-\x1F\x7F-\xFF](.{0,12}[\x00-\x1F\x7F-\xFF])*/', '<...binary data...>', $string); return $string; } } diff --git a/src/unit/renderer/ArcanistUnitConsoleRenderer.php b/src/unit/renderer/ArcanistUnitConsoleRenderer.php index 6294cdee..729c4b8d 100644 --- a/src/unit/renderer/ArcanistUnitConsoleRenderer.php +++ b/src/unit/renderer/ArcanistUnitConsoleRenderer.php @@ -1,99 +1,101 @@ getResult(); $duration = ''; if ($result_code == ArcanistUnitTestResult::RESULT_PASS) { $duration = ' '.$this->formatTestDuration($result->getDuration()); } $test_name = $result->getName(); $test_namespace = $result->getNamespace(); if (strlen($test_namespace)) { $test_name = $test_namespace.'::'.$test_name; } $return = sprintf( " %s %s\n", $this->getFormattedResult($result->getResult()).$duration, $test_name); if ($result_code != ArcanistUnitTestResult::RESULT_PASS && strlen($result->getUserData())) { $return .= $result->getUserData()."\n"; } return $return; } private function getFormattedResult($result) { switch ($result) { case ArcanistUnitTestResult::RESULT_PASS: return phutil_console_format('** %s **', pht('PASS')); case ArcanistUnitTestResult::RESULT_FAIL: return phutil_console_format('** %s **', pht('FAIL')); case ArcanistUnitTestResult::RESULT_SKIP: return phutil_console_format('** %s **', pht('SKIP')); case ArcanistUnitTestResult::RESULT_BROKEN: return phutil_console_format('** %s **', pht('BROKEN')); case ArcanistUnitTestResult::RESULT_UNSOUND: return phutil_console_format( '** %s **', pht('UNSOUND')); default: return null; } } private function formatTestDuration($seconds) { // Very carefully define inclusive upper bounds on acceptable unit test // durations. Times are in milliseconds and are in increasing order. $star = "\xE2\x98\x85"; if (phutil_is_windows()) { // Fall-back to normal asterisk for Windows consoles. $star = '*'; } $acceptableness = array( 50 => "%s{$star} ", 200 => '%s ', 500 => '%s ', - INF => '%s ', ); + $least_acceptable = '%s '; + $milliseconds = $seconds * 1000; $duration = $this->formatTime($seconds); foreach ($acceptableness as $upper_bound => $formatting) { if ($milliseconds <= $upper_bound) { return phutil_console_format($formatting, $duration); } } - return phutil_console_format(end($acceptableness), $duration); + + return phutil_console_format($least_acceptable, $duration); } private function formatTime($seconds) { if ($seconds >= 60) { $minutes = floor($seconds / 60); return pht('%dm%02ds', $minutes, round($seconds % 60)); } if ($seconds >= 1) { return pht('%4.1fs', $seconds); } $milliseconds = $seconds * 1000; if ($milliseconds >= 1) { return pht('%3dms', round($milliseconds)); } return pht(' <%dms', 1); } } diff --git a/src/utils/PhutilArray.php b/src/utils/PhutilArray.php index 8656bab3..1ca02a4a 100644 --- a/src/utils/PhutilArray.php +++ b/src/utils/PhutilArray.php @@ -1,80 +1,90 @@ data = $initial_value; } /* -( Conversion )--------------------------------------------------------- */ public function toArray() { return iterator_to_array($this, true); } /* -( Countable Interface )------------------------------------------------ */ + #[\ReturnTypeWillChange] public function count() { return count($this->data); } /* -( Iterator Interface )------------------------------------------------- */ + #[\ReturnTypeWillChange] public function current() { return current($this->data); } + #[\ReturnTypeWillChange] public function key() { return key($this->data); } + #[\ReturnTypeWillChange] public function next() { return next($this->data); } + #[\ReturnTypeWillChange] public function rewind() { reset($this->data); } + #[\ReturnTypeWillChange] public function valid() { return (key($this->data) !== null); } /* -( ArrayAccess Interface )---------------------------------------------- */ + #[\ReturnTypeWillChange] public function offsetExists($key) { return array_key_exists($key, $this->data); } + #[\ReturnTypeWillChange] public function offsetGet($key) { return $this->data[$key]; } + #[\ReturnTypeWillChange] public function offsetSet($key, $value) { $this->data[$key] = $value; } + #[\ReturnTypeWillChange] public function offsetUnset($key) { unset($this->data[$key]); } } diff --git a/src/utils/PhutilCallbackFilterIterator.php b/src/utils/PhutilCallbackFilterIterator.php index 2eab78df..c1a95a4d 100644 --- a/src/utils/PhutilCallbackFilterIterator.php +++ b/src/utils/PhutilCallbackFilterIterator.php @@ -1,25 +1,26 @@ callback = $callback; } + #[\ReturnTypeWillChange] public function accept() { return call_user_func($this->callback, $this->current()); } } diff --git a/src/workflow/ArcanistDiffWorkflow.php b/src/workflow/ArcanistDiffWorkflow.php index 38aa4b62..7201a003 100644 --- a/src/workflow/ArcanistDiffWorkflow.php +++ b/src/workflow/ArcanistDiffWorkflow.php @@ -1,2896 +1,2899 @@ isRawDiffSource(); } public function requiresConduit() { return true; } public function requiresAuthentication() { return true; } public function requiresRepositoryAPI() { if (!$this->isRawDiffSource()) { return true; } return false; } public function getDiffID() { return $this->diffID; } public function getArguments() { $arguments = array( 'message' => array( 'short' => 'm', 'param' => 'message', 'help' => pht( 'When updating a revision, use the specified message instead of '. 'prompting.'), ), 'message-file' => array( 'short' => 'F', 'param' => 'file', 'paramtype' => 'file', 'help' => pht( 'When creating a revision, read revision information '. 'from this file.'), ), 'edit' => array( 'supports' => array( 'git', 'hg', ), 'nosupport' => array( 'svn' => pht('Edit revisions via the web interface when using SVN.'), ), 'help' => pht( 'When updating a revision under git, edit revision information '. 'before updating.'), ), 'raw' => array( 'help' => pht( 'Read diff from stdin, not from the working copy. This disables '. 'many Arcanist/Phabricator features which depend on having access '. 'to the working copy.'), 'conflicts' => array( 'apply-patches' => pht('%s disables lint.', '--raw'), 'never-apply-patches' => pht('%s disables lint.', '--raw'), 'create' => pht( '%s and %s both need stdin. Use %s.', '--raw', '--create', '--raw-command'), 'edit' => pht( '%s and %s both need stdin. Use %s.', '--raw', '--edit', '--raw-command'), 'raw-command' => null, ), ), 'raw-command' => array( 'param' => 'command', 'help' => pht( 'Generate diff by executing a specified command, not from the '. 'working copy. This disables many Arcanist/Phabricator features '. 'which depend on having access to the working copy.'), 'conflicts' => array( 'apply-patches' => pht('%s disables lint.', '--raw-command'), 'never-apply-patches' => pht('%s disables lint.', '--raw-command'), ), ), 'create' => array( 'help' => pht('Always create a new revision.'), 'conflicts' => array( 'edit' => pht( '%s can not be used with %s.', '--create', '--edit'), 'only' => pht( '%s can not be used with %s.', '--create', '--only'), 'update' => pht( '%s can not be used with %s.', '--create', '--update'), ), ), 'update' => array( 'param' => 'revision_id', 'help' => pht('Always update a specific revision.'), ), 'draft' => array( 'help' => pht( 'Create a draft revision so you can look over your changes before '. 'involving anyone else. Other users will not be notified about the '. 'revision until you later use "Request Review" to publish it. You '. 'can still share the draft by giving someone the link.'), 'conflicts' => array( 'edit' => null, 'only' => null, 'update' => null, ), ), 'nounit' => array( 'help' => pht('Do not run unit tests.'), ), 'nolint' => array( 'help' => pht('Do not run lint.'), 'conflicts' => array( 'apply-patches' => pht('%s suppresses lint.', '--nolint'), 'never-apply-patches' => pht('%s suppresses lint.', '--nolint'), ), ), 'only' => array( 'help' => pht( 'Instead of creating or updating a revision, only create a diff, '. 'which you may later attach to a revision.'), 'conflicts' => array( 'edit' => pht('%s does affect revisions.', '--only'), 'message' => pht('%s does not update any revision.', '--only'), ), ), 'allow-untracked' => array( 'help' => pht('Skip checks for untracked files in the working copy.'), ), 'apply-patches' => array( 'help' => pht( 'Apply patches suggested by lint to the working copy without '. 'prompting.'), 'conflicts' => array( 'never-apply-patches' => true, ), 'passthru' => array( 'lint' => true, ), ), 'never-apply-patches' => array( 'help' => pht('Never apply patches suggested by lint.'), 'conflicts' => array( 'apply-patches' => true, ), 'passthru' => array( 'lint' => true, ), ), 'amend-all' => array( 'help' => pht( 'When linting git repositories, amend HEAD with all patches '. 'suggested by lint without prompting.'), 'passthru' => array( 'lint' => true, ), ), 'amend-autofixes' => array( 'help' => pht( 'When linting git repositories, amend HEAD with autofix '. 'patches suggested by lint without prompting.'), 'passthru' => array( 'lint' => true, ), ), 'add-all' => array( 'short' => 'a', 'help' => pht( 'Automatically add all unstaged and uncommitted '. 'files to the commit.'), ), 'json' => array( 'help' => pht( 'Emit machine-readable JSON. EXPERIMENTAL! Probably does not work!'), ), 'no-amend' => array( 'help' => pht( 'Never amend commits in the working copy with lint patches.'), ), 'uncommitted' => array( 'help' => pht('Suppress warning about uncommitted changes.'), 'supports' => array( 'hg', ), ), 'verbatim' => array( 'help' => pht( 'When creating a revision, try to use the working copy commit '. 'message verbatim, without prompting to edit it. When updating a '. 'revision, update some fields from the local commit message.'), 'supports' => array( 'hg', 'git', ), 'conflicts' => array( 'update' => true, 'only' => true, 'raw' => true, 'raw-command' => true, 'message-file' => true, ), ), 'reviewers' => array( 'param' => 'usernames', 'help' => pht('When creating a revision, add reviewers.'), 'conflicts' => array( 'only' => true, 'update' => true, ), ), 'cc' => array( 'param' => 'usernames', 'help' => pht('When creating a revision, add CCs.'), 'conflicts' => array( 'only' => true, 'update' => true, ), ), 'skip-binaries' => array( 'help' => pht('Do not upload binaries (like images).'), ), 'skip-staging' => array( 'help' => pht('Do not copy changes to the staging area.'), ), 'base' => array( 'param' => 'rules', 'help' => pht('Additional rules for determining base revision.'), 'nosupport' => array( 'svn' => pht('Subversion does not use base commits.'), ), 'supports' => array('git', 'hg'), ), 'coverage' => array( 'help' => pht('Always enable coverage information.'), 'conflicts' => array( 'no-coverage' => null, ), 'passthru' => array( 'unit' => true, ), ), 'no-coverage' => array( 'help' => pht('Always disable coverage information.'), 'passthru' => array( 'unit' => true, ), ), 'browse' => array( 'help' => pht( 'After creating a diff or revision, open it in a web browser.'), ), '*' => 'paths', 'head' => array( 'param' => 'commit', 'help' => pht( 'Specify the end of the commit range. This disables many '. 'Arcanist/Phabricator features which depend on having access to '. 'the working copy.'), 'supports' => array('git'), 'nosupport' => array( 'svn' => pht('Subversion does not support commit ranges.'), 'hg' => pht('Mercurial does not support %s yet.', '--head'), ), ), ); return $arguments; } public function isRawDiffSource() { return $this->getArgument('raw') || $this->getArgument('raw-command'); } public function run() { $this->console = PhutilConsole::getConsole(); $this->runRepositoryAPISetup(); $this->runDiffSetupBasics(); $commit_message = $this->buildCommitMessage(); $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_DIDBUILDMESSAGE, array( 'message' => $commit_message, )); if (!$this->shouldOnlyCreateDiff()) { $revision = $this->buildRevisionFromCommitMessage($commit_message); } $data = $this->runLintUnit(); $lint_result = $data['lintResult']; $this->unresolvedLint = $data['unresolvedLint']; $unit_result = $data['unitResult']; $this->testResults = $data['testResults']; $changes = $this->generateChanges(); if (!$changes) { throw new ArcanistUsageException( pht('There are no changes to generate a diff from!')); } $diff_spec = array( 'changes' => mpull($changes, 'toDictionary'), 'lintStatus' => $this->getLintStatus($lint_result), 'unitStatus' => $this->getUnitStatus($unit_result), ) + $this->buildDiffSpecification(); $conduit = $this->getConduit(); $diff_info = $conduit->callMethodSynchronous( 'differential.creatediff', $diff_spec); $this->diffID = $diff_info['diffid']; $event = $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_WASCREATED, array( 'diffID' => $diff_info['diffid'], 'lintResult' => $lint_result, 'unitResult' => $unit_result, )); $this->submitChangesToStagingArea($this->diffID); $phid = idx($diff_info, 'phid'); if ($phid) { $this->hitAutotargets = $this->updateAutotargets( $phid, $unit_result); } $this->updateLintDiffProperty(); $this->updateUnitDiffProperty(); $this->updateLocalDiffProperty(); $this->updateOntoDiffProperty(); $this->resolveDiffPropertyUpdates(); $output_json = $this->getArgument('json'); if ($this->shouldOnlyCreateDiff()) { if (!$output_json) { echo phutil_console_format( "%s\n **%s** __%s__\n\n", pht('Created a new Differential diff:'), pht('Diff URI:'), $diff_info['uri']); } else { $human = ob_get_clean(); echo json_encode(array( 'diffURI' => $diff_info['uri'], 'diffID' => $this->getDiffID(), 'human' => $human, ))."\n"; ob_start(); } if ($this->shouldOpenCreatedObjectsInBrowser()) { $this->openURIsInBrowser(array($diff_info['uri'])); } } else { $is_draft = $this->getArgument('draft'); $revision['diffid'] = $this->getDiffID(); if ($commit_message->getRevisionID()) { if ($is_draft) { // TODO: In at least some cases, we could raise this earlier in the // workflow to save users some time before the workflow aborts. if ($this->revisionIsDraft) { $this->writeWarn( pht('ALREADY A DRAFT'), pht( 'You are updating a revision ("%s") with the "--draft" flag, '. 'but this revision is already a draft. You only need to '. 'provide the "--draft" flag when creating a revision. Draft '. 'revisions are not published until you explicitly request '. 'review from the web UI.', $commit_message->getRevisionMonogram())); } else { throw new ArcanistUsageException( pht( 'You are updating a revision ("%s") with the "--draft" flag, '. 'but this revision has already been published for review. '. 'You can not turn a revision back into a draft once it has '. 'been published.', $commit_message->getRevisionMonogram())); } } $result = $conduit->callMethodSynchronous( 'differential.updaterevision', $revision); foreach (array('edit-messages.json', 'update-messages.json') as $file) { $messages = $this->readScratchJSONFile($file); unset($messages[$revision['id']]); $this->writeScratchJSONFile($file, $messages); } $result_uri = $result['uri']; $result_id = $result['revisionid']; echo pht('Updated an existing Differential revision:')."\n"; } else { // NOTE: We're either using "differential.revision.edit" (preferred) // if we can, or falling back to "differential.createrevision" // (the older way) if not. $xactions = $this->revisionTransactions; if ($xactions) { $xactions[] = array( 'type' => 'update', 'value' => $diff_info['phid'], ); if ($is_draft) { $xactions[] = array( 'type' => 'draft', 'value' => true, ); } $result = $conduit->callMethodSynchronous( 'differential.revision.edit', array( 'transactions' => $xactions, )); $result_id = idxv($result, array('object', 'id')); if (!$result_id) { throw new Exception( pht( 'Expected a revision ID to be returned by '. '"differential.revision.edit".')); } // TODO: This is hacky, but we don't currently receive a URI back // from "differential.revision.edit". $result_uri = id(new PhutilURI($this->getConduitURI())) ->setPath('/D'.$result_id); } else { if ($is_draft) { throw new ArcanistUsageException( pht( 'You have specified "--draft", but the version of Phabricator '. 'on the server is too old to support draft revisions. Omit '. 'the flag or upgrade the server software.')); } $revision = $this->dispatchWillCreateRevisionEvent($revision); $result = $conduit->callMethodSynchronous( 'differential.createrevision', $revision); $result_uri = $result['uri']; $result_id = $result['revisionid']; } $revised_message = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $result_id, )); if ($this->shouldAmend()) { $repository_api = $this->getRepositoryAPI(); if ($repository_api->supportsAmend()) { echo pht('Updating commit message...')."\n"; $repository_api->amendCommit($revised_message); } else { echo pht( 'Commit message was not amended. Amending commit message is '. 'only supported in git and hg (version 2.2 or newer)'); } } echo pht('Created a new Differential revision:')."\n"; } $uri = $result_uri; echo phutil_console_format( " **%s** __%s__\n\n", pht('Revision URI:'), $uri); if ($this->shouldOpenCreatedObjectsInBrowser()) { $this->openURIsInBrowser(array($uri)); } } echo pht('Included changes:')."\n"; foreach ($changes as $change) { echo ' '.$change->renderTextSummary()."\n"; } if ($output_json) { ob_get_clean(); } $this->removeScratchFile('create-message'); return 0; } private function runRepositoryAPISetup() { if (!$this->requiresRepositoryAPI()) { return; } $repository_api = $this->getRepositoryAPI(); $repository_api->setBaseCommitArgumentRules( $this->getArgument('base', '')); if ($repository_api->supportsCommitRanges()) { $this->parseBaseCommitArgument($this->getArgument('paths')); } $head_commit = $this->getArgument('head'); if ($head_commit !== null) { $repository_api->setHeadCommit($head_commit); } } private function runDiffSetupBasics() { $output_json = $this->getArgument('json'); if ($output_json) { // TODO: We should move this to a higher-level and put an indirection // layer between echoing stuff and stdout. ob_start(); } if ($this->requiresWorkingCopy()) { $repository_api = $this->getRepositoryAPI(); if ($this->getArgument('add-all')) { $this->setCommitMode(self::COMMIT_ENABLE); } else if ($this->getArgument('uncommitted')) { $this->setCommitMode(self::COMMIT_DISABLE); } else { $this->setCommitMode(self::COMMIT_ALLOW); } if ($repository_api instanceof ArcanistSubversionAPI) { $repository_api->limitStatusToPaths($this->getArgument('paths')); } if (!$this->getArgument('head')) { $this->requireCleanWorkingCopy(); } } $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_DIDCOLLECTCHANGES, array()); } private function buildRevisionFromCommitMessage( ArcanistDifferentialCommitMessage $message) { $conduit = $this->getConduit(); $revision_id = $message->getRevisionID(); $revision = array( 'fields' => $message->getFields(), ); $xactions = $message->getTransactions(); if ($revision_id) { // With '--verbatim', pass the (possibly modified) local fields. This // allows the user to edit some fields (like "title" and "summary") // locally without '--edit' and have changes automatically synchronized. // Without '--verbatim', we do not update the revision to reflect local // commit message changes. if ($this->getArgument('verbatim')) { $use_fields = $message->getFields(); } else { $use_fields = array(); } $should_edit = $this->getArgument('edit'); $edit_messages = $this->readScratchJSONFile('edit-messages.json'); $remote_corpus = idx($edit_messages, $revision_id); if (!$should_edit || !$remote_corpus || $use_fields) { if ($this->commitMessageFromRevision) { $remote_corpus = $this->commitMessageFromRevision; } else { $remote_corpus = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $revision_id, 'edit' => 'edit', 'fields' => $use_fields, )); } } if ($should_edit) { $edited = $this->newInteractiveEditor($remote_corpus) ->setName('differential-edit-revision-info') ->setTaskMessage(pht( 'Update the details for a revision, then save and exit.')) ->editInteractively(); if ($edited != $remote_corpus) { $remote_corpus = $edited; $edit_messages[$revision_id] = $remote_corpus; $this->writeScratchJSONFile('edit-messages.json', $edit_messages); } } if ($this->commitMessageFromRevision == $remote_corpus) { $new_message = $message; } else { $remote_corpus = ArcanistCommentRemover::removeComments( $remote_corpus); $new_message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $remote_corpus); $new_message->pullDataFromConduit($conduit); } $revision['fields'] = $new_message->getFields(); $xactions = $new_message->getTransactions(); $revision['id'] = $revision_id; $this->revisionID = $revision_id; $revision['message'] = $this->getArgument('message'); - if (!strlen($revision['message'])) { + if ($revision['message'] === null) { $update_messages = $this->readScratchJSONFile('update-messages.json'); $update_messages[$revision_id] = $this->getUpdateMessage( $revision['fields'], idx($update_messages, $revision_id)); $revision['message'] = ArcanistCommentRemover::removeComments( $update_messages[$revision_id]); if (!strlen(trim($revision['message']))) { throw new ArcanistUserAbortException(); } $this->writeScratchJSONFile('update-messages.json', $update_messages); } } $this->revisionTransactions = $xactions; return $revision; } protected function shouldOnlyCreateDiff() { if ($this->getArgument('create')) { return false; } if ($this->getArgument('update')) { return false; } if ($this->isRawDiffSource()) { return true; } return $this->getArgument('only'); } private function generateAffectedPaths() { if ($this->isRawDiffSource()) { return array(); } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistSubversionAPI) { $file_list = new FileList($this->getArgument('paths', array())); $paths = $repository_api->getSVNStatus($externals = true); foreach ($paths as $path => $mask) { if (!$file_list->contains($repository_api->getPath($path), true)) { unset($paths[$path]); } } $warn_externals = array(); foreach ($paths as $path => $mask) { $any_mod = ($mask & ArcanistRepositoryAPI::FLAG_ADDED) || ($mask & ArcanistRepositoryAPI::FLAG_MODIFIED) || ($mask & ArcanistRepositoryAPI::FLAG_DELETED); if ($mask & ArcanistRepositoryAPI::FLAG_EXTERNALS) { unset($paths[$path]); if ($any_mod) { $warn_externals[] = $path; } } } if ($warn_externals && !$this->hasWarnedExternals) { echo phutil_console_format( "%s\n\n%s\n\n", pht( "The working copy includes changes to '%s' paths. These ". "changes will not be included in the diff because SVN can not ". "commit 'svn:externals' changes alongside normal changes.", 'svn:externals'), pht( "Modified '%s' files:", 'svn:externals'), phutil_console_wrap(implode("\n", $warn_externals), 8)); $prompt = pht('Generate a diff (with just local changes) anyway?'); if (!phutil_console_confirm($prompt)) { throw new ArcanistUserAbortException(); } else { $this->hasWarnedExternals = true; } } } else { $paths = $repository_api->getWorkingCopyStatus(); } foreach ($paths as $path => $mask) { if ($mask & ArcanistRepositoryAPI::FLAG_UNTRACKED) { unset($paths[$path]); } } return $paths; } protected function generateChanges() { $parser = $this->newDiffParser(); $is_raw = $this->isRawDiffSource(); if ($is_raw) { if ($this->getArgument('raw')) { fwrite(STDERR, pht('Reading diff from stdin...')."\n"); $raw_diff = file_get_contents('php://stdin'); } else if ($this->getArgument('raw-command')) { list($raw_diff) = execx('%C', $this->getArgument('raw-command')); } else { throw new Exception(pht('Unknown raw diff source.')); } $changes = $parser->parseDiff($raw_diff); foreach ($changes as $key => $change) { // Remove "message" changes, e.g. from "git show". if ($change->getType() == ArcanistDiffChangeType::TYPE_MESSAGE) { unset($changes[$key]); } } return $changes; } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistSubversionAPI) { $paths = $this->generateAffectedPaths(); $this->primeSubversionWorkingCopyData($paths); // Check to make sure the user is diffing from a consistent base revision. // This is mostly just an abuse sanity check because it's silly to do this // and makes the code more difficult to effectively review, but it also // affects patches and makes them nonportable. $bases = $repository_api->getSVNBaseRevisions(); // Remove all files with baserev "0"; these files are new. foreach ($bases as $path => $baserev) { if ($bases[$path] <= 0) { unset($bases[$path]); } } if ($bases) { $rev = reset($bases); $revlist = array(); foreach ($bases as $path => $baserev) { $revlist[] = ' '.pht('Revision %s, %s', $baserev, $path); } $revlist = implode("\n", $revlist); foreach ($bases as $path => $baserev) { if ($baserev !== $rev) { throw new ArcanistUsageException( pht( "Base revisions of changed paths are mismatched. Update all ". "paths to the same base revision before creating a diff: ". "\n\n%s", $revlist)); } } // If you have a change which affects several files, all of which are // at a consistent base revision, treat that revision as the effective // base revision. The use case here is that you made a change to some // file, which updates it to HEAD, but want to be able to change it // again without updating the entire working copy. This is a little // sketchy but it arises in Facebook Ops workflows with config files and // doesn't have any real material tradeoffs (e.g., these patches are // perfectly applyable). $repository_api->overrideSVNBaseRevisionNumber($rev); } $changes = $parser->parseSubversionDiff( $repository_api, $paths); } else if ($repository_api instanceof ArcanistGitAPI) { $diff = $repository_api->getFullGitDiff( $repository_api->getBaseCommit(), $repository_api->getHeadCommit()); if (!strlen($diff)) { throw new ArcanistUsageException( pht('No changes found. (Did you specify the wrong commit range?)')); } $changes = $parser->parseDiff($diff); } else if ($repository_api instanceof ArcanistMercurialAPI) { $diff = $repository_api->getFullMercurialDiff(); if (!strlen($diff)) { throw new ArcanistUsageException( pht('No changes found. (Did you specify the wrong commit range?)')); } $changes = $parser->parseDiff($diff); } else { throw new Exception(pht('Repository API is not supported.')); } $limit = 1024 * 1024 * 4; foreach ($changes as $change) { $size = 0; foreach ($change->getHunks() as $hunk) { $size += strlen($hunk->getCorpus()); } if ($size > $limit) { $byte_warning = pht( "Diff for '%s' with context is %s bytes in length. ". "Generally, source changes should not be this large.", $change->getCurrentPath(), new PhutilNumber($size)); if ($repository_api instanceof ArcanistSubversionAPI) { throw new ArcanistUsageException( $byte_warning.' '. pht( "If the file is not a text file, mark it as binary with:". "\n\n $ %s\n", 'svn propset svn:mime-type application/octet-stream ')); } else { $confirm = $byte_warning.' '.pht( "If the file is not a text file, you can mark it 'binary'. ". "Mark this file as 'binary' and continue?"); if (phutil_console_confirm($confirm)) { $change->convertToBinaryChange($repository_api); } else { throw new ArcanistUsageException( pht('Aborted generation of gigantic diff.')); } } } } $utf8_problems = array(); foreach ($changes as $change) { foreach ($change->getHunks() as $hunk) { $corpus = $hunk->getCorpus(); if (!phutil_is_utf8($corpus)) { // If this corpus is heuristically binary, don't try to convert it. // mb_check_encoding() and mb_convert_encoding() are both very very // liberal about what they're willing to process. $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus); if (!$is_binary) { try { $try_encoding = $this->getRepositoryEncoding(); } catch (ConduitClientException $e) { if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') { echo phutil_console_wrap( pht('Lookup of encoding in arcanist project failed: %s', $e->getMessage())."\n"); } else { throw $e; } } if ($try_encoding) { $corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding); $name = $change->getCurrentPath(); if (phutil_is_utf8($corpus)) { $this->writeStatusMessage( pht( "Converted a '%s' hunk from '%s' to UTF-8.\n", $name, $try_encoding)); $hunk->setCorpus($corpus); continue; } } } $utf8_problems[] = $change; break; } } } // If there are non-binary files which aren't valid UTF-8, warn the user // and treat them as binary changes. See D327 for discussion of why Arcanist // has this behavior. if ($utf8_problems) { $utf8_warning = sprintf( "%s\n\n%s\n\n %s\n", pht( 'This diff includes %s file(s) which are not valid UTF-8 (they '. 'contain invalid byte sequences). You can either stop this '. 'workflow and fix these files, or continue. If you continue, '. 'these files will be marked as binary.', phutil_count($utf8_problems)), pht( "You can learn more about how Phabricator handles character ". "encodings (and how to configure encoding settings and detect and ". "correct encoding problems) by reading 'User Guide: UTF-8 and ". "Character Encoding' in the Phabricator documentation."), pht( '%s AFFECTED FILE(S)', phutil_count($utf8_problems))); $confirm = pht( 'Do you want to mark these %s file(s) as binary and continue?', phutil_count($utf8_problems)); echo phutil_console_format( "**%s**\n", pht('Invalid Content Encoding (Non-UTF8)')); echo phutil_console_wrap($utf8_warning); $file_list = mpull($utf8_problems, 'getCurrentPath'); $file_list = ' '.implode("\n ", $file_list); echo $file_list; if (!phutil_console_confirm($confirm, $default_no = false)) { throw new ArcanistUsageException(pht('Aborted workflow to fix UTF-8.')); } else { foreach ($utf8_problems as $change) { $change->convertToBinaryChange($repository_api); } } } $this->uploadFilesForChanges($changes); return $changes; } private function getGitParentLogInfo() { $info = array( 'parent' => null, 'base_revision' => null, 'base_path' => null, 'uuid' => null, ); $repository_api = $this->getRepositoryAPI(); $parser = $this->newDiffParser(); $history_messages = $repository_api->getGitHistoryLog(); if (!$history_messages) { // This can occur on the initial commit. return $info; } $history_messages = $parser->parseDiff($history_messages); foreach ($history_messages as $key => $change) { try { $message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $change->getMetadata('message')); if ($message->getRevisionID() && $info['parent'] === null) { $info['parent'] = $message->getRevisionID(); } if ($message->getGitSVNBaseRevision() && $info['base_revision'] === null) { $info['base_revision'] = $message->getGitSVNBaseRevision(); $info['base_path'] = $message->getGitSVNBasePath(); } if ($message->getGitSVNUUID()) { $info['uuid'] = $message->getGitSVNUUID(); } if ($info['parent'] && $info['base_revision']) { break; } } catch (ArcanistDifferentialCommitMessageParserException $ex) { // Ignore. } catch (ArcanistUsageException $ex) { // Ignore an invalid Differential Revision field in the parent commit } } return $info; } protected function primeSubversionWorkingCopyData($paths) { $repository_api = $this->getRepositoryAPI(); $futures = array(); $targets = array(); foreach ($paths as $path => $mask) { $futures[] = $repository_api->buildDiffFuture($path); $targets[] = array('command' => 'diff', 'path' => $path); $futures[] = $repository_api->buildInfoFuture($path); $targets[] = array('command' => 'info', 'path' => $path); } $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { $target = $targets[$key]; if ($target['command'] == 'diff') { $repository_api->primeSVNDiffResult( $target['path'], $future->resolve()); } else { $repository_api->primeSVNInfoResult( $target['path'], $future->resolve()); } } } private function shouldAmend() { if ($this->isRawDiffSource()) { return false; } if ($this->getArgument('no-amend')) { return false; } if ($this->getArgument('head') !== null) { return false; } // Run this last: with --raw or --raw-command, we won't have a repository // API. if ($this->isHistoryImmutable()) { return false; } return true; } /* -( Lint and Unit Tests )------------------------------------------------ */ /** * @task lintunit */ private function runLintUnit() { $lint_result = $this->runLint(); $unit_result = $this->runUnit(); return array( 'lintResult' => $lint_result, 'unresolvedLint' => $this->unresolvedLint, 'unitResult' => $unit_result, 'testResults' => $this->testResults, ); } /** * @task lintunit */ private function runLint() { if ($this->getArgument('nolint') || $this->isRawDiffSource() || $this->getArgument('head')) { return ArcanistLintWorkflow::RESULT_SKIP; } $repository_api = $this->getRepositoryAPI(); $this->console->writeOut("%s\n", pht('Linting...')); try { $argv = $this->getPassthruArgumentsAsArgv('lint'); if ($repository_api->supportsCommitRanges()) { $argv[] = '--rev'; $argv[] = $repository_api->getBaseCommit(); } $lint_workflow = $this->buildChildWorkflow('lint', $argv); if ($this->shouldAmend()) { // TODO: We should offer to create a checkpoint commit. $lint_workflow->setShouldAmendChanges(true); } $lint_result = $lint_workflow->run(); switch ($lint_result) { case ArcanistLintWorkflow::RESULT_OKAY: $this->console->writeOut( "** %s ** %s\n", pht('LINT OKAY'), pht('No lint problems.')); break; case ArcanistLintWorkflow::RESULT_WARNINGS: $this->console->writeOut( "** %s ** %s\n", pht('LINT MESSAGES'), pht('Lint issued unresolved warnings.')); break; case ArcanistLintWorkflow::RESULT_ERRORS: $this->console->writeOut( "** %s ** %s\n", pht('LINT ERRORS'), pht('Lint raised errors!')); break; } $this->unresolvedLint = array(); foreach ($lint_workflow->getUnresolvedMessages() as $message) { $this->unresolvedLint[] = $message->toDictionary(); } return $lint_result; } catch (ArcanistNoEngineException $ex) { $this->console->writeOut( "%s\n", pht('No lint engine configured for this project.')); } catch (ArcanistNoEffectException $ex) { $this->console->writeOut("%s\n", $ex->getMessage()); } return null; } /** * @task lintunit */ private function runUnit() { if ($this->getArgument('nounit') || $this->isRawDiffSource() || $this->getArgument('head')) { return ArcanistUnitWorkflow::RESULT_SKIP; } $repository_api = $this->getRepositoryAPI(); $this->console->writeOut("%s\n", pht('Running unit tests...')); try { $argv = $this->getPassthruArgumentsAsArgv('unit'); if ($repository_api->supportsCommitRanges()) { $argv[] = '--rev'; $argv[] = $repository_api->getBaseCommit(); } $unit_workflow = $this->buildChildWorkflow('unit', $argv); $unit_result = $unit_workflow->run(); switch ($unit_result) { case ArcanistUnitWorkflow::RESULT_OKAY: $this->console->writeOut( "** %s ** %s\n", pht('UNIT OKAY'), pht('No unit test failures.')); break; case ArcanistUnitWorkflow::RESULT_UNSOUND: $continue = phutil_console_confirm( pht( 'Unit test results included failures, but all failing tests '. 'are known to be unsound. Ignore unsound test failures?')); if (!$continue) { throw new ArcanistUserAbortException(); } echo phutil_console_format( "** %s ** %s\n", pht('UNIT UNSOUND'), pht( 'Unit testing raised errors, but all '. 'failing tests are unsound.')); break; case ArcanistUnitWorkflow::RESULT_FAIL: $this->console->writeOut( "** %s ** %s\n", pht('UNIT ERRORS'), pht('Unit testing raised errors!')); break; } $this->testResults = array(); foreach ($unit_workflow->getTestResults() as $test) { $this->testResults[] = $test->toDictionary(); } return $unit_result; } catch (ArcanistNoEngineException $ex) { $this->console->writeOut( "%s\n", pht('No unit test engine is configured for this project.')); } catch (ArcanistNoEffectException $ex) { $this->console->writeOut("%s\n", $ex->getMessage()); } return null; } public function getTestResults() { return $this->testResults; } /* -( Commit and Update Messages )----------------------------------------- */ /** * @task message */ private function buildCommitMessage() { if ($this->getArgument('only')) { return null; } $is_create = $this->getArgument('create'); $is_update = $this->getArgument('update'); $is_raw = $this->isRawDiffSource(); $is_verbatim = $this->getArgument('verbatim'); if ($is_verbatim) { return $this->getCommitMessageFromUser(); } if (!$is_raw && !$is_create && !$is_update) { $repository_api = $this->getRepositoryAPI(); $revisions = $repository_api->loadWorkingCopyDifferentialRevisions( $this->getConduit(), array( 'authors' => array($this->getUserPHID()), 'status' => 'status-open', )); if (!$revisions) { $is_create = true; } else if (count($revisions) == 1) { $revision = head($revisions); $is_update = $revision['id']; } else { throw new ArcanistUsageException( pht( "There are several revisions which match the working copy:\n\n%s\n". "Use '%s' to choose one, or '%s' to create a new revision.", $this->renderRevisionList($revisions), '--update', '--create')); } } $message = null; if ($is_create) { $message_file = $this->getArgument('message-file'); if ($message_file) { return $this->getCommitMessageFromFile($message_file); } else { return $this->getCommitMessageFromUser(); } } else if ($is_update) { $revision_id = $this->normalizeRevisionID($is_update); if (!is_numeric($revision_id)) { throw new ArcanistUsageException( pht( 'Parameter to %s must be a Differential Revision number.', '--update')); } return $this->getCommitMessageFromRevision($revision_id); } else { // This is --raw without enough info to create a revision, so force just // a diff. return null; } } /** * @task message */ private function getCommitMessageFromUser() { $conduit = $this->getConduit(); $template = null; if (!$this->getArgument('verbatim')) { $saved = $this->readScratchFile('create-message'); if ($saved) { $where = $this->getReadableScratchFilePath('create-message'); $preview = explode("\n", $saved); $preview = array_shift($preview); $preview = trim($preview); $preview = id(new PhutilUTF8StringTruncator()) ->setMaximumGlyphs(64) ->truncateString($preview); if ($preview) { $preview = pht('Message begins:')."\n\n {$preview}\n\n"; } else { $preview = null; } echo pht( "You have a saved revision message in '%s'.\n%s". "You can use this message, or discard it.", $where, $preview); $use = phutil_console_confirm( pht('Do you want to use this message?'), $default_no = false); if ($use) { $template = $saved; } else { $this->removeScratchFile('create-message'); } } } $template_is_default = false; $notes = array(); $included = array(); list($fields, $notes, $included_commits) = $this->getDefaultCreateFields(); if ($template) { $fields = array(); $notes = array(); } else { if (!$fields) { $template_is_default = true; } if ($notes) { $commit = head($this->getRepositoryAPI()->getLocalCommitInformation()); $template = $commit['message']; } else { $template = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => null, 'edit' => 'create', 'fields' => $fields, )); } } $old_message = $template; $included = array(); if ($included_commits) { foreach ($included_commits as $commit) { $included[] = ' '.$commit; } if (!$this->isRawDiffSource()) { $message = pht( 'Included commits in branch %s:', $this->getRepositoryAPI()->getBranchName()); } else { $message = pht('Included commits:'); } $included = array_merge( array( '', $message, '', ), $included); } $issues = array_merge( array( pht('NEW DIFFERENTIAL REVISION'), pht('Describe the changes in this new revision.'), ), $included, array( '', pht( 'arc could not identify any existing revision in your working copy.'), pht('If you intended to update an existing revision, use:'), '', ' $ arc diff --update ', )); if ($notes) { $issues = array_merge($issues, array(''), $notes); } $done = false; $first = true; while (!$done) { $template = rtrim($template, "\r\n")."\n\n"; foreach ($issues as $issue) { $template .= rtrim('# '.$issue)."\n"; } $template .= "\n"; if ($first && $this->getArgument('verbatim') && !$template_is_default) { $new_template = $template; } else { $new_template = $this->newInteractiveEditor($template) ->setName('new-commit') ->setTaskMessage(pht( 'Provide the details for a new revision, then save and exit.')) ->editInteractively(); } $first = false; if ($template_is_default && ($new_template == $template)) { throw new ArcanistUsageException(pht('Template not edited.')); } $template = ArcanistCommentRemover::removeComments($new_template); // With --raw-command, we may not have a repository API. if ($this->hasRepositoryAPI()) { $repository_api = $this->getRepositoryAPI(); // special check for whether to amend here. optimizes a common git // workflow. we can't do this for mercurial because the mq extension // is popular and incompatible with hg commit --amend ; see T2011. $should_amend = (count($included_commits) == 1 && $repository_api instanceof ArcanistGitAPI && $this->shouldAmend()); } else { $should_amend = false; } if ($should_amend) { $wrote = (rtrim($old_message) != rtrim($template)); if ($wrote) { $repository_api->amendCommit($template); $where = pht('commit message'); } } else { $wrote = $this->writeScratchFile('create-message', $template); $where = "'".$this->getReadableScratchFilePath('create-message')."'"; } try { $message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $template); $message->pullDataFromConduit($conduit); $this->validateCommitMessage($message); $done = true; } catch (ArcanistDifferentialCommitMessageParserException $ex) { echo pht('Commit message has errors:')."\n\n"; $issues = array(pht('Resolve these errors:')); foreach ($ex->getParserErrors() as $error) { echo phutil_console_wrap("- ".$error."\n", 6); $issues[] = ' - '.$error; } echo "\n"; echo pht('You must resolve these errors to continue.'); $again = phutil_console_confirm( pht('Do you want to edit the message?'), $default_no = false); if ($again) { // Keep going. } else { $saved = null; if ($wrote) { $saved = pht('A copy was saved to %s.', $where); } throw new ArcanistUsageException( pht('Message has unresolved errors.')." {$saved}"); } } catch (Exception $ex) { if ($wrote) { echo phutil_console_wrap(pht('(Message saved to %s.)', $where)."\n"); } throw $ex; } } return $message; } /** * @task message */ private function getCommitMessageFromFile($file) { $conduit = $this->getConduit(); $data = Filesystem::readFile($file); $message = ArcanistDifferentialCommitMessage::newFromRawCorpus($data); $message->pullDataFromConduit($conduit); $this->validateCommitMessage($message); return $message; } /** * @task message */ private function getCommitMessageFromRevision($revision_id) { $id = $revision_id; $revision = $this->getConduit()->callMethodSynchronous( 'differential.query', array( 'ids' => array($id), )); $revision = head($revision); if (!$revision) { throw new ArcanistUsageException( pht( "Revision '%s' does not exist!", $revision_id)); } $this->checkRevisionOwnership($revision); // TODO: Save this status to improve a prompt later. See PHI458. This is // extra awful until we move to "differential.revision.search" because // the "differential.query" method doesn't return a real draft status for // compatibility. $this->revisionIsDraft = (idx($revision, 'statusName') === 'Draft'); $message = $this->getConduit()->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $id, 'edit' => false, )); $this->commitMessageFromRevision = $message; $obj = ArcanistDifferentialCommitMessage::newFromRawCorpus($message); $obj->pullDataFromConduit($this->getConduit()); return $obj; } /** * @task message */ private function validateCommitMessage( ArcanistDifferentialCommitMessage $message) { $futures = array(); $revision_id = $message->getRevisionID(); if ($revision_id) { $futures['revision'] = $this->getConduit()->callMethod( 'differential.query', array( 'ids' => array($revision_id), )); } $reviewers = $message->getFieldValue('reviewerPHIDs'); if ($reviewers) { $futures['reviewers'] = $this->getConduit()->callMethod( 'user.query', array( 'phids' => $reviewers, )); } foreach (new FutureIterator($futures) as $key => $future) { $result = $future->resolve(); switch ($key) { case 'revision': if (empty($result)) { throw new ArcanistUsageException( pht( 'There is no revision %s.', "D{$revision_id}")); } $this->checkRevisionOwnership(head($result)); break; case 'reviewers': $away = array(); foreach ($result as $user) { if (idx($user, 'currentStatus') != 'away') { continue; } $username = $user['userName']; $real_name = $user['realName']; if (strlen($real_name)) { $name = pht('%s (%s)', $username, $real_name); } else { $name = pht('%s', $username); } $away[] = array( 'name' => $name, 'until' => $user['currentStatusUntil'], ); } if ($away) { if (count($away) == count($reviewers)) { $earliest_return = min(ipull($away, 'until')); $message = pht( 'All reviewers are away until %s:', date('l, M j Y', $earliest_return)); } else { $message = pht('Some reviewers are currently away:'); } echo tsprintf( "%s\n\n", $message); $list = id(new PhutilConsoleList()); foreach ($away as $spec) { $list->addItem( pht( '%s (until %s)', $spec['name'], date('l, M j Y', $spec['until']))); } echo tsprintf( '%B', $list->drawConsoleString()); $confirm = pht('Continue even though reviewers are unavailable?'); if (!phutil_console_confirm($confirm)) { throw new ArcanistUsageException( pht('Specify available reviewers and retry.')); } } break; } } } /** * @task message */ private function getUpdateMessage(array $fields, $template = '') { if ($this->getArgument('raw')) { throw new ArcanistUsageException( pht( "When using '%s' to update a revision, specify an update message ". "with '%s'. (Normally, we'd launch an editor to ask you for a ". "message, but can not do that because stdin is the diff source.)", '--raw', '--message')); } // When updating a revision using git without specifying '--message', try // to prefill with the message in HEAD if it isn't a template message. The // idea is that if you do: // // $ git commit -a -m 'fix some junk' // $ arc diff // // ...you shouldn't have to retype the update message. Similar things apply // to Mercurial. if ($template == '') { $comments = $this->getDefaultUpdateMessage(); + $comments = phutil_string_cast($comments); + $comments = rtrim($comments); + $template = sprintf( "%s\n\n# %s\n#\n# %s\n# %s\n#\n# %s\n# $ %s\n\n", - rtrim($comments), + $comments, pht( 'Updating %s: %s', "D{$fields['revisionID']}", $fields['title']), pht( 'Enter a brief description of the changes included in this update.'), pht('The first line is used as subject, next lines as comment.'), pht('If you intended to create a new revision, use:'), 'arc diff --create'); } $comments = $this->newInteractiveEditor($template) ->setName('differential-update-comments') ->setTaskMessage(pht( 'Update the revision comments, then save and exit.')) ->editInteractively(); return $comments; } private function getDefaultCreateFields() { $result = array(array(), array(), array()); if ($this->isRawDiffSource()) { return $result; } $repository_api = $this->getRepositoryAPI(); $local = $repository_api->getLocalCommitInformation(); if ($local) { $result = $this->parseCommitMessagesIntoFields($local); if ($this->getArgument('create')) { unset($result[0]['revisionID']); } } $result[0] = $this->dispatchWillBuildEvent($result[0]); return $result; } /** * Convert a list of commits from `getLocalCommitInformation()` into * a format usable by arc to create a new diff. Specifically, we emit: * * - A dictionary of commit message fields. * - A list of errors encountered while parsing the messages. * - A human-readable list of the commits themselves. * * For example, if the user runs "arc diff HEAD^^^" and selects a diff range * which includes several diffs, we attempt to merge them somewhat * intelligently into a single message, because we can only send one * "Summary:", "Reviewers:", etc., field to Differential. We also return * errors (e.g., if the user typed a reviewer name incorrectly) and a * summary of the commits themselves. * * @param dict Local commit information. * @return list Complex output, see summary. * @task message */ private function parseCommitMessagesIntoFields(array $local) { $conduit = $this->getConduit(); $local = ipull($local, null, 'commit'); // If the user provided "--reviewers" or "--ccs", add a faux message to // the list with the implied fields. $faux_message = array(); if ($this->getArgument('reviewers')) { $faux_message[] = pht('Reviewers: %s', $this->getArgument('reviewers')); } if ($this->getArgument('cc')) { $faux_message[] = pht('CC: %s', $this->getArgument('cc')); } // NOTE: For now, this isn't a real field, so it just ends up as the first // part of the summary. $depends_ref = $this->getDependsOnRevisionRef(); if ($depends_ref) { $faux_message[] = pht( 'Depends on %s. ', $depends_ref->getMonogram()); } // See T12069. After T10312, the first line of a message is always parsed // as a title. Add a placeholder so "Reviewers" and "CC" are never the // first line. $placeholder_title = pht(''); if ($faux_message) { array_unshift($faux_message, $placeholder_title); $faux_message = implode("\n\n", $faux_message); $local = array( '(Flags) ' => array( 'message' => $faux_message, 'summary' => pht('Command-Line Flags'), ), ) + $local; } // Build a human-readable list of the commits, so we can show the user which // commits are included in the diff. $included = array(); foreach ($local as $hash => $info) { $included[] = substr($hash, 0, 12).' '.$info['summary']; } // Parse all of the messages into fields. $messages = array(); foreach ($local as $hash => $info) { $text = $info['message']; $obj = ArcanistDifferentialCommitMessage::newFromRawCorpus($text); $messages[$hash] = $obj; } $notes = array(); $fields = array(); foreach ($messages as $hash => $message) { try { $message->pullDataFromConduit($conduit, $partial = true); $fields[$hash] = $message->getFields(); } catch (ArcanistDifferentialCommitMessageParserException $ex) { if ($this->getArgument('verbatim')) { // In verbatim mode, just bail when we hit an error. The user can // rerun without --verbatim if they want to fix it manually. Most // users will probably `git commit --amend` instead. throw $ex; } $fields[$hash] = $message->getFields(); $frev = substr($hash, 0, 12); $notes[] = pht( 'NOTE: commit %s could not be completely parsed:', $frev); foreach ($ex->getParserErrors() as $error) { $notes[] = " - {$error}"; } } } // Merge commit message fields. We do this somewhat-intelligently so that // multiple "Reviewers" or "CC" fields will merge into the concatenation // of all values. // We have special parsing rules for 'title' because we can't merge // multiple titles, and one-line commit messages like "fix stuff" will // parse as titles. Instead, pick the first title we encounter. When we // encounter subsequent titles, treat them as part of the summary. Then // we merge all the summaries together below. $result = array(); // Process fields in oldest-first order, so earlier commits get to set the // title of record and reviewers/ccs are listed in chronological order. $fields = array_reverse($fields); foreach ($fields as $hash => $dict) { $title = idx($dict, 'title'); if (!strlen($title)) { continue; } if ($title === $placeholder_title) { continue; } if (!isset($result['title'])) { // We don't have a title yet, so use this one. $result['title'] = $title; } else { // We already have a title, so merge this new title into the summary. $summary = idx($dict, 'summary'); if ($summary) { $summary = $title."\n\n".$summary; } else { $summary = $title; } $fields[$hash]['summary'] = $summary; } } // Now, merge all the other fields in a general sort of way. foreach ($fields as $hash => $dict) { foreach ($dict as $key => $value) { if ($key == 'title') { // This has been handled above, and either assigned directly or // merged into the summary. continue; } if (is_array($value)) { // For array values, merge the arrays, appending the new values. // Examples are "Reviewers" and "Cc", where this produces a list of // all users specified as reviewers. $cur = idx($result, $key, array()); $new = array_merge($cur, $value); $result[$key] = $new; continue; } else { if (!strlen(trim($value))) { // Ignore empty fields. continue; } // For string values, append the new field to the old field with // a blank line separating them. Examples are "Test Plan" and // "Summary". $cur = idx($result, $key, ''); if (strlen($cur)) { $new = $cur."\n\n".$value; } else { $new = $value; } $result[$key] = $new; } } } return array($result, $notes, $included); } private function getDefaultUpdateMessage() { if ($this->isRawDiffSource()) { return null; } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistGitAPI) { return $this->getGitUpdateMessage(); } if ($repository_api instanceof ArcanistMercurialAPI) { return $this->getMercurialUpdateMessage(); } return null; } /** * Retrieve the git messages between HEAD and the last update. * * @task message */ private function getGitUpdateMessage() { $repository_api = $this->getRepositoryAPI(); $parser = $this->newDiffParser(); $commit_messages = $repository_api->getGitCommitLog(); $commit_messages = $parser->parseDiff($commit_messages); if (count($commit_messages) == 1) { // If there's only one message, assume this is an amend-based workflow and // that using it to prefill doesn't make sense. return null; } // We have more than one message, so figure out which ones are new. We // do this by pulling the current diff and comparing commit hashes in the // working copy with attached commit hashes. It's not super important that // we always get this 100% right, we're just trying to do something // reasonable. $hashes = $this->loadActiveDiffLocalCommitHashes(); $hashes = array_fuse($hashes); $usable = array(); foreach ($commit_messages as $message) { $text = $message->getMetadata('message'); $parsed = ArcanistDifferentialCommitMessage::newFromRawCorpus($text); if ($parsed->getRevisionID()) { // If this is an amended commit message with a revision ID, it's // certainly not new. Stop marking commits as usable and break out. break; } if (isset($hashes[$message->getCommitHash()])) { // If this commit is currently part of the diff, stop using commit // messages, since anything older than this isn't new. break; } // Otherwise, this looks new, so it's a usable commit message. $usable[] = $text; } if (!$usable) { // No new commit messages, so we don't have anywhere to start from. return null; } return $this->formatUsableLogs($usable); } /** * Retrieve the hg messages between tip and the last update. * * @task message */ private function getMercurialUpdateMessage() { $repository_api = $this->getRepositoryAPI(); $messages = $repository_api->getCommitMessageLog(); if (count($messages) == 1) { // If there's only one message, assume this is an amend-based workflow and // that using it to prefill doesn't make sense. return null; } $hashes = $this->loadActiveDiffLocalCommitHashes(); $hashes = array_fuse($hashes); $usable = array(); foreach ($messages as $rev => $message) { if (isset($hashes[$rev])) { // If this commit is currently part of the active diff on the revision, // stop using commit messages, since anything older than this isn't new. break; } // Otherwise, this looks new, so it's a usable commit message. $usable[] = $message; } if (!$usable) { // No new commit messages, so we don't have anywhere to start from. return null; } return $this->formatUsableLogs($usable); } /** * Format log messages to prefill a diff update. * * @task message */ private function formatUsableLogs(array $usable) { // Flip messages so they'll read chronologically (oldest-first) in the // template, e.g.: // // - Added foobar. // - Fixed foobar bug. // - Documented foobar. $usable = array_reverse($usable); $default = array(); foreach ($usable as $message) { // Pick the first line out of each message. $text = trim($message); $text = head(explode("\n", $text)); $default[] = ' - '.$text."\n"; } return implode('', $default); } private function loadActiveDiffLocalCommitHashes() { // The older "differential.querydiffs" method includes the full diff text, // which can be very slow for large diffs. If we can, try to use // "differential.diff.search" instead. // We expect this to fail if the Phabricator version on the server is // older than April 2018 (D19386), which introduced the "commits" // attachment for "differential.revision.search". // TODO: This can be optimized if we're able to learn the "revisionPHID" // before we get here. See PHI1104. try { $revisions_raw = $this->getConduit()->callMethodSynchronous( 'differential.revision.search', array( 'constraints' => array( 'ids' => array( $this->revisionID, ), ), )); $revisions = $revisions_raw['data']; $revision = head($revisions); if ($revision) { $revision_phid = $revision['phid']; $diffs_raw = $this->getConduit()->callMethodSynchronous( 'differential.diff.search', array( 'constraints' => array( 'revisionPHIDs' => array( $revision_phid, ), ), 'attachments' => array( 'commits' => true, ), 'limit' => 1, )); $diffs = $diffs_raw['data']; $diff = head($diffs); if ($diff) { $commits = idxv($diff, array('attachments', 'commits', 'commits')); if ($commits !== null) { $hashes = ipull($commits, 'identifier'); return array_values($hashes); } } } } catch (Exception $ex) { // If any of this fails, fall back to the older method below. } $current_diff = $this->getConduit()->callMethodSynchronous( 'differential.querydiffs', array( 'revisionIDs' => array($this->revisionID), )); $current_diff = head($current_diff); $properties = idx($current_diff, 'properties', array()); $local = idx($properties, 'local:commits', array()); $hashes = ipull($local, 'commit'); return array_values($hashes); } /* -( Diff Specification )------------------------------------------------- */ /** * @task diffspec */ private function getLintStatus($lint_result) { $map = array( ArcanistLintWorkflow::RESULT_OKAY => 'okay', ArcanistLintWorkflow::RESULT_ERRORS => 'fail', ArcanistLintWorkflow::RESULT_WARNINGS => 'warn', ArcanistLintWorkflow::RESULT_SKIP => 'skip', ); return idx($map, $lint_result, 'none'); } /** * @task diffspec */ private function getUnitStatus($unit_result) { $map = array( ArcanistUnitWorkflow::RESULT_OKAY => 'okay', ArcanistUnitWorkflow::RESULT_FAIL => 'fail', ArcanistUnitWorkflow::RESULT_UNSOUND => 'warn', ArcanistUnitWorkflow::RESULT_SKIP => 'skip', ); return idx($map, $unit_result, 'none'); } /** * @task diffspec */ private function buildDiffSpecification() { $base_revision = null; $base_path = null; $vcs = null; $repo_uuid = null; $parent = null; $source_path = null; $branch = null; $bookmark = null; if (!$this->isRawDiffSource()) { $repository_api = $this->getRepositoryAPI(); $base_revision = $repository_api->getSourceControlBaseRevision(); $base_path = $repository_api->getSourceControlPath(); $vcs = $repository_api->getSourceControlSystemName(); $source_path = $repository_api->getPath(); $branch = $repository_api->getBranchName(); $repo_uuid = $repository_api->getRepositoryUUID(); if ($repository_api instanceof ArcanistGitAPI) { $info = $this->getGitParentLogInfo(); if ($info['parent']) { $parent = $info['parent']; } if ($info['base_revision']) { $base_revision = $info['base_revision']; } if ($info['base_path']) { $base_path = $info['base_path']; } if ($info['uuid']) { $repo_uuid = $info['uuid']; } } else if ($repository_api instanceof ArcanistMercurialAPI) { $bookmark = $repository_api->getActiveBookmark(); $svn_info = $repository_api->getSubversionInfo(); $repo_uuid = idx($svn_info, 'uuid'); $base_path = idx($svn_info, 'base_path', $base_path); $base_revision = idx($svn_info, 'base_revision', $base_revision); // TODO: provide parent info } } $data = array( 'sourceMachine' => php_uname('n'), 'sourcePath' => $source_path, 'branch' => $branch, 'bookmark' => $bookmark, 'sourceControlSystem' => $vcs, 'sourceControlPath' => $base_path, 'sourceControlBaseRevision' => $base_revision, 'creationMethod' => 'arc', ); if (!$this->isRawDiffSource()) { $repository_phid = $this->getRepositoryPHID(); if ($repository_phid) { $data['repositoryPHID'] = $repository_phid; } } return $data; } /* -( Diff Properties )---------------------------------------------------- */ /** * Update lint information for the diff. * * @return void * * @task diffprop */ private function updateLintDiffProperty() { if (!$this->hitAutotargets) { if ($this->unresolvedLint) { $this->updateDiffProperty( 'arc:lint', json_encode($this->unresolvedLint)); } } } /** * Update unit test information for the diff. * * @return void * * @task diffprop */ private function updateUnitDiffProperty() { if (!$this->hitAutotargets) { if ($this->testResults) { $this->updateDiffProperty('arc:unit', json_encode($this->testResults)); } } } /** * Update local commit information for the diff. * * @task diffprop */ private function updateLocalDiffProperty() { if ($this->isRawDiffSource()) { return; } $local_info = $this->getRepositoryAPI()->getLocalCommitInformation(); if (!$local_info) { return; } $this->updateDiffProperty('local:commits', json_encode($local_info)); } private function updateOntoDiffProperty() { $onto = $this->getDiffOntoTargets(); if (!$onto) { return; } $this->updateDiffProperty('arc:onto', json_encode($onto)); } private function getDiffOntoTargets() { if ($this->isRawDiffSource()) { return null; } $api = $this->getRepositoryAPI(); if (!($api instanceof ArcanistGitAPI)) { return null; } // If we track an upstream branch either directly or indirectly, use that. $branch = $api->getBranchName(); if (strlen($branch)) { $upstream_path = $api->getPathToUpstream($branch); $remote_branch = $upstream_path->getRemoteBranchName(); - if (strlen($remote_branch)) { + if ($remote_branch !== null) { return array( array( 'type' => 'branch', 'name' => $remote_branch, 'kind' => 'upstream', ), ); } } // If "arc.land.onto.default" is configured, use that. $config_key = 'arc.land.onto.default'; $onto = $this->getConfigFromAnySource($config_key); - if (strlen($onto)) { + if ($onto !== null) { return array( array( 'type' => 'branch', 'name' => $onto, 'kind' => 'arc.land.onto.default', ), ); } return null; } /** * Update an arbitrary diff property. * * @param string Diff property name. * @param string Diff property value. * @return void * * @task diffprop */ private function updateDiffProperty($name, $data) { $this->diffPropertyFutures[] = $this->getConduit()->callMethod( 'differential.setdiffproperty', array( 'diff_id' => $this->getDiffID(), 'name' => $name, 'data' => $data, )); } /** * Wait for finishing all diff property updates. * * @return void * * @task diffprop */ private function resolveDiffPropertyUpdates() { id(new FutureIterator($this->diffPropertyFutures)) ->resolveAll(); $this->diffPropertyFutures = array(); } private function dispatchWillCreateRevisionEvent(array $fields) { $event = $this->dispatchEvent( ArcanistEventType::TYPE_REVISION_WILLCREATEREVISION, array( 'specification' => $fields, )); return $event->getValue('specification'); } private function dispatchWillBuildEvent(array $fields) { $event = $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_WILLBUILDMESSAGE, array( 'fields' => $fields, )); return $event->getValue('fields'); } private function checkRevisionOwnership(array $revision) { if ($revision['authorPHID'] == $this->getUserPHID()) { return; } $id = $revision['id']; $title = $revision['title']; $prompt = pht( "You don't own revision %s: \"%s\". Normally, you should ". "only update revisions you own. You can \"Commandeer\" this revision ". "from the web interface if you want to become the owner.\n\n". "Update this revision anyway?", "D{$id}", $title); $ok = phutil_console_confirm($prompt, $default_no = true); if (!$ok) { throw new ArcanistUsageException( pht('Aborted update of revision: You are not the owner.')); } } /* -( File Uploads )------------------------------------------------------- */ private function uploadFilesForChanges(array $changes) { assert_instances_of($changes, 'ArcanistDiffChange'); // Collect all the files we need to upload. $need_upload = array(); foreach ($changes as $key => $change) { if ($change->getFileType() != ArcanistDiffChangeType::FILE_BINARY) { continue; } if ($this->getArgument('skip-binaries')) { continue; } $name = basename($change->getCurrentPath()); $need_upload[] = array( 'type' => 'old', 'name' => $name, 'data' => $change->getOriginalFileData(), 'change' => $change, ); $need_upload[] = array( 'type' => 'new', 'name' => $name, 'data' => $change->getCurrentFileData(), 'change' => $change, ); } if (!$need_upload) { return; } // Determine mime types and file sizes. Update changes from "binary" to // "image" if the file is an image. Set image metadata. $type_image = ArcanistDiffChangeType::FILE_IMAGE; foreach ($need_upload as $key => $spec) { $change = $need_upload[$key]['change']; if ($spec['data'] === null) { // This covers the case where a file was added or removed; we don't // need to upload the other half of it (e.g., the old file data for // a file which was just added). This is distinct from an empty // file, which we do upload. unset($need_upload[$key]); continue; } $type = $spec['type']; $size = strlen($spec['data']); $change->setMetadata("{$type}:file:size", $size); $mime = $this->getFileMimeType($spec['data']); if (preg_match('@^image/@', $mime)) { $change->setFileType($type_image); } $change->setMetadata("{$type}:file:mime-type", $mime); } $uploader = id(new ArcanistFileUploader()) ->setConduitEngine($this->getConduitEngine()); foreach ($need_upload as $key => $spec) { $ref = id(new ArcanistFileDataRef()) ->setName($spec['name']) ->setData($spec['data']); $uploader->addFile($ref, $key); } $files = $uploader->uploadFiles(); $errors = false; foreach ($files as $key => $file) { if ($file->getErrors()) { unset($files[$key]); $errors = true; echo pht( 'Failed to upload binary "%s".', $file->getName()); } } if ($errors) { $prompt = pht('Continue?'); $ok = phutil_console_confirm($prompt, $default_no = false); if (!$ok) { throw new ArcanistUsageException( pht( 'Aborted due to file upload failure. You can use %s '. 'to skip binary uploads.', '--skip-binaries')); } } foreach ($files as $key => $file) { $spec = $need_upload[$key]; $phid = $file->getPHID(); $change = $spec['change']; $type = $spec['type']; $change->setMetadata("{$type}:binary-phid", $phid); echo pht('Uploaded binary data for "%s".', $file->getName())."\n"; } echo pht('Upload complete.')."\n"; } private function getFileMimeType($data) { $tmp = new TempFile(); Filesystem::writeFile($tmp, $data); return Filesystem::getMimeType($tmp); } private function shouldOpenCreatedObjectsInBrowser() { return $this->getArgument('browse'); } private function submitChangesToStagingArea($id) { $result = $this->pushChangesToStagingArea($id); // We'll either get a failure constant on error, or a list of pushed // refs on success. $ok = is_array($result); if ($ok) { $staging = array( 'status' => self::STAGING_PUSHED, 'refs' => $result, ); } else { $staging = array( 'status' => $result, 'refs' => array(), ); } $this->updateDiffProperty( 'arc.staging', phutil_json_encode($staging)); } private function pushChangesToStagingArea($id) { if ($this->getArgument('skip-staging')) { $this->writeInfo( pht('SKIP STAGING'), pht('Flag --skip-staging was specified.')); return self::STAGING_USER_SKIP; } if ($this->isRawDiffSource()) { $this->writeInfo( pht('SKIP STAGING'), pht('Raw changes can not be pushed to a staging area.')); return self::STAGING_DIFF_RAW; } if (!$this->getRepositoryPHID()) { $this->writeInfo( pht('SKIP STAGING'), pht('Unable to determine repository for this change.')); return self::STAGING_REPOSITORY_UNKNOWN; } $staging = $this->getRepositoryStagingConfiguration(); if ($staging === null) { $this->writeInfo( pht('SKIP STAGING'), pht('The server does not support staging areas.')); return self::STAGING_REPOSITORY_UNAVAILABLE; } $supported = idx($staging, 'supported'); if (!$supported) { $this->writeInfo( pht('SKIP STAGING'), pht('Phabricator does not support staging areas for this repository.')); return self::STAGING_REPOSITORY_UNSUPPORTED; } $staging_uri = idx($staging, 'uri'); if (!$staging_uri) { $this->writeInfo( pht('SKIP STAGING'), pht('No staging area is configured for this repository.')); return self::STAGING_REPOSITORY_UNCONFIGURED; } $api = $this->getRepositoryAPI(); if (!($api instanceof ArcanistGitAPI)) { $this->writeInfo( pht('SKIP STAGING'), pht('This client version does not support staging this repository.')); return self::STAGING_CLIENT_UNSUPPORTED; } $commit = $api->getHeadCommit(); $prefix = idx($staging, 'prefix', 'phabricator'); $base_tag = "refs/tags/{$prefix}/base/{$id}"; $diff_tag = "refs/tags/{$prefix}/diff/{$id}"; $this->writeOkay( pht('PUSH STAGING'), pht('Pushing changes to staging area...')); $push_flags = array(); if (version_compare($api->getGitVersion(), '1.8.2', '>=')) { $push_flags[] = '--no-verify'; } $refs = array(); $remote = array( 'uri' => $staging_uri, ); $is_lfs = $api->isGitLFSWorkingCopy(); // If the base commit is a real commit, we're going to push it. We don't // use this, but pushing it to a ref reduces the amount of redundant work // that Git does on later pushes by helping it figure out that the remote // already has most of the history. See T10509. // In the future, we could avoid this push if the staging area is the same // as the main repository, or if the staging area is a virtual repository. // In these cases, the staging area should automatically have up-to-date // refs. $base_commit = $api->getSourceControlBaseRevision(); if ($base_commit !== ArcanistGitAPI::GIT_MAGIC_ROOT_COMMIT) { $refs[] = array( 'ref' => $base_tag, 'type' => 'base', 'commit' => $base_commit, 'remote' => $remote, ); } // We're always going to push the change itself. $refs[] = array( 'ref' => $diff_tag, 'type' => 'diff', 'commit' => $is_lfs ? $base_commit : $commit, 'remote' => $remote, ); $ref_list = array(); foreach ($refs as $ref) { $ref_list[] = $ref['commit'].':'.$ref['ref']; } $err = phutil_passthru( 'git push %Ls -- %s %Ls', $push_flags, $staging_uri, $ref_list); if ($err) { $this->writeWarn( pht('STAGING FAILED'), pht('Unable to push changes to the staging area.')); throw new ArcanistUsageException( pht( 'Failed to push changes to staging area. Correct the issue, or '. 'use --skip-staging to skip this step.')); } if ($is_lfs) { $ref = '+'.$commit.':'.$diff_tag; $err = phutil_passthru( 'git push -- %s %s', $staging_uri, $ref); if ($err) { $this->writeWarn( pht('STAGING FAILED'), pht('Unable to push lfs changes to the staging area.')); throw new ArcanistUsageException( pht( 'Failed to push lfs changes to staging area. Correct the issue, '. 'or use --skip-staging to skip this step.')); } } return $refs; } /** * Try to upload lint and unit test results into modern Harbormaster build * targets. * * @return bool True if everything was uploaded to build targets. */ private function updateAutotargets($diff_phid, $unit_result) { $lint_key = 'arcanist.lint'; $unit_key = 'arcanist.unit'; try { $result = $this->getConduit()->callMethodSynchronous( 'harbormaster.queryautotargets', array( 'objectPHID' => $diff_phid, 'targetKeys' => array( $lint_key, $unit_key, ), )); $targets = idx($result, 'targetMap', array()); } catch (Exception $ex) { return false; } $futures = array(); $lint_target = idx($targets, $lint_key); if ($lint_target) { $lint = nonempty($this->unresolvedLint, array()); foreach ($lint as $key => $message) { $lint[$key] = $this->getModernLintDictionary($message); } // Consider this target to have failed if there are any unresolved // errors or warnings. $type = 'pass'; foreach ($lint as $message) { switch (idx($message, 'severity')) { case ArcanistLintSeverity::SEVERITY_WARNING: case ArcanistLintSeverity::SEVERITY_ERROR: $type = 'fail'; break; } } $futures[] = $this->getConduit()->callMethod( 'harbormaster.sendmessage', array( 'buildTargetPHID' => $lint_target, 'lint' => array_values($lint), 'type' => $type, )); } $unit_target = idx($targets, $unit_key); if ($unit_target) { $unit = nonempty($this->testResults, array()); foreach ($unit as $key => $message) { $unit[$key] = $this->getModernUnitDictionary($message); } $type = ArcanistUnitWorkflow::getHarbormasterTypeFromResult($unit_result); $futures[] = $this->getConduit()->callMethod( 'harbormaster.sendmessage', array( 'buildTargetPHID' => $unit_target, 'unit' => array_values($unit), 'type' => $type, )); } try { foreach (new FutureIterator($futures) as $future) { $future->resolve(); } return true; } catch (Exception $ex) { // TODO: Eventually, we should expect these to succeed if we get this // far, but just log errors for now. phlog($ex); return false; } } private function getDependsOnRevisionRef() { // TODO: Restore this behavior after updating for toolsets. Loading the // required hardpoints currently depends on a "WorkingCopy" existing. return null; $api = $this->getRepositoryAPI(); $base_ref = $api->getBaseCommitRef(); $state_ref = id(new ArcanistWorkingCopyStateRef()) ->setCommitRef($base_ref); $this->loadHardpoints( $state_ref, ArcanistWorkingCopyStateRef::HARDPOINT_REVISIONREFS); $revision_refs = $state_ref->getRevisionRefs(); $viewer_phid = $this->getUserPHID(); foreach ($revision_refs as $key => $revision_ref) { // Don't automatically depend on closed revisions. if ($revision_ref->isClosed()) { unset($revision_refs[$key]); continue; } // Don't automatically depend on revisions authored by other users. if ($revision_ref->getAuthorPHID() != $viewer_phid) { unset($revision_refs[$key]); continue; } } if (!$revision_refs) { return null; } if (count($revision_refs) > 1) { return null; } return head($revision_refs); } } diff --git a/src/workflow/ArcanistWorkflow.php b/src/workflow/ArcanistWorkflow.php index 43ed0c0e..85b917b7 100644 --- a/src/workflow/ArcanistWorkflow.php +++ b/src/workflow/ArcanistWorkflow.php @@ -1,2470 +1,2470 @@ toolset = $toolset; return $this; } final public function getToolset() { return $this->toolset; } final public function setRuntime(ArcanistRuntime $runtime) { $this->runtime = $runtime; return $this; } final public function getRuntime() { return $this->runtime; } final public function setConfigurationEngine( ArcanistConfigurationEngine $engine) { $this->configurationEngine = $engine; return $this; } final public function getConfigurationEngine() { return $this->configurationEngine; } final public function setConfigurationSourceList( ArcanistConfigurationSourceList $list) { $this->configurationSourceList = $list; return $this; } final public function getConfigurationSourceList() { return $this->configurationSourceList; } public function newPhutilWorkflow() { $arguments = $this->getWorkflowArguments(); assert_instances_of($arguments, 'ArcanistWorkflowArgument'); $specs = mpull($arguments, 'getPhutilSpecification'); $phutil_workflow = id(new ArcanistPhutilWorkflow()) ->setName($this->getWorkflowName()) ->setWorkflow($this) ->setArguments($specs); $information = $this->getWorkflowInformation(); if ($information !== null) { if (!($information instanceof ArcanistWorkflowInformation)) { throw new Exception( pht( 'Expected workflow ("%s", of class "%s") to return an '. '"ArcanistWorkflowInformation" object from call to '. '"getWorkflowInformation()", got %s.', $this->getWorkflowName(), get_class($this), phutil_describe_type($information))); } } if ($information) { $synopsis = $information->getSynopsis(); if ($synopsis !== null) { $phutil_workflow->setSynopsis($synopsis); } $examples = $information->getExamples(); if ($examples) { $examples = implode("\n", $examples); $phutil_workflow->setExamples($examples); } $help = $information->getHelp(); - if (strlen($help)) { + if ($help !== null) { // Unwrap linebreaks in the help text so we don't get weird formatting. $help = preg_replace("/(?<=\S)\n(?=\S)/", ' ', $help); $phutil_workflow->setHelp($help); } } return $phutil_workflow; } final public function newLegacyPhutilWorkflow() { $phutil_workflow = id(new ArcanistPhutilWorkflow()) ->setName($this->getWorkflowName()); $arguments = $this->getArguments(); $specs = array(); foreach ($arguments as $key => $argument) { if ($key == '*') { $key = $argument; $argument = array( 'wildcard' => true, ); } unset($argument['paramtype']); unset($argument['supports']); unset($argument['nosupport']); unset($argument['passthru']); unset($argument['conflict']); $spec = array( 'name' => $key, ) + $argument; $specs[] = $spec; } $phutil_workflow->setArguments($specs); $synopses = $this->getCommandSynopses(); $phutil_workflow->setSynopsis($synopses); $help = $this->getCommandHelp(); if (strlen($help)) { $phutil_workflow->setHelp($help); } return $phutil_workflow; } final protected function newWorkflowArgument($key) { return id(new ArcanistWorkflowArgument()) ->setKey($key); } final protected function newWorkflowInformation() { return new ArcanistWorkflowInformation(); } final public function executeWorkflow(PhutilArgumentParser $args) { $runtime = $this->getRuntime(); $this->arguments = $args; $caught = null; $runtime->pushWorkflow($this); try { $err = $this->runWorkflow($args); } catch (Exception $ex) { $caught = $ex; } try { $this->runWorkflowCleanup(); } catch (Exception $ex) { phlog($ex); } $runtime->popWorkflow(); if ($caught) { throw $caught; } return $err; } final public function getLogEngine() { return $this->getRuntime()->getLogEngine(); } protected function runWorkflowCleanup() { // TOOLSETS: Do we need this? return; } public function __construct() {} public function run() { throw new PhutilMethodNotImplementedException(); } /** * Finalizes any cleanup operations that need to occur regardless of * whether the command succeeded or failed. */ public function finalize() { $this->finalizeWorkingCopy(); } /** * Return the command used to invoke this workflow from the command like, * e.g. "help" for @{class:ArcanistHelpWorkflow}. * * @return string The command a user types to invoke this workflow. */ abstract public function getWorkflowName(); /** * Return console formatted string with all command synopses. * * @return string 6-space indented list of available command synopses. */ public function getCommandSynopses() { return array(); } /** * Return console formatted string with command help printed in `arc help`. * * @return string 10-space indented help to use the command. */ public function getCommandHelp() { return null; } public function supportsToolset(ArcanistToolset $toolset) { return false; } /* -( Conduit )------------------------------------------------------------ */ /** * Set the URI which the workflow will open a conduit connection to when * @{method:establishConduit} is called. Arcanist makes an effort to set * this by default for all workflows (by reading ##.arcconfig## and/or the * value of ##--conduit-uri##) even if they don't need Conduit, so a workflow * can generally upgrade into a conduit workflow later by just calling * @{method:establishConduit}. * * You generally should not need to call this method unless you are * specifically overriding the default URI. It is normally sufficient to * just invoke @{method:establishConduit}. * * NOTE: You can not call this after a conduit has been established. * * @param string The URI to open a conduit to when @{method:establishConduit} * is called. * @return this * @task conduit */ final public function setConduitURI($conduit_uri) { if ($this->conduit) { throw new Exception( pht( 'You can not change the Conduit URI after a '. 'conduit is already open.')); } $this->conduitURI = $conduit_uri; return $this; } /** * Returns the URI the conduit connection within the workflow uses. * * @return string * @task conduit */ final public function getConduitURI() { return $this->conduitURI; } /** * Open a conduit channel to the server which was previously configured by * calling @{method:setConduitURI}. Arcanist will do this automatically if * the workflow returns ##true## from @{method:requiresConduit}, or you can * later upgrade a workflow and build a conduit by invoking it manually. * * You must establish a conduit before you can make conduit calls. * * NOTE: You must call @{method:setConduitURI} before you can call this * method. * * @return this * @task conduit */ final public function establishConduit() { if ($this->conduit) { return $this; } if (!$this->conduitURI) { throw new Exception( pht( 'You must specify a Conduit URI with %s before you can '. 'establish a conduit.', 'setConduitURI()')); } $this->conduit = new ConduitClient($this->conduitURI); if ($this->conduitTimeout) { $this->conduit->setTimeout($this->conduitTimeout); } return $this; } final public function getConfigFromAnySource($key) { $source_list = $this->getConfigurationSourceList(); if ($source_list) { $value_list = $source_list->getStorageValueList($key); if ($value_list) { return last($value_list)->getValue(); } return null; } return $this->configurationManager->getConfigFromAnySource($key); } /** * Set credentials which will be used to authenticate against Conduit. These * credentials can then be used to establish an authenticated connection to * conduit by calling @{method:authenticateConduit}. Arcanist sets some * defaults for all workflows regardless of whether or not they return true * from @{method:requireAuthentication}, based on the ##~/.arcrc## and * ##.arcconf## files if they are present. Thus, you can generally upgrade a * workflow which does not require authentication into an authenticated * workflow by later invoking @{method:requireAuthentication}. You should not * normally need to call this method unless you are specifically overriding * the defaults. * * NOTE: You can not call this method after calling * @{method:authenticateConduit}. * * @param dict A credential dictionary, see @{method:authenticateConduit}. * @return this * @task conduit */ final public function setConduitCredentials(array $credentials) { if ($this->isConduitAuthenticated()) { throw new Exception( pht('You may not set new credentials after authenticating conduit.')); } $this->conduitCredentials = $credentials; return $this; } /** * Get the protocol version the client should identify with. * * @return int Version the client should claim to be. * @task conduit */ final public function getConduitVersion() { return 6; } /** * Open and authenticate a conduit connection to a Phabricator server using * provided credentials. Normally, Arcanist does this for you automatically * when you return true from @{method:requiresAuthentication}, but you can * also upgrade an existing workflow to one with an authenticated conduit * by invoking this method manually. * * You must authenticate the conduit before you can make authenticated conduit * calls (almost all calls require authentication). * * This method uses credentials provided via @{method:setConduitCredentials} * to authenticate to the server: * * - ##user## (required) The username to authenticate with. * - ##certificate## (required) The Conduit certificate to use. * - ##description## (optional) Description of the invoking command. * * Successful authentication allows you to call @{method:getUserPHID} and * @{method:getUserName}, as well as use the client you access with * @{method:getConduit} to make authenticated calls. * * NOTE: You must call @{method:setConduitURI} and * @{method:setConduitCredentials} before you invoke this method. * * @return this * @task conduit */ final public function authenticateConduit() { if ($this->isConduitAuthenticated()) { return $this; } $this->establishConduit(); $credentials = $this->conduitCredentials; try { if (!$credentials) { throw new Exception( pht( 'Set conduit credentials with %s before authenticating conduit!', 'setConduitCredentials()')); } // If we have `token`, this server supports the simpler, new-style // token-based authentication. Use that instead of all the certificate // stuff. $token = idx($credentials, 'token'); if (strlen($token)) { $conduit = $this->getConduit(); $conduit->setConduitToken($token); try { $result = $this->getConduit()->callMethodSynchronous( 'user.whoami', array()); $this->userName = $result['userName']; $this->userPHID = $result['phid']; $this->conduitAuthenticated = true; return $this; } catch (Exception $ex) { $conduit->setConduitToken(null); throw $ex; } } if (empty($credentials['user'])) { throw new ConduitClientException( 'ERR-INVALID-USER', pht('Empty user in credentials.')); } if (empty($credentials['certificate'])) { throw new ConduitClientException( 'ERR-NO-CERTIFICATE', pht('Empty certificate in credentials.')); } $description = idx($credentials, 'description', ''); $user = $credentials['user']; $certificate = $credentials['certificate']; $connection = $this->getConduit()->callMethodSynchronous( 'conduit.connect', array( 'client' => 'arc', 'clientVersion' => $this->getConduitVersion(), 'clientDescription' => php_uname('n').':'.$description, 'user' => $user, 'certificate' => $certificate, 'host' => $this->conduitURI, )); } catch (ConduitClientException $ex) { if ($ex->getErrorCode() == 'ERR-NO-CERTIFICATE' || $ex->getErrorCode() == 'ERR-INVALID-USER' || $ex->getErrorCode() == 'ERR-INVALID-AUTH') { $conduit_uri = $this->conduitURI; $message = phutil_console_format( "\n%s\n\n %s\n\n%s\n%s", pht('YOU NEED TO __INSTALL A CERTIFICATE__ TO LOGIN TO PHABRICATOR'), pht('To do this, run: **%s**', 'arc install-certificate'), pht("The server '%s' rejected your request:", $conduit_uri), $ex->getMessage()); throw new ArcanistUsageException($message); } else if ($ex->getErrorCode() == 'NEW-ARC-VERSION') { // Cleverly disguise this as being AWESOME!!! echo phutil_console_format("**%s**\n\n", pht('New Version Available!')); echo phutil_console_wrap($ex->getMessage()); echo "\n\n"; echo pht('In most cases, arc can be upgraded automatically.')."\n"; $ok = phutil_console_confirm( pht('Upgrade arc now?'), $default_no = false); if (!$ok) { throw $ex; } $root = dirname(phutil_get_library_root('arcanist')); chdir($root); $err = phutil_passthru('%s upgrade', $root.'/bin/arc'); if (!$err) { echo "\n".pht('Try running your arc command again.')."\n"; } exit(1); } else { throw $ex; } } $this->userName = $user; $this->userPHID = $connection['userPHID']; $this->conduitAuthenticated = true; return $this; } /** * @return bool True if conduit is authenticated, false otherwise. * @task conduit */ final protected function isConduitAuthenticated() { return (bool)$this->conduitAuthenticated; } /** * Override this to return true if your workflow requires a conduit channel. * Arc will build the channel for you before your workflow executes. This * implies that you only need an unauthenticated channel; if you need * authentication, override @{method:requiresAuthentication}. * * @return bool True if arc should build a conduit channel before running * the workflow. * @task conduit */ public function requiresConduit() { return false; } /** * Override this to return true if your workflow requires an authenticated * conduit channel. This implies that it requires a conduit. Arc will build * and authenticate the channel for you before the workflow executes. * * @return bool True if arc should build an authenticated conduit channel * before running the workflow. * @task conduit */ public function requiresAuthentication() { return false; } /** * Returns the PHID for the user once they've authenticated via Conduit. * * @return phid Authenticated user PHID. * @task conduit */ final public function getUserPHID() { if (!$this->userPHID) { $workflow = get_class($this); throw new Exception( pht( "This workflow ('%s') requires authentication, override ". "%s to return true.", $workflow, 'requiresAuthentication()')); } return $this->userPHID; } /** * Return the username for the user once they've authenticated via Conduit. * * @return string Authenticated username. * @task conduit */ final public function getUserName() { return $this->userName; } /** * Get the established @{class@libphutil:ConduitClient} in order to make * Conduit method calls. Before the client is available it must be connected, * either implicitly by making @{method:requireConduit} or * @{method:requireAuthentication} return true, or explicitly by calling * @{method:establishConduit} or @{method:authenticateConduit}. * * @return @{class@libphutil:ConduitClient} Live conduit client. * @task conduit */ final public function getConduit() { if (!$this->conduit) { $workflow = get_class($this); throw new Exception( pht( "This workflow ('%s') requires a Conduit, override ". "%s to return true.", $workflow, 'requiresConduit()')); } return $this->conduit; } final public function setArcanistConfiguration( ArcanistConfiguration $arcanist_configuration) { $this->arcanistConfiguration = $arcanist_configuration; return $this; } final public function getArcanistConfiguration() { return $this->arcanistConfiguration; } final public function setConfigurationManager( ArcanistConfigurationManager $arcanist_configuration_manager) { $this->configurationManager = $arcanist_configuration_manager; return $this; } final public function getConfigurationManager() { return $this->configurationManager; } public function requiresWorkingCopy() { return false; } public function desiresWorkingCopy() { return false; } public function requiresRepositoryAPI() { return false; } public function desiresRepositoryAPI() { return false; } final public function setCommand($command) { $this->command = $command; return $this; } final public function getCommand() { return $this->command; } public function getArguments() { return array(); } final public function setWorkingDirectory($working_directory) { $this->workingDirectory = $working_directory; return $this; } final public function getWorkingDirectory() { return $this->workingDirectory; } private function setParentWorkflow($parent_workflow) { $this->parentWorkflow = $parent_workflow; return $this; } final protected function getParentWorkflow() { return $this->parentWorkflow; } final public function buildChildWorkflow($command, array $argv) { $arc_config = $this->getArcanistConfiguration(); $workflow = $arc_config->buildWorkflow($command); $workflow->setParentWorkflow($this); $workflow->setConduitEngine($this->getConduitEngine()); $workflow->setCommand($command); $workflow->setConfigurationManager($this->getConfigurationManager()); if ($this->repositoryAPI) { $workflow->setRepositoryAPI($this->repositoryAPI); } if ($this->userPHID) { $workflow->userPHID = $this->getUserPHID(); $workflow->userName = $this->getUserName(); } if ($this->conduit) { $workflow->conduit = $this->conduit; $workflow->setConduitCredentials($this->conduitCredentials); $workflow->conduitAuthenticated = $this->conduitAuthenticated; } $workflow->setArcanistConfiguration($arc_config); $workflow->parseArguments(array_values($argv)); return $workflow; } final public function getArgument($key, $default = null) { // TOOLSETS: Remove this legacy code. if (is_array($this->arguments)) { return idx($this->arguments, $key, $default); } return $this->arguments->getArg($key); } final public function getCompleteArgumentSpecification() { $spec = $this->getArguments(); $arc_config = $this->getArcanistConfiguration(); $command = $this->getCommand(); $spec += $arc_config->getCustomArgumentsForCommand($command); return $spec; } final public function parseArguments(array $args) { $spec = $this->getCompleteArgumentSpecification(); $dict = array(); $more_key = null; if (!empty($spec['*'])) { $more_key = $spec['*']; unset($spec['*']); $dict[$more_key] = array(); } $short_to_long_map = array(); foreach ($spec as $long => $options) { if (!empty($options['short'])) { $short_to_long_map[$options['short']] = $long; } } foreach ($spec as $long => $options) { if (!empty($options['repeat'])) { $dict[$long] = array(); } } $more = array(); $size = count($args); for ($ii = 0; $ii < $size; $ii++) { $arg = $args[$ii]; $arg_name = null; $arg_key = null; if ($arg == '--') { $more = array_merge( $more, array_slice($args, $ii + 1)); break; } else if (!strncmp($arg, '--', 2)) { $arg_key = substr($arg, 2); $parts = explode('=', $arg_key, 2); if (count($parts) == 2) { list($arg_key, $val) = $parts; array_splice($args, $ii, 1, array('--'.$arg_key, $val)); $size++; } if (!array_key_exists($arg_key, $spec)) { $corrected = PhutilArgumentSpellingCorrector::newFlagCorrector() ->correctSpelling($arg_key, array_keys($spec)); if (count($corrected) == 1) { PhutilConsole::getConsole()->writeErr( pht( "(Assuming '%s' is the British spelling of '%s'.)", '--'.$arg_key, '--'.head($corrected))."\n"); $arg_key = head($corrected); } else { throw new ArcanistUsageException( pht( "Unknown argument '%s'. Try '%s'.", $arg_key, 'arc help')); } } } else if (!strncmp($arg, '-', 1)) { $arg_key = substr($arg, 1); if (empty($short_to_long_map[$arg_key])) { throw new ArcanistUsageException( pht( "Unknown argument '%s'. Try '%s'.", $arg_key, 'arc help')); } $arg_key = $short_to_long_map[$arg_key]; } else { $more[] = $arg; continue; } $options = $spec[$arg_key]; if (empty($options['param'])) { $dict[$arg_key] = true; } else { if ($ii == $size - 1) { throw new ArcanistUsageException( pht( "Option '%s' requires a parameter.", $arg)); } if (!empty($options['repeat'])) { $dict[$arg_key][] = $args[$ii + 1]; } else { $dict[$arg_key] = $args[$ii + 1]; } $ii++; } } if ($more) { if ($more_key) { $dict[$more_key] = $more; } else { $example = reset($more); throw new ArcanistUsageException( pht( "Unrecognized argument '%s'. Try '%s'.", $example, 'arc help')); } } foreach ($dict as $key => $value) { if (empty($spec[$key]['conflicts'])) { continue; } foreach ($spec[$key]['conflicts'] as $conflict => $more) { if (isset($dict[$conflict])) { if ($more) { $more = ': '.$more; } else { $more = '.'; } // TODO: We'll always display these as long-form, when the user might // have typed them as short form. throw new ArcanistUsageException( pht( "Arguments '%s' and '%s' are mutually exclusive", "--{$key}", "--{$conflict}").$more); } } } $this->arguments = $dict; $this->didParseArguments(); return $this; } protected function didParseArguments() { // Override this to customize workflow argument behavior. } final public function getWorkingCopy() { $configuration_engine = $this->getConfigurationEngine(); // TOOLSETS: Remove this once all workflows are toolset workflows. if (!$configuration_engine) { throw new Exception( pht( 'This workflow has not yet been updated to Toolsets and can '. 'not retrieve a modern WorkingCopy object. Use '. '"getWorkingCopyIdentity()" to retrieve a previous-generation '. 'object.')); } return $configuration_engine->getWorkingCopy(); } final public function getWorkingCopyIdentity() { $configuration_engine = $this->getConfigurationEngine(); if ($configuration_engine) { $working_copy = $configuration_engine->getWorkingCopy(); $working_path = $working_copy->getWorkingDirectory(); return ArcanistWorkingCopyIdentity::newFromPath($working_path); } $working_copy = $this->getConfigurationManager()->getWorkingCopyIdentity(); if (!$working_copy) { $workflow = get_class($this); throw new Exception( pht( "This workflow ('%s') requires a working copy, override ". "%s to return true.", $workflow, 'requiresWorkingCopy()')); } return $working_copy; } final public function setRepositoryAPI($api) { $this->repositoryAPI = $api; return $this; } final public function hasRepositoryAPI() { try { return (bool)$this->getRepositoryAPI(); } catch (Exception $ex) { return false; } } final public function getRepositoryAPI() { $configuration_engine = $this->getConfigurationEngine(); if ($configuration_engine) { $working_copy = $configuration_engine->getWorkingCopy(); return $working_copy->getRepositoryAPI(); } if (!$this->repositoryAPI) { $workflow = get_class($this); throw new Exception( pht( "This workflow ('%s') requires a Repository API, override ". "%s to return true.", $workflow, 'requiresRepositoryAPI()')); } return $this->repositoryAPI; } final protected function shouldRequireCleanUntrackedFiles() { return empty($this->arguments['allow-untracked']); } final public function setCommitMode($mode) { $this->commitMode = $mode; return $this; } final public function finalizeWorkingCopy() { if ($this->stashed) { $api = $this->getRepositoryAPI(); $api->unstashChanges(); echo pht('Restored stashed changes to the working directory.')."\n"; } } final public function requireCleanWorkingCopy() { $api = $this->getRepositoryAPI(); $must_commit = array(); $working_copy_desc = phutil_console_format( " %s: __%s__\n\n", pht('Working copy'), $api->getPath()); // NOTE: this is a subversion-only concept. $incomplete = $api->getIncompleteChanges(); if ($incomplete) { throw new ArcanistUsageException( sprintf( "%s\n\n%s %s\n %s\n\n%s", pht( "You have incompletely checked out directories in this working ". "copy. Fix them before proceeding.'"), $working_copy_desc, pht('Incomplete directories in working copy:'), implode("\n ", $incomplete), pht( "You can fix these paths by running '%s' on them.", 'svn update'))); } $conflicts = $api->getMergeConflicts(); if ($conflicts) { throw new ArcanistUsageException( sprintf( "%s\n\n%s %s\n %s", pht( 'You have merge conflicts in this working copy. Resolve merge '. 'conflicts before proceeding.'), $working_copy_desc, pht('Conflicts in working copy:'), implode("\n ", $conflicts))); } $missing = $api->getMissingChanges(); if ($missing) { throw new ArcanistUsageException( sprintf( "%s\n\n%s %s\n %s\n", pht( 'You have missing files in this working copy. Revert or formally '. 'remove them (with `%s`) before proceeding.', 'svn rm'), $working_copy_desc, pht('Missing files in working copy:'), implode("\n ", $missing))); } $externals = $api->getDirtyExternalChanges(); // TODO: This state can exist in Subversion, but it is currently handled // elsewhere. It should probably be handled here, eventually. if ($api instanceof ArcanistSubversionAPI) { $externals = array(); } if ($externals) { $message = pht( '%s submodule(s) have uncommitted or untracked changes:', new PhutilNumber(count($externals))); $prompt = pht( 'Ignore the changes to these %s submodule(s) and continue?', new PhutilNumber(count($externals))); $list = id(new PhutilConsoleList()) ->setWrap(false) ->addItems($externals); id(new PhutilConsoleBlock()) ->addParagraph($message) ->addList($list) ->draw(); $ok = phutil_console_confirm($prompt, $default_no = false); if (!$ok) { throw new ArcanistUserAbortException(); } } $uncommitted = $api->getUncommittedChanges(); $unstaged = $api->getUnstagedChanges(); // We already dealt with externals. $unstaged = array_diff($unstaged, $externals); // We only want files which are purely uncommitted. $uncommitted = array_diff($uncommitted, $unstaged); $uncommitted = array_diff($uncommitted, $externals); $untracked = $api->getUntrackedChanges(); if (!$this->shouldRequireCleanUntrackedFiles()) { $untracked = array(); } if ($untracked) { echo sprintf( "%s\n\n%s", pht('You have untracked files in this working copy.'), $working_copy_desc); if ($api instanceof ArcanistGitAPI) { $hint = pht( '(To ignore these %s change(s), add them to "%s".)', phutil_count($untracked), '.git/info/exclude'); } else if ($api instanceof ArcanistSubversionAPI) { $hint = pht( '(To ignore these %s change(s), add them to "%s".)', phutil_count($untracked), 'svn:ignore'); } else if ($api instanceof ArcanistMercurialAPI) { $hint = pht( '(To ignore these %s change(s), add them to "%s".)', phutil_count($untracked), '.hgignore'); } $untracked_list = " ".implode("\n ", $untracked); echo sprintf( " %s\n %s\n%s", pht('Untracked changes in working copy:'), $hint, $untracked_list); $prompt = pht( 'Ignore these %s untracked file(s) and continue?', phutil_count($untracked)); if (!phutil_console_confirm($prompt)) { throw new ArcanistUserAbortException(); } } $should_commit = false; if ($unstaged || $uncommitted) { // NOTE: We're running this because it builds a cache and can take a // perceptible amount of time to arrive at an answer, but we don't want // to pause in the middle of printing the output below. $this->getShouldAmend(); echo sprintf( "%s\n\n%s", pht('You have uncommitted changes in this working copy.'), $working_copy_desc); $lists = array(); if ($unstaged) { $unstaged_list = " ".implode("\n ", $unstaged); $lists[] = sprintf( " %s\n%s", pht('Unstaged changes in working copy:'), $unstaged_list); } if ($uncommitted) { $uncommitted_list = " ".implode("\n ", $uncommitted); $lists[] = sprintf( "%s\n%s", pht('Uncommitted changes in working copy:'), $uncommitted_list); } echo implode("\n\n", $lists)."\n"; $all_uncommitted = array_merge($unstaged, $uncommitted); if ($this->askForAdd($all_uncommitted)) { if ($unstaged) { $api->addToCommit($unstaged); } $should_commit = true; } else { $permit_autostash = $this->getConfigFromAnySource('arc.autostash'); if ($permit_autostash && $api->canStashChanges()) { echo pht( 'Stashing uncommitted changes. (You can restore them with `%s`).', 'git stash pop')."\n"; $api->stashChanges(); $this->stashed = true; } else { throw new ArcanistUsageException( pht( 'You can not continue with uncommitted changes. '. 'Commit or discard them before proceeding.')); } } } if ($should_commit) { if ($this->getShouldAmend()) { $commit = head($api->getLocalCommitInformation()); $api->amendCommit($commit['message']); } else if ($api->supportsLocalCommits()) { $template = sprintf( "\n\n# %s\n#\n# %s\n#\n", pht('Enter a commit message.'), pht('Changes:')); $paths = array_merge($uncommitted, $unstaged); $paths = array_unique($paths); sort($paths); foreach ($paths as $path) { $template .= "# ".$path."\n"; } $commit_message = $this->newInteractiveEditor($template) ->setName(pht('commit-message')) ->setTaskMessage(pht( 'Supply commit message for uncommitted changes, then save and '. 'exit.')) ->editInteractively(); if ($commit_message === $template) { throw new ArcanistUsageException( pht('You must provide a commit message.')); } $commit_message = ArcanistCommentRemover::removeComments( $commit_message); if (!strlen($commit_message)) { throw new ArcanistUsageException( pht('You must provide a nonempty commit message.')); } $api->doCommit($commit_message); } } } private function getShouldAmend() { if ($this->shouldAmend === null) { $this->shouldAmend = $this->calculateShouldAmend(); } return $this->shouldAmend; } private function calculateShouldAmend() { $api = $this->getRepositoryAPI(); if ($this->isHistoryImmutable() || !$api->supportsAmend()) { return false; } $commits = $api->getLocalCommitInformation(); if (!$commits) { return false; } $commit = reset($commits); $message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $commit['message']); if ($message->getGitSVNBaseRevision()) { return false; } if ($api->getAuthor() != $commit['author']) { return false; } if ($message->getRevisionID() && $this->getArgument('create')) { return false; } // TODO: Check commits since tracking branch. If empty then return false. // Don't amend the current commit if it has already been published. $repository = $this->loadProjectRepository(); if ($repository) { $repo_id = $repository['id']; $commit_hash = $commit['commit']; $callsign = idx($repository, 'callsign'); if ($callsign) { // The server might be too old to support the new style commit names, // so prefer the old way $commit_name = "r{$callsign}{$commit_hash}"; } else { $commit_name = "R{$repo_id}:{$commit_hash}"; } $result = $this->getConduit()->callMethodSynchronous( 'diffusion.querycommits', array('names' => array($commit_name))); $known_commit = idx($result['identifierMap'], $commit_name); if ($known_commit) { return false; } } if (!$message->getRevisionID()) { return true; } $in_working_copy = $api->loadWorkingCopyDifferentialRevisions( $this->getConduit(), array( 'authors' => array($this->getUserPHID()), 'status' => 'status-open', )); if ($in_working_copy) { return true; } return false; } private function askForAdd(array $files) { if ($this->commitMode == self::COMMIT_DISABLE) { return false; } if ($this->commitMode == self::COMMIT_ENABLE) { return true; } $prompt = $this->getAskForAddPrompt($files); return phutil_console_confirm($prompt); } private function getAskForAddPrompt(array $files) { if ($this->getShouldAmend()) { $prompt = pht( 'Do you want to amend these %s change(s) to the current commit?', phutil_count($files)); } else { $prompt = pht( 'Do you want to create a new commit with these %s change(s)?', phutil_count($files)); } return $prompt; } final protected function loadDiffBundleFromConduit( ConduitClient $conduit, $diff_id) { return $this->loadBundleFromConduit( $conduit, array( 'ids' => array($diff_id), )); } final protected function loadRevisionBundleFromConduit( ConduitClient $conduit, $revision_id) { return $this->loadBundleFromConduit( $conduit, array( 'revisionIDs' => array($revision_id), )); } private function loadBundleFromConduit( ConduitClient $conduit, $params) { $future = $conduit->callMethod('differential.querydiffs', $params); $diff = head($future->resolve()); if ($diff == null) { throw new Exception( phutil_console_wrap( pht("The diff or revision you specified is either invalid or you ". "don't have permission to view it.")) ); } $changes = array(); foreach ($diff['changes'] as $changedict) { $changes[] = ArcanistDiffChange::newFromDictionary($changedict); } $bundle = ArcanistBundle::newFromChanges($changes); $bundle->setConduit($conduit); // since the conduit method has changes, assume that these fields // could be unset $bundle->setBaseRevision(idx($diff, 'sourceControlBaseRevision')); $bundle->setRevisionID(idx($diff, 'revisionID')); $bundle->setAuthorName(idx($diff, 'authorName')); $bundle->setAuthorEmail(idx($diff, 'authorEmail')); return $bundle; } /** * Return a list of lines changed by the current diff, or ##null## if the * change list is meaningless (for example, because the path is a directory * or binary file). * * @param string Path within the repository. * @param string Change selection mode (see ArcanistDiffHunk). * @return list|null List of changed line numbers, or null to indicate that * the path is not a line-oriented text file. */ final protected function getChangedLines($path, $mode) { $repository_api = $this->getRepositoryAPI(); $full_path = $repository_api->getPath($path); if (is_dir($full_path)) { return null; } if (!file_exists($full_path)) { return null; } $change = $this->getChange($path); if ($change->getFileType() !== ArcanistDiffChangeType::FILE_TEXT) { return null; } $lines = $change->getChangedLines($mode); return array_keys($lines); } final protected function getChange($path) { $repository_api = $this->getRepositoryAPI(); // TODO: Very gross $is_git = ($repository_api instanceof ArcanistGitAPI); $is_hg = ($repository_api instanceof ArcanistMercurialAPI); $is_svn = ($repository_api instanceof ArcanistSubversionAPI); if ($is_svn) { // NOTE: In SVN, we don't currently support a "get all local changes" // operation, so special case it. if (empty($this->changeCache[$path])) { $diff = $repository_api->getRawDiffText($path); $parser = $this->newDiffParser(); $changes = $parser->parseDiff($diff); if (count($changes) != 1) { throw new Exception(pht('Expected exactly one change.')); } $this->changeCache[$path] = reset($changes); } } else if ($is_git || $is_hg) { if (empty($this->changeCache)) { $changes = $repository_api->getAllLocalChanges(); foreach ($changes as $change) { $this->changeCache[$change->getCurrentPath()] = $change; } } } else { throw new Exception(pht('Missing VCS support.')); } if (empty($this->changeCache[$path])) { if ($is_git || $is_hg) { // This can legitimately occur under git/hg if you make a change, // "git/hg commit" it, and then revert the change in the working copy // and run "arc lint". $change = new ArcanistDiffChange(); $change->setCurrentPath($path); return $change; } else { throw new Exception( pht( "Trying to get change for unchanged path '%s'!", $path)); } } return $this->changeCache[$path]; } final public function willRunWorkflow() { $spec = $this->getCompleteArgumentSpecification(); foreach ($this->arguments as $arg => $value) { if (empty($spec[$arg])) { continue; } $options = $spec[$arg]; if (!empty($options['supports'])) { $system_name = $this->getRepositoryAPI()->getSourceControlSystemName(); if (!in_array($system_name, $options['supports'])) { $extended_info = null; if (!empty($options['nosupport'][$system_name])) { $extended_info = ' '.$options['nosupport'][$system_name]; } throw new ArcanistUsageException( pht( "Option '%s' is not supported under %s.", "--{$arg}", $system_name). $extended_info); } } } } final protected function normalizeRevisionID($revision_id) { return preg_replace('/^D/i', '', $revision_id); } protected function shouldShellComplete() { return true; } protected function getShellCompletions(array $argv) { return array(); } public function getSupportedRevisionControlSystems() { return array('git', 'hg', 'svn'); } final protected function getPassthruArgumentsAsMap($command) { $map = array(); foreach ($this->getCompleteArgumentSpecification() as $key => $spec) { if (!empty($spec['passthru'][$command])) { if (isset($this->arguments[$key])) { $map[$key] = $this->arguments[$key]; } } } return $map; } final protected function getPassthruArgumentsAsArgv($command) { $spec = $this->getCompleteArgumentSpecification(); $map = $this->getPassthruArgumentsAsMap($command); $argv = array(); foreach ($map as $key => $value) { $argv[] = '--'.$key; if (!empty($spec[$key]['param'])) { $argv[] = $value; } } return $argv; } /** * Write a message to stderr so that '--json' flags or stdout which is meant * to be piped somewhere aren't disrupted. * * @param string Message to write to stderr. * @return void */ final protected function writeStatusMessage($msg) { fwrite(STDERR, $msg); } final public function writeInfo($title, $message) { $this->writeStatusMessage( phutil_console_format( "** %s ** %s\n", $title, $message)); } final public function writeWarn($title, $message) { $this->writeStatusMessage( phutil_console_format( "** %s ** %s\n", $title, $message)); } final public function writeOkay($title, $message) { $this->writeStatusMessage( phutil_console_format( "** %s ** %s\n", $title, $message)); } final protected function isHistoryImmutable() { $repository_api = $this->getRepositoryAPI(); $config = $this->getConfigFromAnySource('history.immutable'); if ($config !== null) { return $config; } return $repository_api->isHistoryDefaultImmutable(); } /** * Workflows like 'lint' and 'unit' operate on a list of working copy paths. * The user can either specify the paths explicitly ("a.js b.php"), or by * specifying a revision ("--rev a3f10f1f") to select all paths modified * since that revision, or by omitting both and letting arc choose the * default relative revision. * * This method takes the user's selections and returns the paths that the * workflow should act upon. * * @param list List of explicitly provided paths. * @param string|null Revision name, if provided. * @param mask Mask of ArcanistRepositoryAPI flags to exclude. * Defaults to ArcanistRepositoryAPI::FLAG_UNTRACKED. * @return list List of paths the workflow should act on. */ final protected function selectPathsForWorkflow( array $paths, $rev, $omit_mask = null) { if ($omit_mask === null) { $omit_mask = ArcanistRepositoryAPI::FLAG_UNTRACKED; } if ($paths) { $working_copy = $this->getWorkingCopyIdentity(); foreach ($paths as $key => $path) { $full_path = Filesystem::resolvePath($path); if (!Filesystem::pathExists($full_path)) { throw new ArcanistUsageException( pht( "Path '%s' does not exist!", $path)); } $relative_path = Filesystem::readablePath( $full_path, $working_copy->getProjectRoot()); $paths[$key] = $relative_path; } } else { $repository_api = $this->getRepositoryAPI(); if ($rev) { $this->parseBaseCommitArgument(array($rev)); } $paths = $repository_api->getWorkingCopyStatus(); foreach ($paths as $path => $flags) { if ($flags & $omit_mask) { unset($paths[$path]); } } $paths = array_keys($paths); } return array_values($paths); } final protected function renderRevisionList(array $revisions) { $list = array(); foreach ($revisions as $revision) { $list[] = ' - D'.$revision['id'].': '.$revision['title']."\n"; } return implode('', $list); } /* -( Scratch Files )------------------------------------------------------ */ /** * Try to read a scratch file, if it exists and is readable. * * @param string Scratch file name. * @return mixed String for file contents, or false for failure. * @task scratch */ final protected function readScratchFile($path) { if (!$this->repositoryAPI) { return false; } return $this->getRepositoryAPI()->readScratchFile($path); } /** * Try to read a scratch JSON file, if it exists and is readable. * * @param string Scratch file name. * @return array Empty array for failure. * @task scratch */ final protected function readScratchJSONFile($path) { $file = $this->readScratchFile($path); if (!$file) { return array(); } return phutil_json_decode($file); } /** * Try to write a scratch file, if there's somewhere to put it and we can * write there. * * @param string Scratch file name to write. * @param string Data to write. * @return bool True on success, false on failure. * @task scratch */ final protected function writeScratchFile($path, $data) { if (!$this->repositoryAPI) { return false; } return $this->getRepositoryAPI()->writeScratchFile($path, $data); } /** * Try to write a scratch JSON file, if there's somewhere to put it and we can * write there. * * @param string Scratch file name to write. * @param array Data to write. * @return bool True on success, false on failure. * @task scratch */ final protected function writeScratchJSONFile($path, array $data) { return $this->writeScratchFile($path, json_encode($data)); } /** * Try to remove a scratch file. * * @param string Scratch file name to remove. * @return bool True if the file was removed successfully. * @task scratch */ final protected function removeScratchFile($path) { if (!$this->repositoryAPI) { return false; } return $this->getRepositoryAPI()->removeScratchFile($path); } /** * Get a human-readable description of the scratch file location. * * @param string Scratch file name. * @return mixed String, or false on failure. * @task scratch */ final protected function getReadableScratchFilePath($path) { if (!$this->repositoryAPI) { return false; } return $this->getRepositoryAPI()->getReadableScratchFilePath($path); } /** * Get the path to a scratch file, if possible. * * @param string Scratch file name. * @return mixed File path, or false on failure. * @task scratch */ final protected function getScratchFilePath($path) { if (!$this->repositoryAPI) { return false; } return $this->getRepositoryAPI()->getScratchFilePath($path); } final protected function getRepositoryEncoding() { return nonempty( idx($this->loadProjectRepository(), 'encoding'), 'UTF-8'); } final protected function loadProjectRepository() { list($info, $reasons) = $this->loadRepositoryInformation(); return coalesce($info, array()); } final protected function newInteractiveEditor($text) { $editor = new PhutilInteractiveEditor($text); $preferred = $this->getConfigFromAnySource('editor'); if ($preferred) { $editor->setPreferredEditor($preferred); } return $editor; } final protected function newDiffParser() { $parser = new ArcanistDiffParser(); if ($this->repositoryAPI) { $parser->setRepositoryAPI($this->getRepositoryAPI()); } $parser->setWriteDiffOnFailure(true); return $parser; } final protected function dispatchEvent($type, array $data) { $data += array( 'workflow' => $this, ); $event = new PhutilEvent($type, $data); PhutilEventEngine::dispatchEvent($event); return $event; } final public function parseBaseCommitArgument(array $argv) { if (!count($argv)) { return; } $api = $this->getRepositoryAPI(); if (!$api->supportsCommitRanges()) { throw new ArcanistUsageException( pht('This version control system does not support commit ranges.')); } if (count($argv) > 1) { throw new ArcanistUsageException( pht( 'Specify exactly one base commit. The end of the commit range is '. 'always the working copy state.')); } $api->setBaseCommit(head($argv)); return $this; } final protected function getRepositoryVersion() { if (!$this->repositoryVersion) { $api = $this->getRepositoryAPI(); $commit = $api->getSourceControlBaseRevision(); $versions = array('' => $commit); foreach ($api->getChangedFiles($commit) as $path => $mask) { $versions[$path] = (Filesystem::pathExists($path) ? md5_file($path) : ''); } $this->repositoryVersion = md5(json_encode($versions)); } return $this->repositoryVersion; } /* -( Phabricator Repositories )------------------------------------------- */ /** * Get the PHID of the Phabricator repository this working copy corresponds * to. Returns `null` if no repository can be identified. * * @return phid|null Repository PHID, or null if no repository can be * identified. * * @task phabrep */ final protected function getRepositoryPHID() { return idx($this->getRepositoryInformation(), 'phid'); } /** * Get the name of the Phabricator repository this working copy * corresponds to. Returns `null` if no repository can be identified. * * @return string|null Repository name, or null if no repository can be * identified. * * @task phabrep */ final protected function getRepositoryName() { return idx($this->getRepositoryInformation(), 'name'); } /** * Get the URI of the Phabricator repository this working copy * corresponds to. Returns `null` if no repository can be identified. * * @return string|null Repository URI, or null if no repository can be * identified. * * @task phabrep */ final protected function getRepositoryURI() { return idx($this->getRepositoryInformation(), 'uri'); } final protected function getRepositoryStagingConfiguration() { return idx($this->getRepositoryInformation(), 'staging'); } /** * Get human-readable reasoning explaining how `arc` evaluated which * Phabricator repository corresponds to this working copy. Used by * `arc which` to explain the process to users. * * @return list Human-readable explanation of the repository * association process. * * @task phabrep */ final protected function getRepositoryReasons() { $this->getRepositoryInformation(); return $this->repositoryReasons; } /** * @task phabrep */ private function getRepositoryInformation() { if ($this->repositoryInfo === null) { list($info, $reasons) = $this->loadRepositoryInformation(); $this->repositoryInfo = nonempty($info, array()); $this->repositoryReasons = $reasons; } return $this->repositoryInfo; } /** * @task phabrep */ private function loadRepositoryInformation() { list($query, $reasons) = $this->getRepositoryQuery(); if (!$query) { return array(null, $reasons); } try { $method = 'repository.query'; $results = $this->getConduitEngine() ->newFuture($method, $query) ->resolve(); } catch (ConduitClientException $ex) { if ($ex->getErrorCode() == 'ERR-CONDUIT-CALL') { $reasons[] = pht( 'This version of Arcanist is more recent than the version of '. 'Phabricator you are connecting to: the Phabricator install is '. 'out of date and does not have support for identifying '. 'repositories by callsign or URI. Update Phabricator to enable '. 'these features.'); return array(null, $reasons); } throw $ex; } $result = null; if (!$results) { $reasons[] = pht( 'No repositories matched the query. Check that your configuration '. 'is correct, or use "%s" to select a repository explicitly.', 'repository.callsign'); } else if (count($results) > 1) { $reasons[] = pht( 'Multiple repostories (%s) matched the query. You can use the '. '"%s" configuration to select the one you want.', implode(', ', ipull($results, 'callsign')), 'repository.callsign'); } else { $result = head($results); $reasons[] = pht('Found a unique matching repository.'); } return array($result, $reasons); } /** * @task phabrep */ private function getRepositoryQuery() { $reasons = array(); $callsign = $this->getConfigFromAnySource('repository.callsign'); if ($callsign) { $query = array( 'callsigns' => array($callsign), ); $reasons[] = pht( 'Configuration value "%s" is set to "%s".', 'repository.callsign', $callsign); return array($query, $reasons); } else { $reasons[] = pht( 'Configuration value "%s" is empty.', 'repository.callsign'); } $uuid = $this->getRepositoryAPI()->getRepositoryUUID(); if ($uuid !== null) { $query = array( 'uuids' => array($uuid), ); $reasons[] = pht( 'The UUID for this working copy is "%s".', $uuid); return array($query, $reasons); } else { $reasons[] = pht( 'This repository has no VCS UUID (this is normal for git/hg).'); } // TODO: Swap this for a RemoteRefQuery. $remote_uri = $this->getRepositoryAPI()->getRemoteURI(); if ($remote_uri !== null) { $query = array( 'remoteURIs' => array($remote_uri), ); $reasons[] = pht( 'The remote URI for this working copy is "%s".', $remote_uri); return array($query, $reasons); } else { $reasons[] = pht( 'Unable to determine the remote URI for this repository.'); } return array(null, $reasons); } /** * Build a new lint engine for the current working copy. * * Optionally, you can pass an explicit engine class name to build an engine * of a particular class. Normally this is used to implement an `--engine` * flag from the CLI. * * @param string Optional explicit engine class name. * @return ArcanistLintEngine Constructed engine. */ protected function newLintEngine($engine_class = null) { $working_copy = $this->getWorkingCopyIdentity(); $config = $this->getConfigurationManager(); if (!$engine_class) { $engine_class = $config->getConfigFromAnySource('lint.engine'); } if (!$engine_class) { if (Filesystem::pathExists($working_copy->getProjectPath('.arclint'))) { $engine_class = 'ArcanistConfigurationDrivenLintEngine'; } } if (!$engine_class) { throw new ArcanistNoEngineException( pht( "No lint engine is configured for this project. Create an '%s' ". "file, or configure an advanced engine with '%s' in '%s'.", '.arclint', 'lint.engine', '.arcconfig')); } $base_class = 'ArcanistLintEngine'; if (!class_exists($engine_class) || !is_subclass_of($engine_class, $base_class)) { throw new ArcanistUsageException( pht( 'Configured lint engine "%s" is not a subclass of "%s", but must be.', $engine_class, $base_class)); } $engine = newv($engine_class, array()) ->setWorkingCopy($working_copy) ->setConfigurationManager($config); return $engine; } /** * Build a new unit test engine for the current working copy. * * Optionally, you can pass an explicit engine class name to build an engine * of a particular class. Normally this is used to implement an `--engine` * flag from the CLI. * * @param string Optional explicit engine class name. * @return ArcanistUnitTestEngine Constructed engine. */ protected function newUnitTestEngine($engine_class = null) { $working_copy = $this->getWorkingCopyIdentity(); $config = $this->getConfigurationManager(); if (!$engine_class) { $engine_class = $config->getConfigFromAnySource('unit.engine'); } if (!$engine_class) { if (Filesystem::pathExists($working_copy->getProjectPath('.arcunit'))) { $engine_class = 'ArcanistConfigurationDrivenUnitTestEngine'; } } if (!$engine_class) { throw new ArcanistNoEngineException( pht( "No unit test engine is configured for this project. Create an ". "'%s' file, or configure an advanced engine with '%s' in '%s'.", '.arcunit', 'unit.engine', '.arcconfig')); } $base_class = 'ArcanistUnitTestEngine'; if (!class_exists($engine_class) || !is_subclass_of($engine_class, $base_class)) { throw new ArcanistUsageException( pht( 'Configured unit test engine "%s" is not a subclass of "%s", '. 'but must be.', $engine_class, $base_class)); } $engine = newv($engine_class, array()) ->setWorkingCopy($working_copy) ->setConfigurationManager($config); return $engine; } protected function openURIsInBrowser(array $uris) { $browser = $this->getBrowserCommand(); // The "browser" may actually be a list of arguments. if (!is_array($browser)) { $browser = array($browser); } foreach ($uris as $uri) { $err = phutil_passthru('%LR %R', $browser, $uri); if ($err) { throw new ArcanistUsageException( pht( 'Failed to open URI "%s" in browser ("%s"). '. 'Check your "browser" config option.', $uri, implode(' ', $browser))); } } } private function getBrowserCommand() { $config = $this->getConfigFromAnySource('browser'); if ($config) { return $config; } if (phutil_is_windows()) { // See T13504. We now use "bypass_shell", so "start" alone is no longer // a valid binary to invoke directly. return array( 'cmd', '/c', 'start', ); } $candidates = array( 'sensible-browser' => array('sensible-browser'), 'xdg-open' => array('xdg-open'), 'open' => array('open', '--'), ); // NOTE: The "open" command works well on OS X, but on many Linuxes "open" // exists and is not a browser. For now, we're just looking for other // commands first, but we might want to be smarter about selecting "open" // only on OS X. foreach ($candidates as $cmd => $argv) { if (Filesystem::binaryExists($cmd)) { return $argv; } } throw new ArcanistUsageException( pht( "Unable to find a browser command to run. Set '%s' in your ". "Arcanist config to specify a command to use.", 'browser')); } /** * Ask Phabricator to update the current repository as soon as possible. * * Calling this method after pushing commits allows Phabricator to discover * the commits more quickly, so the system overall is more responsive. * * @return void */ protected function askForRepositoryUpdate() { // If we know which repository we're in, try to tell Phabricator that we // pushed commits to it so it can update. This hint can help pull updates // more quickly, especially in rarely-used repositories. if ($this->getRepositoryPHID()) { try { $this->getConduit()->callMethodSynchronous( 'diffusion.looksoon', array( 'repositories' => array($this->getRepositoryPHID()), )); } catch (ConduitClientException $ex) { // If we hit an exception, just ignore it. Likely, we are running // against a Phabricator which is too old to support this method. // Since this hint is purely advisory, it doesn't matter if it has // no effect. } } } protected function getModernLintDictionary(array $map) { $map = $this->getModernCommonDictionary($map); return $map; } protected function getModernUnitDictionary(array $map) { $map = $this->getModernCommonDictionary($map); $details = idx($map, 'userData'); if (strlen($details)) { $map['details'] = (string)$details; } unset($map['userData']); return $map; } private function getModernCommonDictionary(array $map) { foreach ($map as $key => $value) { if ($value === null) { unset($map[$key]); } } return $map; } final public function setConduitEngine( ArcanistConduitEngine $conduit_engine) { $this->conduitEngine = $conduit_engine; return $this; } final public function getConduitEngine() { return $this->conduitEngine; } final public function getRepositoryRef() { $configuration_engine = $this->getConfigurationEngine(); if ($configuration_engine) { // This is a toolset workflow and can always build a repository ref. } else { if (!$this->getConfigurationManager()->getWorkingCopyIdentity()) { return null; } if (!$this->repositoryAPI) { return null; } } if (!$this->repositoryRef) { $ref = id(new ArcanistRepositoryRef()) ->setPHID($this->getRepositoryPHID()) ->setBrowseURI($this->getRepositoryURI()); $this->repositoryRef = $ref; } return $this->repositoryRef; } final public function getToolsetKey() { return $this->getToolset()->getToolsetKey(); } final public function getConfig($key) { return $this->getConfigurationSourceList()->getConfig($key); } public function canHandleSignal($signo) { return false; } public function handleSignal($signo) { return; } final public function newCommand(PhutilExecutableFuture $future) { return id(new ArcanistCommand()) ->setLogEngine($this->getLogEngine()) ->setExecutableFuture($future); } final public function loadHardpoints( $objects, $requests) { return $this->getRuntime()->loadHardpoints($objects, $requests); } protected function newPrompts() { return array(); } protected function newPrompt($key) { return id(new ArcanistPrompt()) ->setWorkflow($this) ->setKey($key); } public function hasPrompt($key) { $map = $this->getPromptMap(); return isset($map[$key]); } public function getPromptMap() { if ($this->promptMap === null) { $prompts = $this->newPrompts(); assert_instances_of($prompts, 'ArcanistPrompt'); // TODO: Move this somewhere modular. $prompts[] = $this->newPrompt('arc.state.stash') ->setDescription( pht( 'Prompts the user to stash changes and continue when the '. 'working copy has untracked, uncommitted, or unstaged '. 'changes.')); // TODO: Swap to ArrayCheck? $map = array(); foreach ($prompts as $prompt) { $key = $prompt->getKey(); if (isset($map[$key])) { throw new Exception( pht( 'Workflow ("%s") generates two prompts with the same '. 'key ("%s"). Each prompt a workflow generates must have a '. 'unique key.', get_class($this), $key)); } $map[$key] = $prompt; } $this->promptMap = $map; } return $this->promptMap; } final public function getPrompt($key) { $map = $this->getPromptMap(); $prompt = idx($map, $key); if (!$prompt) { throw new Exception( pht( 'Workflow ("%s") is requesting a prompt ("%s") but it did not '. 'generate any prompt with that name in "newPrompts()".', get_class($this), $key)); } return clone $prompt; } final protected function getSymbolEngine() { return $this->getRuntime()->getSymbolEngine(); } final protected function getViewer() { return $this->getRuntime()->getViewer(); } final protected function readStdin() { $log = $this->getLogEngine(); $log->writeWaitingForInput(); // NOTE: We can't just "file_get_contents()" here because signals don't // interrupt it. If the user types "^C", we want to interrupt the read. $raw_handle = fopen('php://stdin', 'rb'); $stdin = new PhutilSocketChannel($raw_handle); while ($stdin->update()) { PhutilChannel::waitForAny(array($stdin)); } return $stdin->read(); } final public function getAbsoluteURI($raw_uri) { // TODO: "ArcanistRevisionRef", at least, may return a relative URI. // If we get a relative URI, guess the correct absolute URI based on // the Conduit URI. This might not be correct for Conduit over SSH. $raw_uri = new PhutilURI($raw_uri); if (!strlen($raw_uri->getDomain())) { $base_uri = $this->getConduitEngine() ->getConduitURI(); $raw_uri = id(new PhutilURI($base_uri)) ->setPath($raw_uri->getPath()); } $raw_uri = phutil_string_cast($raw_uri); return $raw_uri; } final public function writeToPager($corpus) { $is_tty = (function_exists('posix_isatty') && posix_isatty(STDOUT)); if (!$is_tty) { echo $corpus; } else { $pager = $this->getConfig('pager'); if (!$pager) { $pager = array('less', '-R', '--'); } // Try to show the content through a pager. $err = id(new PhutilExecPassthru('%Ls', $pager)) ->write($corpus) ->resolve(); // If the pager exits with an error, print the content normally. if ($err) { echo $corpus; } } return $this; } } diff --git a/support/lib/extract-symbols.php b/support/lib/extract-symbols.php index dac4ca71..a1cd28fa 100755 --- a/support/lib/extract-symbols.php +++ b/support/lib/extract-symbols.php @@ -1,609 +1,615 @@ #!/usr/bin/env php setTagline(pht('identify symbols in a PHP source file')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'all', 'help' => pht( 'Emit all symbols, including built-ins and declared externals.'), ), array( 'name' => 'builtins', 'help' => pht('Emit builtin symbols.'), ), array( 'name' => 'ugly', 'help' => pht('Do not prettify JSON output.'), ), array( 'name' => 'path', 'wildcard' => true, 'help' => pht('PHP Source file to analyze.'), ), )); $paths = $args->getArg('path'); $show_all = $args->getArg('all'); $show_builtins = $args->getArg('builtins'); if ($show_all && $show_builtins) { throw new PhutilArgumentUsageException( pht( 'Flags "--all" and "--builtins" are not compatible.')); } if ($show_builtins && $paths) { throw new PhutilArgumentUsageException( pht( 'Flag "--builtins" may not be used with a path.')); } if ($show_builtins) { $path = ''; $source_code = ''; } else { if (count($paths) !== 1) { throw new Exception(pht('Specify exactly one path!')); } $path = Filesystem::resolvePath(head($paths)); $source_code = Filesystem::readFile($path); } try { $tree = XHPASTTree::newFromData($source_code); } catch (XHPASTSyntaxErrorException $ex) { $result = array( 'error' => $ex->getMessage(), 'line' => $ex->getErrorLine(), 'file' => $path, ); $json = new PhutilJSON(); echo $json->encodeFormatted($result); exit(0); } $root = $tree->getRootNode(); $root->buildSelectCache(); // -( Unsupported Constructs )------------------------------------------------ $namespaces = $root->selectDescendantsOfType('n_NAMESPACE'); foreach ($namespaces as $namespace) { phutil_fail_on_unsupported_feature($namespace, $path, pht('namespaces')); } $uses = $root->selectDescendantsOfType('n_USE'); foreach ($namespaces as $namespace) { phutil_fail_on_unsupported_feature( $namespace, $path, pht('namespace `%s` statements', 'use')); } $possible_traits = $root->selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($possible_traits as $possible_trait) { $attributes = $possible_trait->getChildByIndex(0); // Can't use getChildByIndex here because not all classes have attributes foreach ($attributes->getChildren() as $attribute) { if (strtolower($attribute->getConcreteString()) === 'trait') { phutil_fail_on_unsupported_feature($possible_trait, $path, pht('traits')); } } } // -( Marked Externals )------------------------------------------------------ // Identify symbols marked with "@phutil-external-symbol", so we exclude them // from the dependency list. $externals = array(); $doc_parser = new PhutilDocblockParser(); foreach ($root->getTokens() as $token) { if ($token->getTypeName() === 'T_DOC_COMMENT') { list($block, $special) = $doc_parser->parse($token->getValue()); $ext_list = idx($special, 'phutil-external-symbol'); $ext_list = (array)$ext_list; $ext_list = array_filter($ext_list); foreach ($ext_list as $ext_ref) { $matches = null; if (preg_match('/^\s*(\S+)\s+(\S+)/', $ext_ref, $matches)) { $externals[$matches[1]][$matches[2]] = true; } } } } // -( Declarations and Dependencies )----------------------------------------- // The first stage of analysis is to find all the symbols we declare in the // file (like functions and classes) and all the symbols we use in the file // (like calling functions and invoking classes). Later, we filter this list // to exclude builtins. $have = array(); // For symbols we declare. $need = array(); // For symbols we use. $xmap = array(); // For extended classes and implemented interfaces. // -( Functions )------------------------------------------------------------- // Find functions declared in this file. // This is "function f() { ... }". $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION'); foreach ($functions as $function) { $name = $function->getChildByIndex(2); if ($name->getTypeName() === 'n_EMPTY') { // This is an anonymous function; don't record it into the symbol // index. continue; } $have[] = array( 'type' => 'function', 'symbol' => $name, ); } // Find functions used by this file. Uses: // // - Explicit Call // - String literal passed to call_user_func() or call_user_func_array() // - String literal in array literal in call_user_func()/call_user_func_array() // // TODO: Possibly support these: // // - String literal in ReflectionFunction(). // This is "f();". $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL'); foreach ($calls as $call) { $name = $call->getChildByIndex(0); if ($name->getTypeName() === 'n_VARIABLE' || $name->getTypeName() === 'n_VARIABLE_VARIABLE') { // Ignore these, we can't analyze them. continue; } if ($name->getTypeName() === 'n_CLASS_STATIC_ACCESS') { // These are "C::f()", we'll pick this up later on. continue; } $call_name = $name->getConcreteString(); if ($call_name === 'call_user_func' || $call_name === 'call_user_func_array') { $params = $call->getChildByIndex(1)->getChildren(); if (!count($params)) { // This is a bare call_user_func() with no arguments; just ignore it. continue; } $symbol = array_shift($params); $type = 'function'; $symbol_value = $symbol->getStringLiteralValue(); - $pos = strpos($symbol_value, '::'); + + if ($symbol_value !== null) { + $pos = strpos($symbol_value, '::'); + } else { + $pos = false; + } + if ($pos) { $type = 'class'; $symbol_value = substr($symbol_value, 0, $pos); } else if ($symbol->getTypeName() === 'n_ARRAY_LITERAL') { try { $type = 'class'; $symbol_value = idx($symbol->evalStatic(), 0); } catch (Exception $ex) {} } if ($symbol_value && strpos($symbol_value, '$') === false) { $need[] = array( 'type' => $type, 'name' => $symbol_value, 'symbol' => $symbol, ); } } else { $need[] = array( 'type' => 'function', 'symbol' => $name, ); } } // -( Classes )--------------------------------------------------------------- // Find classes declared by this file. // This is "class X ... { ... }". $classes = $root->selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($classes as $class) { $class_name = $class->getChildByIndex(1); $have[] = array( 'type' => 'class', 'symbol' => $class_name, ); } // Find classes used by this file. We identify these: // // - class ... extends X // - new X // - Static method call // - Static property access // - Use of class constant // - typehints // - catch // - instanceof // - newv() // // TODO: Possibly support these: // // - String literal in ReflectionClass(). // This is "class X ... { ... }". $classes = $root->selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($classes as $class) { $class_name = $class->getChildByIndex(1)->getConcreteString(); $extends = $class->getChildByIndex(2); foreach ($extends->selectDescendantsOfType('n_CLASS_NAME') as $parent) { $need[] = array( 'type' => 'class', 'symbol' => $parent, ); // Track all 'extends' in the extension map. $xmap[$class_name][] = $parent->getConcreteString(); } } // This is "new X()". $uses_of_new = $root->selectDescendantsOfType('n_NEW'); foreach ($uses_of_new as $new_operator) { $name = $new_operator->getChildByIndex(0); if ($name->getTypeName() === 'n_VARIABLE' || $name->getTypeName() === 'n_VARIABLE_VARIABLE') { continue; } $need[] = array( 'type' => 'class', 'symbol' => $name, ); } // This covers all of "X::$y", "X::y()" and "X::CONST". $static_uses = $root->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); foreach ($static_uses as $static_use) { $name = $static_use->getChildByIndex(0); if ($name->getTypeName() !== 'n_CLASS_NAME') { continue; } $need[] = array( 'type' => 'class', 'symbol' => $name, ); } // This is "function (X $x)". $parameters = $root->selectDescendantsOfType('n_DECLARATION_PARAMETER'); foreach ($parameters as $parameter) { $hint = $parameter->getChildByIndex(0); if ($hint->getTypeName() !== 'n_CLASS_NAME') { continue; } $need[] = array( 'type' => 'class/interface', 'symbol' => $hint, ); } $returns = $root->selectDescendantsOfType('n_DECLARATION_RETURN'); foreach ($returns as $return) { $hint = $return->getChildByIndex(0); if ($hint->getTypeName() !== 'n_CLASS_NAME') { continue; } $need[] = array( 'type' => 'class/interface', 'symbol' => $hint, ); } // This is "catch (Exception $ex)". $catches = $root->selectDescendantsOfType('n_CATCH'); foreach ($catches as $catch) { $need[] = array( 'type' => 'class/interface', 'symbol' => $catch->getChildOfType(0, 'n_CLASS_NAME'), ); } // This is "$x instanceof X". $instanceofs = $root->selectDescendantsOfType('n_BINARY_EXPRESSION'); foreach ($instanceofs as $instanceof) { $operator = $instanceof->getChildOfType(1, 'n_OPERATOR'); if ($operator->getConcreteString() !== 'instanceof') { continue; } $class = $instanceof->getChildByIndex(2); if ($class->getTypeName() !== 'n_CLASS_NAME') { continue; } $need[] = array( 'type' => 'class/interface', 'symbol' => $class, ); } // This is "newv('X')". $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL'); foreach ($calls as $call) { $call_name = $call->getChildByIndex(0)->getConcreteString(); if ($call_name !== 'newv') { continue; } $params = $call->getChildByIndex(1)->getChildren(); if (!count($params)) { continue; } $symbol = reset($params); $symbol_value = $symbol->getStringLiteralValue(); if ($symbol_value && strpos($symbol_value, '$') === false) { $need[] = array( 'type' => 'class', 'name' => $symbol_value, 'symbol' => $symbol, ); } } // -( Interfaces )------------------------------------------------------------ // Find interfaces declared in this file. // This is "interface X .. { ... }". $interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION'); foreach ($interfaces as $interface) { $interface_name = $interface->getChildByIndex(1); $have[] = array( 'type' => 'interface', 'symbol' => $interface_name, ); } // Find interfaces used by this file. We identify these: // // - class ... implements X // - interface ... extends X // This is "class X ... { ... }". $classes = $root->selectDescendantsOfType('n_CLASS_DECLARATION'); foreach ($classes as $class) { $class_name = $class->getChildByIndex(1)->getConcreteString(); $implements = $class->getChildByIndex(3); $interfaces = $implements->selectDescendantsOfType('n_CLASS_NAME'); foreach ($interfaces as $interface) { $need[] = array( 'type' => 'interface', 'symbol' => $interface, ); // Track 'class ... implements' in the extension map. $xmap[$class_name][] = $interface->getConcreteString(); } } // This is "interface X ... { ... }". $interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION'); foreach ($interfaces as $interface) { $interface_name = $interface->getChildByIndex(1)->getConcreteString(); $extends = $interface->getChildByIndex(2); foreach ($extends->selectDescendantsOfType('n_CLASS_NAME') as $parent) { $need[] = array( 'type' => 'interface', 'symbol' => $parent, ); // Track 'interface ... extends' in the extension map. $xmap[$interface_name][] = $parent->getConcreteString(); } } // -( Analysis )-------------------------------------------------------------- $declared_symbols = array(); foreach ($have as $key => $spec) { $name = $spec['symbol']->getConcreteString(); $declared_symbols[$spec['type']][$name] = $spec['symbol']->getOffset(); } $required_symbols = array(); foreach ($need as $key => $spec) { $name = idx($spec, 'name'); if (!$name) { $name = $spec['symbol']->getConcreteString(); } $type = $spec['type']; foreach (explode('/', $type) as $libtype) { if (!$show_all) { if (!empty($externals[$libtype][$name])) { // Ignore symbols declared as externals. continue 2; } if (!empty($builtins[$libtype][$name])) { // Ignore symbols declared as builtins. continue 2; } } if (!empty($declared_symbols[$libtype][$name])) { // We declare this symbol, so don't treat it as a requirement. continue 2; } } if (!empty($required_symbols[$type][$name])) { // Report only the first use of a symbol, since reporting all of them // isn't terribly informative. continue; } $required_symbols[$type][$name] = $spec['symbol']->getOffset(); } if ($show_builtins) { foreach ($builtins as $type => $builtin_symbols) { foreach ($builtin_symbols as $builtin_symbol => $ignored) { $declared_symbols[$type][$builtin_symbol] = null; } } } $result = array( 'have' => $declared_symbols, 'need' => $required_symbols, 'xmap' => $xmap, ); // -( Output )---------------------------------------------------------------- if ($args->getArg('ugly')) { echo json_encode($result); } else { $json = new PhutilJSON(); echo $json->encodeFormatted($result); } // -( Library )--------------------------------------------------------------- function phutil_fail_on_unsupported_feature(XHPASTNode $node, $file, $what) { $line = $node->getLineNumber(); $message = phutil_console_wrap( pht( '`%s` has limited support for features introduced after PHP 5.2.3. '. 'This library uses an unsupported feature (%s) on line %d of %s.', 'arc liberate', $what, $line, Filesystem::readablePath($file))); $result = array( 'error' => $message, 'line' => $line, 'file' => $file, ); $json = new PhutilJSON(); echo $json->encodeFormatted($result); exit(0); } function phutil_symbols_get_builtins() { $builtin = array(); $builtin['classes'] = get_declared_classes(); $builtin['interfaces'] = get_declared_interfaces(); $funcs = get_defined_functions(); $builtin['functions'] = $funcs['internal']; $compat = json_decode( file_get_contents( dirname(__FILE__).'/../../resources/php/symbol-information.json'), true); foreach (array('functions', 'classes', 'interfaces') as $type) { // Developers may not have every extension that a library potentially uses // installed. We supplement the list of declared functions and classes with // a list of known extension functions to avoid raising false positives just // because you don't have pcntl, etc. $extensions = array_keys($compat[$type]); $builtin[$type] = array_merge($builtin[$type], $extensions); } return array( 'class' => array_fill_keys($builtin['classes'], true) + array( 'static' => true, 'parent' => true, 'self' => true, // PHP7 defines these new parent classes of "Exception", but they do not // exist prior to PHP7. It's possible to use them safely in PHP5, in // some cases, to write code which is compatible with either PHP5 or // PHP7, but it's hard for us tell if a particular use is safe or not. // For now, assume users know what they're doing and that uses are safe. // For discussion, see T12855. 'Throwable' => true, 'Error' => true, 'ParseError' => true, // PHP7 types. 'bool' => true, 'float' => true, 'int' => true, 'string' => true, 'iterable' => true, 'object' => true, 'void' => true, ), 'function' => array_filter( array( 'empty' => true, 'isset' => true, 'die' => true, // HPHP/i defines these functions as 'internal', but they are NOT // builtins and do not exist in vanilla PHP. Make sure we don't mark // them as builtin since we need to add dependencies for them. 'idx' => false, 'id' => false, ) + array_fill_keys($builtin['functions'], true)), 'interface' => array_fill_keys($builtin['interfaces'], true), ); }