diff --git a/src/future/Future.php b/src/future/Future.php index f69f14bc..8ce48eb6 100644 --- a/src/future/Future.php +++ b/src/future/Future.php @@ -1,211 +1,254 @@ resolve()" is no longer '. 'supported. Update the caller so it no longer passes a '. 'timeout.')); } if ($this->hasException()) { throw $this->getException(); } if (!$this->hasResult()) { $graph = new FutureIterator(array($this)); $graph->resolveAll(); } return $this->getResult(); } final public function startFuture() { if ($this->hasStarted) { throw new Exception( pht( 'Future has already started; futures can not start more '. 'than once.')); } $this->hasStarted = true; + $this->startServiceProfiler(); $this->isReady(); } final public function updateFuture() { if ($this->hasException()) { return; } if ($this->hasResult()) { return; } try { $this->isReady(); } catch (Exception $ex) { $this->setException($ex); } catch (Throwable $ex) { $this->setException($ex); } } final public function endFuture() { if (!$this->hasException() && !$this->hasResult()) { throw new Exception( pht( 'Trying to end a future which has no exception and no result. '. 'Futures must resolve before they can be ended.')); } if ($this->hasEnded) { throw new Exception( pht( 'Future has already ended; futures can not end more '. 'than once.')); } $this->hasEnded = true; + + $this->endServiceProfiler(); + } + + private function startServiceProfiler() { + + // NOTE: This is a soft dependency so that we don't need to build the + // ServiceProfiler into the Phage agent. Normally, this class is always + // available. + + if (!class_exists('PhutilServiceProfiler')) { + return; + } + + $params = $this->getServiceProfilerStartParameters(); + + $profiler = PhutilServiceProfiler::getInstance(); + $call_id = $profiler->beginServiceCall($params); + + $this->serviceProfilerCallID = $call_id; } + private function endServiceProfiler() { + $call_id = $this->serviceProfilerCallID; + if ($call_id === null) { + return; + } + + $params = $this->getServiceProfilerResultParameters(); + + $profiler = PhutilServiceProfiler::getInstance(); + $profiler->endServiceCall($call_id, $params); + } + + protected function getServiceProfilerStartParameters() { + return array(); + } + + protected function getServiceProfilerResultParameters() { + return array(); + } + + /** * Retrieve a list of sockets which we can wait to become readable while * a future is resolving. If your future has sockets which can be * `select()`ed, return them here (or in @{method:getWriteSockets}) to make * the resolve loop do a `select()`. If you do not return sockets in either * case, you'll get a busy wait. * * @return list A list of sockets which we expect to become readable. */ public function getReadSockets() { return array(); } /** * Retrieve a list of sockets which we can wait to become writable while a * future is resolving. See @{method:getReadSockets}. * * @return list A list of sockets which we expect to become writable. */ public function getWriteSockets() { return array(); } /** * Default amount of time to wait on stream select for this future. Normally * 1 second is fine, but if the future has a timeout sooner than that it * should return the amount of time left before the timeout. */ public function getDefaultWait() { return 1; } public function start() { $this->isReady(); return $this; } /** * Retrieve the final result of the future. * * @return wild Final resolution of this future. */ final protected function getResult() { if (!$this->hasResult()) { throw new Exception( pht( 'Future has not yet resolved. Resolve futures before retrieving '. 'results.')); } return $this->result; } final protected function setResult($result) { if ($this->hasResult()) { throw new Exception( pht( 'Future has already resolved. Futures may not resolve more than '. 'once.')); } $this->hasResult = true; $this->result = $result; return $this; } final public function hasResult() { return $this->hasResult; } final private function setException($exception) { // NOTE: The parameter may be an Exception or a Throwable. $this->exception = $exception; return $this; } final private function getException() { return $this->exception; } final public function hasException() { return ($this->exception !== null); } final public function setFutureKey($key) { if ($this->futureKey !== null) { throw new Exception( pht( 'Future already has a key ("%s") assigned.', $key)); } $this->futureKey = $key; return $this; } final public function getFutureKey() { static $next_key = 1; if ($this->futureKey === null) { $this->futureKey = sprintf('Future/%d', $next_key++); } return $this->futureKey; } } diff --git a/src/future/FutureProxy.php b/src/future/FutureProxy.php index 65cbffa2..77c8c5bb 100644 --- a/src/future/FutureProxy.php +++ b/src/future/FutureProxy.php @@ -1,68 +1,76 @@ setProxiedFuture($proxied); } } public function setProxiedFuture(Future $proxied) { $this->proxied = $proxied; return $this; } protected function getProxiedFuture() { if (!$this->proxied) { throw new Exception(pht('The proxied future has not been provided yet.')); } return $this->proxied; } public function isReady() { if ($this->hasResult()) { return true; } $proxied = $this->getProxiedFuture(); $is_ready = $proxied->isReady(); if ($proxied->hasResult()) { $result = $proxied->getResult(); $result = $this->didReceiveResult($result); $this->setResult($result); } return $is_ready; } public function resolve() { $this->getProxiedFuture()->resolve(); $this->isReady(); return $this->getResult(); } public function getReadSockets() { return $this->getProxiedFuture()->getReadSockets(); } public function getWriteSockets() { return $this->getProxiedFuture()->getWriteSockets(); } public function start() { $this->getProxiedFuture()->start(); return $this; } + protected function getServiceProfilerStartParameters() { + return $this->getProxiedFuture()->getServiceProfilerStartParameters(); + } + + protected function getServiceProfilerResultParameters() { + return $this->getProxiedFuture()->getServiceProfilerResultParameters(); + } + abstract protected function didReceiveResult($result); } diff --git a/src/future/exec/ExecFuture.php b/src/future/exec/ExecFuture.php index df8ce772..afb0fd14 100644 --- a/src/future/exec/ExecFuture.php +++ b/src/future/exec/ExecFuture.php @@ -1,961 +1,954 @@ array('pipe', 'r'), // stdin 1 => array('pipe', 'w'), // stdout 2 => array('pipe', 'w'), // stderr ); protected function didConstruct() { $this->stdin = new PhutilRope(); } /* -( Command Information )------------------------------------------------ */ /** * Retrieve the byte limit for the stderr buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStderrSizeLimit() { return $this->stderrSizeLimit; } /** * Retrieve the byte limit for the stdout buffer. * * @return int Maximum buffer size, in bytes. * @task info */ public function getStdoutSizeLimit() { return $this->stdoutSizeLimit; } /** * Get the process's pid. This only works after execution is initiated, e.g. * by a call to start(). * * @return int Process ID of the executing process. * @task info */ public function getPID() { $status = $this->procGetStatus(); return $status['pid']; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a maximum size for the stdout read buffer. To limit stderr, see * @{method:setStderrSizeLimit}. The major use of these methods is to use less * memory if you are running a command which sometimes produces huge volumes * of output that you don't really care about. * * NOTE: Setting this to 0 means "no buffer", not "unlimited buffer". * * @param int Maximum size of the stdout read buffer. * @return this * @task config */ public function setStdoutSizeLimit($limit) { $this->stdoutSizeLimit = $limit; return $this; } /** * Set a maximum size for the stderr read buffer. * See @{method:setStdoutSizeLimit} for discussion. * * @param int Maximum size of the stderr read buffer. * @return this * @task config */ public function setStderrSizeLimit($limit) { $this->stderrSizeLimit = $limit; return $this; } /** * Set the maximum internal read buffer size this future. The future will * block reads once the internal stdout or stderr buffer exceeds this size. * * NOTE: If you @{method:resolve} a future with a read buffer limit, you may * block forever! * * TODO: We should probably release the read buffer limit during * @{method:resolve}, or otherwise detect this. For now, be careful. * * @param int|null Maximum buffer size, or `null` for unlimited. * @return this */ public function setReadBufferSize($read_buffer_size) { $this->readBufferSize = $read_buffer_size; return $this; } /* -( Interacting With Commands )------------------------------------------ */ /** * Read and return output from stdout and stderr, if any is available. This * method keeps a read cursor on each stream, but the entire streams are * still returned when the future resolves. You can call read() again after * resolving the future to retrieve only the parts of the streams you did not * previously read: * * $future = new ExecFuture('...'); * // ... * list($stdout) = $future->read(); // Returns output so far * list($stdout) = $future->read(); // Returns new output since first call * // ... * list($stdout) = $future->resolvex(); // Returns ALL output * list($stdout) = $future->read(); // Returns unread output * * NOTE: If you set a limit with @{method:setStdoutSizeLimit} or * @{method:setStderrSizeLimit}, this method will not be able to read data * past the limit. * * NOTE: If you call @{method:discardBuffers}, all the stdout/stderr data * will be thrown away and the cursors will be reset. * * @return pair <$stdout, $stderr> pair with new output since the last call * to this method. * @task interact */ public function read() { $stdout = $this->readStdout(); $result = array( $stdout, (string)substr($this->stderr, $this->stderrPos), ); $this->stderrPos = strlen($this->stderr); return $result; } public function readStdout() { if ($this->start) { $this->isReady(); // Sync } $result = (string)substr($this->stdout, $this->stdoutPos); $this->stdoutPos = strlen($this->stdout); return $result; } /** * Write data to stdin of the command. * * @param string Data to write. * @param bool If true, keep the pipe open for writing. By default, the pipe * will be closed as soon as possible so that commands which * listen for EOF will execute. If you want to keep the pipe open * past the start of command execution, do an empty write with * `$keep_pipe = true` first. * @return this * @task interact */ public function write($data, $keep_pipe = false) { if (strlen($data)) { if (!$this->stdin) { throw new Exception(pht('Writing to a closed pipe!')); } $this->stdin->append($data); } $this->closePipe = !$keep_pipe; return $this; } /** * Permanently discard the stdout and stderr buffers and reset the read * cursors. This is basically useful only if you are streaming a large amount * of data from some process. * * Conceivably you might also need to do this if you're writing a client using * @{class:ExecFuture} and `netcat`, but you probably should not do that. * * NOTE: This completely discards the data. It won't be available when the * future resolves. This is almost certainly only useful if you need the * buffer memory for some reason. * * @return this * @task interact */ public function discardBuffers() { $this->discardStdoutBuffer(); $this->stderr = ''; $this->stderrPos = 0; return $this; } public function discardStdoutBuffer() { $this->stdout = ''; $this->stdoutPos = 0; return $this; } /** * Returns true if this future was killed by a timeout configured with * @{method:setTimeout}. * * @return bool True if the future was killed for exceeding its time limit. */ public function getWasKilledByTimeout() { return $this->killedByTimeout; } /* -( Configuring Execution )---------------------------------------------- */ /** * Set a hard limit on execution time. If the command runs longer, it will * be terminated and the future will resolve with an error code. You can test * if a future was killed by a timeout with @{method:getWasKilledByTimeout}. * * The subprocess will be sent a `TERM` signal, and then a `KILL` signal a * short while later if it fails to exit. * * @param int Maximum number of seconds this command may execute for before * it is signaled. * @return this * @task config */ public function setTimeout($seconds) { $this->terminateTimeout = $seconds; $this->killTimeout = $seconds + min($seconds, 60); return $this; } /* -( Resolving Execution )------------------------------------------------ */ /** * Resolve a command you expect to exit with return code 0. Works like * @{method:resolve}, but throws if $err is nonempty. Returns only * $stdout and $stderr. See also @{function:execx}. * * list($stdout, $stderr) = $future->resolvex(); * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return pair <$stdout, $stderr> pair. * @task resolve */ public function resolvex() { list($err, $stdout, $stderr) = $this->resolve(); if ($err) { $cmd = $this->getCommand(); if ($this->getWasKilledByTimeout()) { // NOTE: The timeout can be a float and PhutilNumber only handles // integers, so just use "%s" to render it. $message = pht( 'Command killed by timeout after running for more than %s seconds.', $this->terminateTimeout); } else { $message = pht('Command failed with error #%d!', $err); } throw new CommandException( $message, $cmd, $err, $stdout, $stderr); } return array($stdout, $stderr); } /** * Resolve a command you expect to return valid JSON. Works like * @{method:resolvex}, but also throws if stderr is nonempty, or stdout is not * valid JSON. Returns a PHP array, decoded from the JSON command output. * * @param float Optional timeout after which resolution will pause and * execution will return to the caller. * @return array PHP array, decoded from JSON command output. * @task resolve */ public function resolveJSON() { list($stdout, $stderr) = $this->resolvex(); if (strlen($stderr)) { $cmd = $this->getCommand(); throw new CommandException( pht( "JSON command '%s' emitted text to stderr when none was expected: %d", $cmd, $stderr), $cmd, 0, $stdout, $stderr); } try { return phutil_json_decode($stdout); } catch (PhutilJSONParserException $ex) { $cmd = $this->getCommand(); throw new CommandException( pht( "JSON command '%s' did not produce a valid JSON object on stdout: %s", $cmd, $stdout), $cmd, 0, $stdout, $stderr); } } /** * Resolve the process by abruptly terminating it. * * @return list List of results. * @task resolve */ public function resolveKill() { if (!$this->hasResult()) { $signal = 9; if ($this->proc) { proc_terminate($this->proc, $signal); } $result = array( 128 + $signal, $this->stdout, $this->stderr, ); $this->setResult($result); $this->closeProcess(); } return $this->getResult(); } /* -( Internals )---------------------------------------------------------- */ /** * Provides read sockets to the future core. * * @return list List of read sockets. * @task internal */ public function getReadSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdout) && !feof($stdout)) { $sockets[] = $stdout; } if (isset($stderr) && !feof($stderr)) { $sockets[] = $stderr; } return $sockets; } /** * Provides write sockets to the future core. * * @return list List of write sockets. * @task internal */ public function getWriteSockets() { list($stdin, $stdout, $stderr) = $this->pipes; $sockets = array(); if (isset($stdin) && $this->stdin->getByteLength() && !feof($stdin)) { $sockets[] = $stdin; } return $sockets; } /** * Determine if the read buffer is empty. * * @return bool True if the read buffer is empty. * @task internal */ public function isReadBufferEmpty() { return !strlen($this->stdout); } /** * Determine if the write buffer is empty. * * @return bool True if the write buffer is empty. * @task internal */ public function isWriteBufferEmpty() { return !$this->getWriteBufferSize(); } /** * Determine the number of bytes in the write buffer. * * @return int Number of bytes in the write buffer. * @task internal */ public function getWriteBufferSize() { if (!$this->stdin) { return 0; } return $this->stdin->getByteLength(); } /** * Reads some bytes from a stream, discarding output once a certain amount * has been accumulated. * * @param resource Stream to read from. * @param int Maximum number of bytes to return from $stream. If * additional bytes are available, they will be read and * discarded. * @param string Human-readable description of stream, for exception * message. * @param int Maximum number of bytes to read. * @return string The data read from the stream. * @task internal */ private function readAndDiscard($stream, $limit, $description, $length) { $output = ''; if ($length <= 0) { return ''; } do { $data = fread($stream, min($length, 64 * 1024)); if (false === $data) { throw new Exception(pht('Failed to read from %s', $description)); } $read_bytes = strlen($data); if ($read_bytes > 0 && $limit > 0) { if ($read_bytes > $limit) { $data = substr($data, 0, $limit); } $output .= $data; $limit -= strlen($data); } if (strlen($output) >= $length) { break; } } while ($read_bytes > 0); return $output; } /** * Begin or continue command execution. * * @return bool True if future has resolved. * @task internal */ public function isReady() { - // NOTE: We have soft dependencies on PhutilServiceProfiler and - // PhutilErrorTrap here. These dependencies are soft to avoid the need to - // build them into the Phage agent. Under normal circumstances, these - // classes are always available. + // NOTE: We have a soft dependencies on PhutilErrorTrap here, to avoid + // the need to build it into the Phage agent. Under normal circumstances, + // this class are always available. if (!$this->pipes) { $is_windows = phutil_is_windows(); - // NOTE: See note above about Phage. - if (class_exists('PhutilServiceProfiler')) { - $profiler = PhutilServiceProfiler::getInstance(); - $this->profilerCallID = $profiler->beginServiceCall( - array( - 'type' => 'exec', - 'command' => phutil_string_cast($this->getCommand()), - )); - } - if (!$this->start) { // We might already have started the timer via initiating resolution. $this->start = microtime(true); } $unmasked_command = $this->getCommand(); $unmasked_command = $unmasked_command->getUnmaskedString(); $pipes = array(); if ($this->hasEnv()) { $env = $this->getEnv(); } else { $env = null; } $cwd = $this->getCWD(); // NOTE: See note above about Phage. if (class_exists('PhutilErrorTrap')) { $trap = new PhutilErrorTrap(); } else { $trap = null; } $spec = self::$descriptorSpec; if ($is_windows) { $stdout_file = new TempFile(); $stderr_file = new TempFile(); $stdout_handle = fopen($stdout_file, 'wb'); if (!$stdout_handle) { throw new Exception( pht( 'Unable to open stdout temporary file ("%s") for writing.', $stdout_file)); } $stderr_handle = fopen($stderr_file, 'wb'); if (!$stderr_handle) { throw new Exception( pht( 'Unable to open stderr temporary file ("%s") for writing.', $stderr_file)); } $spec = array( 0 => self::$descriptorSpec[0], 1 => $stdout_handle, 2 => $stderr_handle, ); } $proc = @proc_open( $unmasked_command, $spec, $pipes, $cwd, $env, array( 'bypass_shell' => true, )); if ($trap) { $err = $trap->getErrorsAsString(); $trap->destroy(); } else { $err = error_get_last(); } if ($is_windows) { fclose($stdout_handle); fclose($stderr_handle); } if (!is_resource($proc)) { // When you run an invalid command on a Linux system, the "proc_open()" // works and then the process (really a "/bin/sh -c ...") exits after // it fails to resolve the command. // When you run an invalid command on a Windows system, we bypass the // shell and the "proc_open()" itself fails. Throw a "CommandException" // here for consistency with the Linux behavior in this common failure // case. throw new CommandException( pht( 'Call to "proc_open()" to open a subprocess failed: %s', $err), $this->getCommand(), 1, '', ''); } if ($is_windows) { $stdout_handle = fopen($stdout_file, 'rb'); if (!$stdout_handle) { throw new Exception( pht( 'Unable to open stdout temporary file ("%s") for reading.', $stdout_file)); } $stderr_handle = fopen($stderr_file, 'rb'); if (!$stderr_handle) { throw new Exception( pht( 'Unable to open stderr temporary file ("%s") for reading.', $stderr_file)); } $pipes = array( 0 => $pipes[0], 1 => $stdout_handle, 2 => $stderr_handle, ); $this->windowsStdoutTempFile = $stdout_file; $this->windowsStderrTempFile = $stderr_file; } $this->pipes = $pipes; $this->proc = $proc; list($stdin, $stdout, $stderr) = $pipes; if (!$is_windows) { // On Windows, we redirect process standard output and standard error // through temporary files. Files don't block, so we don't need to make // these streams nonblocking. if ((!stream_set_blocking($stdout, false)) || (!stream_set_blocking($stderr, false)) || (!stream_set_blocking($stdin, false))) { $this->__destruct(); throw new Exception(pht('Failed to set streams nonblocking.')); } } $this->tryToCloseStdin(); return false; } if (!$this->proc) { return true; } list($stdin, $stdout, $stderr) = $this->pipes; while (isset($this->stdin) && $this->stdin->getByteLength()) { $write_segment = $this->stdin->getAnyPrefix(); $bytes = fwrite($stdin, $write_segment); if ($bytes === false) { throw new Exception(pht('Unable to write to stdin!')); } else if ($bytes) { $this->stdin->removeBytesFromHead($bytes); } else { // Writes are blocked for now. break; } } $this->tryToCloseStdin(); // Read status before reading pipes so that we can never miss data that // arrives between our last read and the process exiting. $status = $this->procGetStatus(); $read_buffer_size = $this->readBufferSize; $max_stdout_read_bytes = PHP_INT_MAX; $max_stderr_read_bytes = PHP_INT_MAX; if ($read_buffer_size !== null) { $max_stdout_read_bytes = $read_buffer_size - strlen($this->stdout); $max_stderr_read_bytes = $read_buffer_size - strlen($this->stderr); } if ($max_stdout_read_bytes > 0) { $this->stdout .= $this->readAndDiscard( $stdout, $this->getStdoutSizeLimit() - strlen($this->stdout), 'stdout', $max_stdout_read_bytes); } if ($max_stderr_read_bytes > 0) { $this->stderr .= $this->readAndDiscard( $stderr, $this->getStderrSizeLimit() - strlen($this->stderr), 'stderr', $max_stderr_read_bytes); } $is_done = false; if (!$status['running']) { // We may still have unread bytes on stdout or stderr, particularly if // this future is being buffered and streamed. If we do, we don't want to // consider the subprocess to have exited until we've read everything. // See T9724 for context. if (feof($stdout) && feof($stderr)) { $is_done = true; } } if ($is_done) { // If the subprocess got nuked with `kill -9`, we get a -1 exitcode. // Upgrade this to a slightly more informative value by examining the // terminating signal code. $err = $status['exitcode']; if ($err == -1) { if ($status['signaled']) { $err = 128 + $status['termsig']; } } $result = array( $err, $this->stdout, $this->stderr, ); $this->setResult($result); $this->closeProcess(); return true; } $elapsed = (microtime(true) - $this->start); if ($this->terminateTimeout && ($elapsed >= $this->terminateTimeout)) { if (!$this->didTerminate) { $this->killedByTimeout = true; $this->sendTerminateSignal(); return false; } } if ($this->killTimeout && ($elapsed >= $this->killTimeout)) { $this->killedByTimeout = true; $this->resolveKill(); return true; } } /** * @return void * @task internal */ public function __destruct() { if (!$this->proc) { return; } // NOTE: If we try to proc_close() an open process, we hang indefinitely. To // avoid this, kill the process explicitly if it's still running. $status = $this->procGetStatus(); if ($status['running']) { $this->sendTerminateSignal(); if (!$this->waitForExit(5)) { $this->resolveKill(); } } else { $this->closeProcess(); } } /** * Close and free resources if necessary. * * @return void * @task internal */ private function closeProcess() { foreach ($this->pipes as $pipe) { if (isset($pipe)) { @fclose($pipe); } } $this->pipes = array(null, null, null); if ($this->proc) { @proc_close($this->proc); $this->proc = null; } $this->stdin = null; unset($this->windowsStdoutTempFile); unset($this->windowsStderrTempFile); - - if ($this->profilerCallID !== null) { - if ($this->hasResult()) { - $result = $this->getResult(); - $err = idx($result, 0); - } else { - $err = null; - } - - $profiler = PhutilServiceProfiler::getInstance(); - $profiler->endServiceCall( - $this->profilerCallID, - array( - 'err' => $err, - )); - $this->profilerCallID = null; - } } /** * Execute `proc_get_status()`, but avoid pitfalls. * * @return dict Process status. * @task internal */ private function procGetStatus() { // After the process exits, we only get one chance to read proc_get_status() // before it starts returning garbage. Make sure we don't throw away the // last good read. if ($this->procStatus) { if (!$this->procStatus['running']) { return $this->procStatus; } } $this->procStatus = proc_get_status($this->proc); return $this->procStatus; } /** * Try to close stdin, if we're done using it. This keeps us from hanging if * the process on the other end of the pipe is waiting for EOF. * * @return void * @task internal */ private function tryToCloseStdin() { if (!$this->closePipe) { // We've been told to keep the pipe open by a call to write(..., true). return; } if ($this->stdin->getByteLength()) { // We still have bytes to write. return; } list($stdin) = $this->pipes; if (!$stdin) { // We've already closed stdin. return; } // There's nothing stopping us from closing stdin, so close it. @fclose($stdin); $this->pipes[0] = null; } public function getDefaultWait() { $wait = parent::getDefaultWait(); $next_timeout = $this->getNextTimeout(); if ($next_timeout) { if (!$this->start) { $this->start = microtime(true); } $elapsed = (microtime(true) - $this->start); $wait = max(0, min($next_timeout - $elapsed, $wait)); } return $wait; } private function getNextTimeout() { if ($this->didTerminate) { return $this->killTimeout; } else { return $this->terminateTimeout; } } private function sendTerminateSignal() { $this->didTerminate = true; proc_terminate($this->proc); return $this; } private function waitForExit($duration) { $start = microtime(true); while (true) { $status = $this->procGetStatus(); if (!$status['running']) { return true; } $waited = (microtime(true) - $start); if ($waited > $duration) { return false; } } } + protected function getServiceProfilerStartParameters() { + return array( + 'type' => 'exec', + 'command' => phutil_string_cast($this->getCommand()), + ); + } + + protected function getServiceProfilerResultParameters() { + if ($this->hasResult()) { + $result = $this->getResult(); + $err = idx($result, 0); + } else { + $err = null; + } + + return array( + 'err' => $err, + ); + } + + } diff --git a/src/future/exec/PhutilExecPassthru.php b/src/future/exec/PhutilExecPassthru.php index 8a761923..44eef97e 100644 --- a/src/future/exec/PhutilExecPassthru.php +++ b/src/future/exec/PhutilExecPassthru.php @@ -1,114 +1,122 @@ execute(); * * You can set the current working directory for the command with * @{method:setCWD}, and set the environment with @{method:setEnv}. * * @task command Executing Passthru Commands */ final class PhutilExecPassthru extends PhutilExecutableFuture { /* -( Executing Passthru Commands )---------------------------------------- */ /** * Execute this command. * * @return int Error code returned by the subprocess. * * @task command */ public function execute() { $command = $this->getCommand(); - $profiler = PhutilServiceProfiler::getInstance(); - $call_id = $profiler->beginServiceCall( - array( - 'type' => 'exec', - 'subtype' => 'passthru', - 'command' => $command, - )); - $spec = array(STDIN, STDOUT, STDERR); $pipes = array(); $unmasked_command = $command->getUnmaskedString(); if ($this->hasEnv()) { $env = $this->getEnv(); } else { $env = null; } $cwd = $this->getCWD(); $options = array(); if (phutil_is_windows()) { // Without 'bypass_shell', things like launching vim don't work properly, // and we can't execute commands with spaces in them, and all commands // invoked from git bash fail horridly, and everything is a mess in // general. $options['bypass_shell'] = true; } $trap = new PhutilErrorTrap(); $proc = @proc_open( $unmasked_command, $spec, $pipes, $cwd, $env, $options); $errors = $trap->getErrorsAsString(); $trap->destroy(); if (!is_resource($proc)) { throw new Exception( pht( 'Failed to passthru %s: %s', 'proc_open()', $errors)); } $err = proc_close($proc); - $profiler->endServiceCall( - $call_id, - array( - 'err' => $err, - )); - return $err; } /* -( Future )------------------------------------------------------------- */ public function isReady() { // This isn't really a future because it executes synchronously and has // full control of the console. We're just implementing the interfaces to // make it easier to share code with ExecFuture. if (!$this->hasResult()) { $result = $this->execute(); $this->setResult($result); } return true; } + + + protected function getServiceProfilerStartParameters() { + return array( + 'type' => 'exec', + 'subtype' => 'passthru', + 'command' => phutil_string_cast($this->getCommand()), + ); + } + + protected function getServiceProfilerResultParameters() { + if ($this->hasResult()) { + $err = $this->getResult(); + } else { + $err = null; + } + + return array( + 'err' => $err, + ); + } + } diff --git a/src/future/http/HTTPFuture.php b/src/future/http/HTTPFuture.php index 738cde27..91dd709a 100644 --- a/src/future/http/HTTPFuture.php +++ b/src/future/http/HTTPFuture.php @@ -1,306 +1,303 @@ resolvex(); * * Or * * $future = new HTTPFuture('http://www.example.com/'); * list($http_response_status_object, * $response_body, * $headers) = $future->resolve(); * * Prefer @{method:resolvex} to @{method:resolve} as the former throws * @{class:HTTPFutureHTTPResponseStatus} on failures, which includes an * informative exception message. */ final class HTTPFuture extends BaseHTTPFuture { private $host; private $port = 80; private $fullRequestPath; private $socket; private $writeBuffer; private $response; private $stateConnected = false; private $stateWriteComplete = false; private $stateReady = false; private $stateStartTime; private $profilerCallID; public function setURI($uri) { $parts = parse_url($uri); if (!$parts) { throw new Exception(pht("Could not parse URI '%s'.", $uri)); } if (empty($parts['scheme']) || $parts['scheme'] !== 'http') { throw new Exception( pht( "URI '%s' must be fully qualified with '%s' scheme.", $uri, 'http://')); } if (!isset($parts['host'])) { throw new Exception( pht("URI '%s' must be fully qualified and include host name.", $uri)); } $this->host = $parts['host']; if (!empty($parts['port'])) { $this->port = $parts['port']; } if (isset($parts['user']) || isset($parts['pass'])) { throw new Exception( pht('HTTP Basic Auth is not supported by %s.', __CLASS__)); } if (isset($parts['path'])) { $this->fullRequestPath = $parts['path']; } else { $this->fullRequestPath = '/'; } if (isset($parts['query'])) { $this->fullRequestPath .= '?'.$parts['query']; } return parent::setURI($uri); } public function __destruct() { if ($this->socket) { @fclose($this->socket); $this->socket = null; } } public function getReadSockets() { if ($this->socket) { return array($this->socket); } return array(); } public function getWriteSockets() { if (strlen($this->writeBuffer)) { return array($this->socket); } return array(); } public function isWriteComplete() { return $this->stateWriteComplete; } private function getDefaultUserAgent() { return __CLASS__.'/1.0'; } public function isReady() { if ($this->stateReady) { return true; } if (!$this->socket) { $this->stateStartTime = microtime(true); $this->socket = $this->buildSocket(); if (!$this->socket) { return $this->stateReady; } - - $profiler = PhutilServiceProfiler::getInstance(); - $this->profilerCallID = $profiler->beginServiceCall( - array( - 'type' => 'http', - 'uri' => $this->getURI(), - )); } if (!$this->stateConnected) { $read = array(); $write = array($this->socket); $except = array(); $select = stream_select($read, $write, $except, $tv_sec = 0); if ($write) { $this->stateConnected = true; } } if ($this->stateConnected) { if (strlen($this->writeBuffer)) { $bytes = @fwrite($this->socket, $this->writeBuffer); if ($bytes === false) { throw new Exception(pht('Failed to write to buffer.')); } else if ($bytes) { $this->writeBuffer = substr($this->writeBuffer, $bytes); } } if (!strlen($this->writeBuffer)) { $this->stateWriteComplete = true; } while (($data = fread($this->socket, 32768)) || strlen($data)) { $this->response .= $data; } if ($data === false) { throw new Exception(pht('Failed to read socket.')); } } return $this->checkSocket(); } private function buildSocket() { $errno = null; $errstr = null; $socket = @stream_socket_client( 'tcp://'.$this->host.':'.$this->port, $errno, $errstr, $ignored_connection_timeout = 1.0, STREAM_CLIENT_CONNECT | STREAM_CLIENT_ASYNC_CONNECT); if (!$socket) { $this->stateReady = true; $this->setResult( $this->buildErrorResult( HTTPFutureTransportResponseStatus::ERROR_CONNECTION_FAILED)); return null; } $ok = stream_set_blocking($socket, 0); if (!$ok) { throw new Exception(pht('Failed to set stream nonblocking.')); } $this->writeBuffer = $this->buildHTTPRequest(); return $socket; } private function checkSocket() { $timeout = false; $now = microtime(true); if (($now - $this->stateStartTime) > $this->getTimeout()) { $timeout = true; } if (!feof($this->socket) && !$timeout) { return false; } $this->stateReady = true; if ($timeout) { $this->setResult( $this->buildErrorResult( HTTPFutureTransportResponseStatus::ERROR_TIMEOUT)); } else if (!$this->stateConnected) { $this->setResult( $this->buildErrorResult( HTTPFutureTransportResponseStatus::ERROR_CONNECTION_REFUSED)); } else if (!$this->stateWriteComplete) { $this->setResult( $this->buildErrorResult( HTTPFutureTransportResponseStatus::ERROR_CONNECTION_FAILED)); } else { $this->setResult($this->parseRawHTTPResponse($this->response)); } - $profiler = PhutilServiceProfiler::getInstance(); - $profiler->endServiceCall($this->profilerCallID, array()); - return true; } private function buildErrorResult($error) { return array( $status = new HTTPFutureTransportResponseStatus($error, $this->getURI()), $body = null, $headers = array(), ); } private function buildHTTPRequest() { $data = $this->getData(); $method = $this->getMethod(); $uri = $this->fullRequestPath; $add_headers = array(); if ($this->getMethod() == 'GET') { if (is_array($data)) { $data = phutil_build_http_querystring($data); if (strpos($uri, '?') !== false) { $uri .= '&'.$data; } else { $uri .= '?'.$data; } $data = ''; } } else { if (is_array($data)) { $data = phutil_build_http_querystring($data)."\r\n"; $add_headers[] = array( 'Content-Type', 'application/x-www-form-urlencoded', ); } } $length = strlen($data); $add_headers[] = array( 'Content-Length', $length, ); if (!$this->getHeaders('User-Agent')) { $add_headers[] = array( 'User-Agent', $this->getDefaultUserAgent(), ); } if (!$this->getHeaders('Host')) { $add_headers[] = array( 'Host', $this->host, ); } $headers = array_merge($this->getHeaders(), $add_headers); foreach ($headers as $key => $header) { list($name, $value) = $header; if (strlen($value)) { $value = ': '.$value; } $headers[$key] = $name.$value."\r\n"; } return "{$method} {$uri} HTTP/1.0\r\n". implode('', $headers). "\r\n". $data; } + protected function getServiceProfilerStartParameters() { + return array( + 'type' => 'http', + 'uri' => phutil_string_cast($this->getURI()), + ); + } + } diff --git a/src/future/http/HTTPSFuture.php b/src/future/http/HTTPSFuture.php index f65af3e8..70804332 100644 --- a/src/future/http/HTTPSFuture.php +++ b/src/future/http/HTTPSFuture.php @@ -1,824 +1,824 @@ cabundle = $temp; return $this; } /** * Set the SSL certificate to use for this session, given a path. * * @param string The path to a valid SSL certificate for this session * @return this */ public function setCABundleFromPath($path) { $this->cabundle = $path; return $this; } /** * Get the path to the SSL certificate for this session. * * @return string|null */ public function getCABundle() { return $this->cabundle; } /** * Set whether Location headers in the response will be respected. * The default is true. * * @param boolean true to follow any Location header present in the response, * false to return the request directly * @return this */ public function setFollowLocation($follow) { $this->followLocation = $follow; return $this; } /** * Get whether Location headers in the response will be respected. * * @return boolean */ public function getFollowLocation() { return $this->followLocation; } /** * Set the fallback CA certificate if one is not specified * for the session, given a path. * * @param string The path to a valid SSL certificate * @return void */ public static function setGlobalCABundleFromPath($path) { self::$globalCABundle = $path; } /** * Set the fallback CA certificate if one is not specified * for the session, given a string. * * @param string The certificate * @return void */ public static function setGlobalCABundleFromString($certificate) { $temp = new TempFile(); Filesystem::writeFile($temp, $certificate); self::$globalCABundle = $temp; } /** * Get the fallback global CA certificate * * @return string */ public static function getGlobalCABundle() { return self::$globalCABundle; } /** * Load contents of remote URI. Behaves pretty much like * `@file_get_contents($uri)` but doesn't require `allow_url_fopen`. * * @param string * @param float * @return string|false */ public static function loadContent($uri, $timeout = null) { $future = new HTTPSFuture($uri); if ($timeout !== null) { $future->setTimeout($timeout); } try { list($body) = $future->resolvex(); return $body; } catch (HTTPFutureResponseStatus $ex) { return false; } } public function setDownloadPath($download_path) { $this->downloadPath = $download_path; if (Filesystem::pathExists($download_path)) { throw new Exception( pht( 'Specified download path "%s" already exists, refusing to '. 'overwrite.')); } return $this; } public function setProgressSink(PhutilProgressSink $progress_sink) { $this->progressSink = $progress_sink; return $this; } public function getProgressSink() { return $this->progressSink; } /** * Attach a file to the request. * * @param string HTTP parameter name. * @param string File content. * @param string File name. * @param string File mime type. * @return this */ public function attachFileData($key, $data, $name, $mime_type) { if (isset($this->files[$key])) { throw new Exception( pht( '%s currently supports only one file attachment for each '. 'parameter name. You are trying to attach two different files with '. 'the same parameter, "%s".', __CLASS__, $key)); } $this->files[$key] = array( 'data' => $data, 'name' => $name, 'mime' => $mime_type, ); return $this; } public function isReady() { if ($this->hasResult()) { return true; } $uri = $this->getURI(); $domain = id(new PhutilURI($uri))->getDomain(); $is_download = $this->isDownload(); // See T13396. For now, use the streaming response parser only if we're // downloading the response to disk. $use_streaming_parser = (bool)$is_download; if (!$this->handle) { $uri_object = new PhutilURI($uri); $proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object); - $profiler = PhutilServiceProfiler::getInstance(); - $this->profilerCallID = $profiler->beginServiceCall( - array( - 'type' => 'http', - 'uri' => $uri, - 'proxy' => (string)$proxy, - )); + // TODO: Currently, the "proxy" is not passed to the ServiceProfiler + // because of changes to how ServiceProfiler is integrated. It would + // be nice to pass it again. if (!self::$multi) { self::$multi = curl_multi_init(); if (!self::$multi) { throw new Exception(pht('%s failed!', 'curl_multi_init()')); } } if (!empty(self::$pool[$domain])) { $curl = array_pop(self::$pool[$domain]); } else { $curl = curl_init(); if (!$curl) { throw new Exception(pht('%s failed!', 'curl_init()')); } } $this->handle = $curl; curl_multi_add_handle(self::$multi, $curl); curl_setopt($curl, CURLOPT_URL, $uri); if (defined('CURLOPT_PROTOCOLS')) { // cURL supports a lot of protocols, and by default it will honor // redirects across protocols (for instance, from HTTP to POP3). Beyond // being very silly, this also has security implications: // // http://blog.volema.com/curl-rce.html // // Disable all protocols other than HTTP and HTTPS. $allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP; curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols); curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols); } if (strlen($this->rawBody)) { if ($this->getData()) { throw new Exception( pht( 'You can not execute an HTTP future with both a raw request '. 'body and structured request data.')); } // We aren't actually going to use this file handle, since we are // just pushing data through the callback, but cURL gets upset if // we don't hand it a real file handle. $tmp = new TempFile(); $this->fileHandle = fopen($tmp, 'r'); // NOTE: We must set CURLOPT_PUT here to make cURL use CURLOPT_INFILE. // We'll possibly overwrite the method later on, unless this is really // a PUT request. curl_setopt($curl, CURLOPT_PUT, true); curl_setopt($curl, CURLOPT_INFILE, $this->fileHandle); curl_setopt($curl, CURLOPT_INFILESIZE, strlen($this->rawBody)); curl_setopt($curl, CURLOPT_READFUNCTION, array($this, 'willWriteBody')); } else { $data = $this->formatRequestDataForCURL(); curl_setopt($curl, CURLOPT_POSTFIELDS, $data); } $headers = $this->getHeaders(); $saw_expect = false; for ($ii = 0; $ii < count($headers); $ii++) { list($name, $value) = $headers[$ii]; $headers[$ii] = $name.': '.$value; if (!strncasecmp($name, 'Expect', strlen('Expect'))) { $saw_expect = true; } } if (!$saw_expect) { // cURL sends an "Expect" header by default for certain requests. While // there is some reasoning behind this, it causes a practical problem // in that lighttpd servers reject these requests with a 417. Both sides // are locked in an eternal struggle (lighttpd has introduced a // 'server.reject-expect-100-with-417' option to deal with this case). // // The ostensibly correct way to suppress this behavior on the cURL side // is to add an empty "Expect:" header. If we haven't seen some other // explicit "Expect:" header, do so. // // See here, for example, although this issue is fairly widespread: // http://curl.haxx.se/mail/archive-2009-07/0008.html $headers[] = 'Expect:'; } curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); // Set the requested HTTP method, e.g. GET / POST / PUT. curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod()); // Make sure we get the headers and data back. curl_setopt($curl, CURLOPT_HEADER, true); curl_setopt($curl, CURLOPT_WRITEFUNCTION, array($this, 'didReceiveDataCallback')); if ($this->followLocation) { curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); curl_setopt($curl, CURLOPT_MAXREDIRS, 20); } if (defined('CURLOPT_TIMEOUT_MS')) { // If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout. $timeout = max(1, ceil(1000 * $this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout); } else { // Otherwise, fall back to the lower-precision timeout. $timeout = max(1, ceil($this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT, $timeout); } // We're going to try to set CAINFO below. This doesn't work at all on // OSX around Yosemite (see T5913). On these systems, we'll use the // system CA and then try to tell the user that their settings were // ignored and how to fix things if we encounter a CA-related error. // Assume we have custom CA settings to start with; we'll clear this // flag if we read the default CA info below. // Try some decent fallbacks here: // - First, check if a bundle is set explicitly for this request, via // `setCABundle()` or similar. // - Then, check if a global bundle is set explicitly for all requests, // via `setGlobalCABundle()` or similar. // - Then, if a local custom.pem exists, use that, because it probably // means that the user wants to override everything (also because the // user might not have access to change the box's php.ini to add // curl.cainfo). // - Otherwise, try using curl.cainfo. If it's set explicitly, it's // probably reasonable to try using it before we fall back to what // libphutil ships with. // - Lastly, try the default that libphutil ships with. If it doesn't // work, give up and yell at the user. if (!$this->getCABundle()) { $caroot = dirname(phutil_get_library_root('arcanist')); $caroot = $caroot.'/resources/ssl/'; $ini_val = ini_get('curl.cainfo'); if (self::getGlobalCABundle()) { $this->setCABundleFromPath(self::getGlobalCABundle()); } else if (Filesystem::pathExists($caroot.'custom.pem')) { $this->setCABundleFromPath($caroot.'custom.pem'); } else if ($ini_val) { // TODO: We can probably do a pathExists() here, even. $this->setCABundleFromPath($ini_val); } else { $this->setCABundleFromPath($caroot.'default.pem'); } } if ($this->canSetCAInfo()) { curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle()); } $verify_peer = 1; $verify_host = 2; $extensions = PhutilHTTPEngineExtension::getAllExtensions(); foreach ($extensions as $extension) { if ($extension->shouldTrustAnySSLAuthorityForURI($uri_object)) { $verify_peer = 0; } if ($extension->shouldTrustAnySSLHostnameForURI($uri_object)) { $verify_host = 0; } } curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, $verify_peer); curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, $verify_host); curl_setopt($curl, CURLOPT_SSLVERSION, 0); // See T13391. Recent versions of cURL default to "HTTP/2" on some // connections, but do not support HTTP/2 proxies. Until HTTP/2 // stabilizes, force HTTP/1.1 explicitly. curl_setopt($curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); if ($proxy) { curl_setopt($curl, CURLOPT_PROXY, (string)$proxy); } if ($is_download) { $this->downloadHandle = @fopen($this->downloadPath, 'wb+'); if (!$this->downloadHandle) { throw new Exception( pht( 'Failed to open filesystem path "%s" for writing.', $this->downloadPath)); } } if ($use_streaming_parser) { $streaming_parser = id(new PhutilHTTPResponseParser()) ->setFollowLocationHeaders($this->getFollowLocation()); if ($this->downloadHandle) { $streaming_parser->setWriteHandle($this->downloadHandle); } $progress_sink = $this->getProgressSink(); if ($progress_sink) { $streaming_parser->setProgressSink($progress_sink); } $this->parser = $streaming_parser; } } else { $curl = $this->handle; if (!self::$results) { // NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does // not check the return value of &maxfd for -1 until recent versions // of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual // situations; if it does, PHP enters select() with nfds=0, which blocks // until the timeout is reached. // // We could try to guess whether this will happen or not by examining // the version identifier, but we can also just sleep for only a short // period of time. curl_multi_select(self::$multi, 0.01); } } do { $active = null; $result = curl_multi_exec(self::$multi, $active); } while ($result == CURLM_CALL_MULTI_PERFORM); while ($info = curl_multi_info_read(self::$multi)) { if ($info['msg'] == CURLMSG_DONE) { self::$results[(int)$info['handle']] = $info; } } if (!array_key_exists((int)$curl, self::$results)) { return false; } // The request is complete, so release any temporary files we wrote // earlier. $this->temporaryFiles = array(); $info = self::$results[(int)$curl]; $result = $this->responseBuffer; $err_code = $info['result']; if ($err_code) { if (($err_code == CURLE_SSL_CACERT) && !$this->canSetCAInfo()) { $status = new HTTPFutureCertificateResponseStatus( HTTPFutureCertificateResponseStatus::ERROR_IMMUTABLE_CERTIFICATES, $uri); } else { $status = new HTTPFutureCURLResponseStatus($err_code, $uri); } $body = null; $headers = array(); $this->setResult(array($status, $body, $headers)); } else if ($this->parser) { $streaming_parser = $this->parser; try { $responses = $streaming_parser->getResponses(); $final_response = last($responses); $result = array( $final_response->getStatus(), $final_response->getBody(), $final_response->getHeaders(), ); } catch (HTTPFutureParseResponseStatus $ex) { $result = array($ex, null, array()); } $this->setResult($result); } else { // cURL returns headers of all redirects, we strip all but the final one. $redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT); $result = preg_replace('/^(.*\r\n\r\n){'.$redirects.'}/sU', '', $result); $this->setResult($this->parseRawHTTPResponse($result)); } curl_multi_remove_handle(self::$multi, $curl); unset(self::$results[(int)$curl]); // NOTE: We want to use keepalive if possible. Return the handle to a // pool for the domain; don't close it. if ($this->shouldReuseHandles()) { self::$pool[$domain][] = $curl; } if ($is_download) { if ($this->downloadHandle) { fflush($this->downloadHandle); fclose($this->downloadHandle); $this->downloadHandle = null; } } $sink = $this->getProgressSink(); if ($sink) { $status = head($this->getResult()); if ($status->isError()) { $sink->didFailWork(); } else { $sink->didCompleteWork(); } } - $profiler = PhutilServiceProfiler::getInstance(); - $profiler->endServiceCall($this->profilerCallID, array()); - return true; } /** * Callback invoked by cURL as it reads HTTP data from the response. We save * the data to a buffer. */ public function didReceiveDataCallback($handle, $data) { if ($this->parser) { $this->parser->readBytes($data); } else { $this->responseBuffer .= $data; } return strlen($data); } /** * Read data from the response buffer. * * NOTE: Like @{class:ExecFuture}, this method advances a read cursor but * does not discard the data. The data will still be buffered, and it will * all be returned when the future resolves. To discard the data after * reading it, call @{method:discardBuffers}. * * @return string Response data, if available. */ public function read() { if ($this->isDownload()) { throw new Exception( pht( 'You can not read the result buffer while streaming results '. 'to disk: there is no in-memory buffer to read.')); } if ($this->parser) { throw new Exception( pht( 'Streaming reads are not currently supported by the streaming '. 'parser.')); } $result = substr($this->responseBuffer, $this->responseBufferPos); $this->responseBufferPos = strlen($this->responseBuffer); return $result; } /** * Discard any buffered data. Normally, you call this after reading the * data with @{method:read}. * * @return this */ public function discardBuffers() { if ($this->isDownload()) { throw new Exception( pht( 'You can not discard the result buffer while streaming results '. 'to disk: there is no in-memory buffer to discard.')); } if ($this->parser) { throw new Exception( pht( 'Buffer discards are not currently supported by the streaming '. 'parser.')); } $this->responseBuffer = ''; $this->responseBufferPos = 0; return $this; } /** * Produces a value safe to pass to `CURLOPT_POSTFIELDS`. * * @return wild Some value, suitable for use in `CURLOPT_POSTFIELDS`. */ private function formatRequestDataForCURL() { // We're generating a value to hand to cURL as CURLOPT_POSTFIELDS. The way // cURL handles this value has some tricky caveats. // First, we can return either an array or a query string. If we return // an array, we get a "multipart/form-data" request. If we return a // query string, we get an "application/x-www-form-urlencoded" request. // Second, if we return an array we can't duplicate keys. The user might // want to send the same parameter multiple times. // Third, if we return an array and any of the values start with "@", // cURL includes arbitrary files off disk and sends them to an untrusted // remote server. For example, an array like: // // array('name' => '@/usr/local/secret') // // ...will attempt to read that file off disk and transmit its contents with // the request. This behavior is pretty surprising, and it can easily // become a relatively severe security vulnerability which allows an // attacker to read any file the HTTP process has access to. Since this // feature is very dangerous and not particularly useful, we prevent its // use. Broadly, this means we must reject some requests because they // contain an "@" in an inconvenient place. // Generally, to avoid the "@" case and because most servers usually // expect "application/x-www-form-urlencoded" data, we try to return a // string unless there are files attached to this request. $data = $this->getData(); $files = $this->files; $any_data = ($data || (is_string($data) && strlen($data))); $any_files = (bool)$this->files; if (!$any_data && !$any_files) { // No files or data, so just bail. return null; } if (!$any_files) { // If we don't have any files, just encode the data as a query string, // make sure it's not including any files, and we're good to go. if (is_array($data)) { $data = phutil_build_http_querystring($data); } $this->checkForDangerousCURLMagic($data, $is_query_string = true); return $data; } // If we've made it this far, we have some files, so we need to return // an array. First, convert the other data into an array if it isn't one // already. if (is_string($data)) { // NOTE: We explicitly don't want fancy array parsing here, so just // do a basic parse and then convert it into a dictionary ourselves. $parser = new PhutilQueryStringParser(); $pairs = $parser->parseQueryStringToPairList($data); $map = array(); foreach ($pairs as $pair) { list($key, $value) = $pair; if (array_key_exists($key, $map)) { throw new Exception( pht( 'Request specifies two values for key "%s", but parameter '. 'names must be unique if you are posting file data due to '. 'limitations with cURL.', $key)); } $map[$key] = $value; } $data = $map; } foreach ($data as $key => $value) { $this->checkForDangerousCURLMagic($value, $is_query_string = false); } foreach ($this->files as $name => $info) { if (array_key_exists($name, $data)) { throw new Exception( pht( 'Request specifies a file with key "%s", but that key is also '. 'defined by normal request data. Due to limitations with cURL, '. 'requests that post file data must use unique keys.', $name)); } $tmp = new TempFile($info['name']); Filesystem::writeFile($tmp, $info['data']); $this->temporaryFiles[] = $tmp; // In 5.5.0 and later, we can use CURLFile. Prior to that, we have to // use this "@" stuff. if (class_exists('CURLFile', false)) { $file_value = new CURLFile((string)$tmp, $info['mime'], $info['name']); } else { $file_value = '@'.(string)$tmp; } $data[$name] = $file_value; } return $data; } /** * Detect strings which will cause cURL to do horrible, insecure things. * * @param string Possibly dangerous string. * @param bool True if this string is being used as part of a query string. * @return void */ private function checkForDangerousCURLMagic($string, $is_query_string) { if (empty($string[0]) || ($string[0] != '@')) { // This isn't an "@..." string, so it's fine. return; } if ($is_query_string) { if (version_compare(phpversion(), '5.2.0', '<')) { throw new Exception( pht( 'Attempting to make an HTTP request, but query string data begins '. 'with "%s". Prior to PHP 5.2.0 this reads files off disk, which '. 'creates a wide attack window for security vulnerabilities. '. 'Upgrade PHP or avoid making cURL requests which begin with "%s".', '@', '@')); } // This is safe if we're on PHP 5.2.0 or newer. return; } throw new Exception( pht( 'Attempting to make an HTTP request which includes file data, but the '. 'value of a query parameter begins with "%s". PHP interprets these '. 'values to mean that it should read arbitrary files off disk and '. 'transmit them to remote servers. Declining to make this request.', '@')); } /** * Determine whether CURLOPT_CAINFO is usable on this system. */ private function canSetCAInfo() { // We cannot set CAInfo on OSX after Yosemite. $osx_version = PhutilExecutionEnvironment::getOSXVersion(); if ($osx_version) { if (version_compare($osx_version, 14, '>=')) { return false; } } return true; } /** * Write a raw HTTP body into the request. * * You must write the entire body before starting the request. * * @param string Raw body. * @return this */ public function write($raw_body) { $this->rawBody = $raw_body; return $this; } /** * Callback to pass data to cURL. */ public function willWriteBody($handle, $infile, $len) { $bytes = substr($this->rawBody, $this->rawBodyPos, $len); $this->rawBodyPos += $len; return $bytes; } private function shouldReuseHandles() { $curl_version = curl_version(); $version = idx($curl_version, 'version'); // NOTE: cURL 7.43.0 has a bug where the POST body length is not recomputed // properly when a handle is reused. For this version of cURL, disable // handle reuse and accept a small performance penalty. See T8654. if ($version == '7.43.0') { return false; } return true; } private function isDownload() { return ($this->downloadPath !== null); } + protected function getServiceProfilerStartParameters() { + return array( + 'type' => 'http', + 'uri' => phutil_string_cast($this->getURI()), + ); + } + }