diff --git a/src/future/http/HTTPSFuture.php b/src/future/http/HTTPSFuture.php --- a/src/future/http/HTTPSFuture.php +++ b/src/future/http/HTTPSFuture.php @@ -22,6 +22,10 @@ private $rawBodyPos = 0; private $fileHandle; + private $downloadPath; + private $downloadHandle; + private $parser; + /** * Create a temp file containing an SSL cert, and use it for this session. * @@ -137,6 +141,19 @@ } } + public function setDownloadPath($download_path) { + $this->downloadPath = $download_path; + + if (Filesystem::pathExists($download_path)) { + throw new Exception( + pht( + 'Specified download path "%s" already exists, refusing to '. + 'overwrite.')); + } + + return $this; + } + /** * Attach a file to the request. * @@ -174,6 +191,12 @@ $uri = $this->getURI(); $domain = id(new PhutilURI($uri))->getDomain(); + $is_download = $this->isDownload(); + + // See T13396. For now, use the streaming response parser only if we're + // downloading the response to disk. + $use_streaming_parser = (bool)$is_download; + if (!$this->handle) { $uri_object = new PhutilURI($uri); $proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object); @@ -364,6 +387,27 @@ if ($proxy) { curl_setopt($curl, CURLOPT_PROXY, (string)$proxy); } + + if ($is_download) { + $this->downloadHandle = @fopen($this->downloadPath, 'wb+'); + if (!$this->downloadHandle) { + throw new Exception( + pht( + 'Failed to open filesystem path "%s" for writing.', + $this->downloadPath)); + } + } + + if ($use_streaming_parser) { + $streaming_parser = id(new PhutilHTTPResponseParser()) + ->setFollowLocationHeaders($this->getFollowLocation()); + + if ($this->downloadHandle) { + $streaming_parser->setWriteHandle($this->downloadHandle); + } + + $this->parser = $streaming_parser; + } } else { $curl = $this->handle; @@ -416,6 +460,21 @@ $body = null; $headers = array(); $this->result = array($status, $body, $headers); + } else if ($this->parser) { + $streaming_parser = $this->parser; + try { + $responses = $streaming_parser->getResponses(); + $final_response = last($responses); + $result = array( + $final_response->getStatus(), + $final_response->getBody(), + $final_response->getHeaders(), + ); + } catch (HTTPFutureParseResponseStatus $ex) { + $result = array($ex, null, array()); + } + + $this->result = $result; } else { // cURL returns headers of all redirects, we strip all but the final one. $redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT); @@ -432,6 +491,14 @@ self::$pool[$domain][] = $curl; } + if ($is_download) { + if ($this->downloadHandle) { + fflush($this->downloadHandle); + fclose($this->downloadHandle); + $this->downloadHandle = null; + } + } + $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); @@ -444,7 +511,12 @@ * the data to a buffer. */ public function didReceiveDataCallback($handle, $data) { - $this->responseBuffer .= $data; + if ($this->parser) { + $this->parser->readBytes($data); + } else { + $this->responseBuffer .= $data; + } + return strlen($data); } @@ -460,6 +532,20 @@ * @return string Response data, if available. */ public function read() { + if ($this->isDownload()) { + throw new Exception( + pht( + 'You can not read the result buffer while streaming results '. + 'to disk: there is no in-memory buffer to read.')); + } + + if ($this->parser) { + throw new Exception( + pht( + 'Streaming reads are not currently supported by the streaming '. + 'parser.')); + } + $result = substr($this->responseBuffer, $this->responseBufferPos); $this->responseBufferPos = strlen($this->responseBuffer); return $result; @@ -473,6 +559,20 @@ * @return this */ public function discardBuffers() { + if ($this->isDownload()) { + throw new Exception( + pht( + 'You can not discard the result buffer while streaming results '. + 'to disk: there is no in-memory buffer to discard.')); + } + + if ($this->parser) { + throw new Exception( + pht( + 'Buffer discards are not currently supported by the streaming '. + 'parser.')); + } + $this->responseBuffer = ''; $this->responseBufferPos = 0; return $this; @@ -690,5 +790,8 @@ return true; } + private function isDownload() { + return ($this->downloadPath !== null); + } } diff --git a/src/parser/http/PhutilHTTPResponse.php b/src/parser/http/PhutilHTTPResponse.php --- a/src/parser/http/PhutilHTTPResponse.php +++ b/src/parser/http/PhutilHTTPResponse.php @@ -6,6 +6,7 @@ private $headers = array(); private $body; private $status; + private $writeHandle; public function __construct() { $this->body = new PhutilRope(); @@ -30,11 +31,32 @@ } public function appendBody($bytes) { - $this->body->append($bytes); + if ($this->writeHandle !== null) { + $result = @fwrite($this->writeHandle, $bytes); + if ($result !== strlen($bytes)) { + throw new Exception( + pht('Failed to write response to disk. (Maybe the disk is full?)')); + } + } else { + $this->body->append($bytes); + } } public function getBody() { + if ($this->writeHandle !== null) { + return null; + } + return $this->body->getAsString(); } + public function setWriteHandle($write_handle) { + $this->writeHandle = $write_handle; + return $this; + } + + public function getWriteHandle() { + return $this->writeHandle; + } + } diff --git a/src/parser/http/PhutilHTTPResponseParser.php b/src/parser/http/PhutilHTTPResponseParser.php --- a/src/parser/http/PhutilHTTPResponseParser.php +++ b/src/parser/http/PhutilHTTPResponseParser.php @@ -7,6 +7,7 @@ private $response; private $buffer; private $state = 'headers'; + private $writeHandle; public function setFollowLocationHeaders($follow_location_headers) { $this->followLocationHeaders = $follow_location_headers; @@ -17,6 +18,15 @@ return $this->followLocationHeaders; } + public function setWriteHandle($write_handle) { + $this->writeHandle = $write_handle; + return $this; + } + + public function getWriteHandle() { + return $this->writeHandle; + } + public function readBytes($bytes) { if ($this->state == 'discard') { return $this; @@ -90,7 +100,7 @@ HTTPFutureParseResponseStatus::ERROR_MALFORMED_RESPONSE, $raw_headers); - $this->newHTTPRepsonse() + $this->newHTTPResponse() ->setStatus($malformed); $this->buffer = ''; @@ -166,6 +176,12 @@ private function newHTTPResponse() { $response = new PhutilHTTPResponse(); + + $write_handle = $this->getWriteHandle(); + if ($write_handle) { + $response->setWriteHandle($write_handle); + } + $this->responses[] = $response; $this->response = $response; return $response;