Page MenuHomePhabricator
Files F15426709

D20745.id49465.diff
No OneTemporary
Actions

D20745.id49465.diff
View Options

diff --git a/src/future/http/HTTPSFuture.php b/src/future/http/HTTPSFuture.php
--- a/src/future/http/HTTPSFuture.php
+++ b/src/future/http/HTTPSFuture.php
@@ -22,6 +22,10 @@
private $rawBodyPos = 0;
private $fileHandle;
+ private $downloadPath;
+ private $downloadHandle;
+ private $parser;
+
/**
* Create a temp file containing an SSL cert, and use it for this session.
*
@@ -137,6 +141,19 @@
}
}
+ public function setDownloadPath($download_path) {
+ $this->downloadPath = $download_path;
+
+ if (Filesystem::pathExists($download_path)) {
+ throw new Exception(
+ pht(
+ 'Specified download path "%s" already exists, refusing to '.
+ 'overwrite.'));
+ }
+
+ return $this;
+ }
+
/**
* Attach a file to the request.
*
@@ -174,6 +191,12 @@
$uri = $this->getURI();
$domain = id(new PhutilURI($uri))->getDomain();
+ $is_download = $this->isDownload();
+
+ // See T13396. For now, use the streaming response parser only if we're
+ // downloading the response to disk.
+ $use_streaming_parser = (bool)$is_download;
+
if (!$this->handle) {
$uri_object = new PhutilURI($uri);
$proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object);
@@ -364,6 +387,27 @@
if ($proxy) {
curl_setopt($curl, CURLOPT_PROXY, (string)$proxy);
}
+
+ if ($is_download) {
+ $this->downloadHandle = @fopen($this->downloadPath, 'wb+');
+ if (!$this->downloadHandle) {
+ throw new Exception(
+ pht(
+ 'Failed to open filesystem path "%s" for writing.',
+ $this->downloadPath));
+ }
+ }
+
+ if ($use_streaming_parser) {
+ $streaming_parser = id(new PhutilHTTPResponseParser())
+ ->setFollowLocationHeaders($this->getFollowLocation());
+
+ if ($this->downloadHandle) {
+ $streaming_parser->setWriteHandle($this->downloadHandle);
+ }
+
+ $this->parser = $streaming_parser;
+ }
} else {
$curl = $this->handle;
@@ -416,6 +460,21 @@
$body = null;
$headers = array();
$this->result = array($status, $body, $headers);
+ } else if ($this->parser) {
+ $streaming_parser = $this->parser;
+ try {
+ $responses = $streaming_parser->getResponses();
+ $final_response = last($responses);
+ $result = array(
+ $final_response->getStatus(),
+ $final_response->getBody(),
+ $final_response->getHeaders(),
+ );
+ } catch (HTTPFutureParseResponseStatus $ex) {
+ $result = array($ex, null, array());
+ }
+
+ $this->result = $result;
} else {
// cURL returns headers of all redirects, we strip all but the final one.
$redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT);
@@ -432,6 +491,14 @@
self::$pool[$domain][] = $curl;
}
+ if ($is_download) {
+ if ($this->downloadHandle) {
+ fflush($this->downloadHandle);
+ fclose($this->downloadHandle);
+ $this->downloadHandle = null;
+ }
+ }
+
$profiler = PhutilServiceProfiler::getInstance();
$profiler->endServiceCall($this->profilerCallID, array());
@@ -444,7 +511,12 @@
* the data to a buffer.
*/
public function didReceiveDataCallback($handle, $data) {
- $this->responseBuffer .= $data;
+ if ($this->parser) {
+ $this->parser->readBytes($data);
+ } else {
+ $this->responseBuffer .= $data;
+ }
+
return strlen($data);
}
@@ -460,6 +532,20 @@
* @return string Response data, if available.
*/
public function read() {
+ if ($this->isDownload()) {
+ throw new Exception(
+ pht(
+ 'You can not read the result buffer while streaming results '.
+ 'to disk: there is no in-memory buffer to read.'));
+ }
+
+ if ($this->parser) {
+ throw new Exception(
+ pht(
+ 'Streaming reads are not currently supported by the streaming '.
+ 'parser.'));
+ }
+
$result = substr($this->responseBuffer, $this->responseBufferPos);
$this->responseBufferPos = strlen($this->responseBuffer);
return $result;
@@ -473,6 +559,20 @@
* @return this
*/
public function discardBuffers() {
+ if ($this->isDownload()) {
+ throw new Exception(
+ pht(
+ 'You can not discard the result buffer while streaming results '.
+ 'to disk: there is no in-memory buffer to discard.'));
+ }
+
+ if ($this->parser) {
+ throw new Exception(
+ pht(
+ 'Buffer discards are not currently supported by the streaming '.
+ 'parser.'));
+ }
+
$this->responseBuffer = '';
$this->responseBufferPos = 0;
return $this;
@@ -690,5 +790,8 @@
return true;
}
+ private function isDownload() {
+ return ($this->downloadPath !== null);
+ }
}
diff --git a/src/parser/http/PhutilHTTPResponse.php b/src/parser/http/PhutilHTTPResponse.php
--- a/src/parser/http/PhutilHTTPResponse.php
+++ b/src/parser/http/PhutilHTTPResponse.php
@@ -6,6 +6,7 @@
private $headers = array();
private $body;
private $status;
+ private $writeHandle;
public function __construct() {
$this->body = new PhutilRope();
@@ -30,11 +31,32 @@
}
public function appendBody($bytes) {
- $this->body->append($bytes);
+ if ($this->writeHandle !== null) {
+ $result = @fwrite($this->writeHandle, $bytes);
+ if ($result !== strlen($bytes)) {
+ throw new Exception(
+ pht('Failed to write response to disk. (Maybe the disk is full?)'));
+ }
+ } else {
+ $this->body->append($bytes);
+ }
}
public function getBody() {
+ if ($this->writeHandle !== null) {
+ return null;
+ }
+
return $this->body->getAsString();
}
+ public function setWriteHandle($write_handle) {
+ $this->writeHandle = $write_handle;
+ return $this;
+ }
+
+ public function getWriteHandle() {
+ return $this->writeHandle;
+ }
+
}
diff --git a/src/parser/http/PhutilHTTPResponseParser.php b/src/parser/http/PhutilHTTPResponseParser.php
--- a/src/parser/http/PhutilHTTPResponseParser.php
+++ b/src/parser/http/PhutilHTTPResponseParser.php
@@ -7,6 +7,7 @@
private $response;
private $buffer;
private $state = 'headers';
+ private $writeHandle;
public function setFollowLocationHeaders($follow_location_headers) {
$this->followLocationHeaders = $follow_location_headers;
@@ -17,6 +18,15 @@
return $this->followLocationHeaders;
}
+ public function setWriteHandle($write_handle) {
+ $this->writeHandle = $write_handle;
+ return $this;
+ }
+
+ public function getWriteHandle() {
+ return $this->writeHandle;
+ }
+
public function readBytes($bytes) {
if ($this->state == 'discard') {
return $this;
@@ -90,7 +100,7 @@
HTTPFutureParseResponseStatus::ERROR_MALFORMED_RESPONSE,
$raw_headers);
- $this->newHTTPRepsonse()
+ $this->newHTTPResponse()
->setStatus($malformed);
$this->buffer = '';
@@ -166,6 +176,12 @@
private function newHTTPResponse() {
$response = new PhutilHTTPResponse();
+
+ $write_handle = $this->getWriteHandle();
+ if ($write_handle) {
+ $response->setWriteHandle($write_handle);
+ }
+
$this->responses[] = $response;
$this->response = $response;
return $response;

File Metadata

Mime Type
text/plain
Expires
Mon, Mar 24, 10:32 AM (6 d, 19 h ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7710601
Default Alt Text
D20745.id49465.diff (7 KB)

Event Timeline

Phabricator · Built by Phacility