diff --git a/src/upload/ArcanistFileDataRef.php b/src/upload/ArcanistFileDataRef.php index 053c6b57..795fbd67 100644 --- a/src/upload/ArcanistFileDataRef.php +++ b/src/upload/ArcanistFileDataRef.php @@ -1,289 +1,313 @@ name = $name; return $this; } /** * @task config */ public function getName() { return $this->name; } /** + * Set the data to upload as a single raw blob. + * + * You can specify file data by calling this method with a single blob of + * data, or by calling @{method:setPath} and providing a path to a file on + * disk. + * + * @param bytes Blob of file data. * @task config */ public function setData($data) { $this->data = $data; return $this; } /** * @task config */ public function getData() { return $this->data; } /** + * Set the data to upload by pointing to a file on disk. + * + * You can specify file data by calling this method with a path, or by + * providing a blob of raw data to @{method:setData}. + * + * The path itself only provides data. If you want to name the file, you + * should also call @{method:setName}. + * + * @param string Path on disk to a file containing data to upload. + * @return this * @task config */ public function setPath($path) { $this->path = $path; return $this; } /** * @task config */ public function getPath() { return $this->path; } /* -( Handling Upload Results )-------------------------------------------- */ /** * @task results */ public function getErrors() { return $this->errors; } /** * @task results */ public function getPHID() { return $this->phid; } /* -( Uploader API )------------------------------------------------------- */ /** * @task uploader */ public function willUpload() { $have_data = ($this->data !== null); $have_path = ($this->path !== null); if (!$have_data && !$have_path) { throw new Exception( pht( 'Specify setData() or setPath() when building a file data '. 'reference.')); } if ($have_data && $have_path) { throw new Exception( pht( 'Specify either setData() or setPath() when building a file data '. 'reference, but not both.')); } if ($have_path) { $path = $this->path; if (!Filesystem::pathExists($path)) { throw new Exception( pht( 'Unable to upload file: path "%s" does not exist.', $path)); } try { Filesystem::assertIsFile($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not a file.', $path)); } try { Filesystem::assertReadable($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not readable.', $path)); } $hash = @sha1_file($path); if ($hash === false) { throw new Exception( pht( 'Unable to upload file: failed to calculate file data hash for '. 'path "%s".', $path)); } $size = @filesize($path); if ($size === false) { throw new Exception( pht( 'Unable to upload file: failed to determine filesize of '. 'path "%s".', $path)); } $this->hash = $hash; $this->size = $size; } else { $data = $this->data; $this->hash = sha1($data); $this->size = strlen($data); } } /** * @task uploader */ public function didFail($error) { $this->errors[] = $error; return $this; } /** * @task uploader */ public function setPHID($phid) { $this->phid = $phid; return $this; } /** * @task uploader */ public function getByteSize() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->size; } /** * @task uploader */ public function getContentHash() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->hash; } /** * @task uploader */ public function didUpload() { if ($this->fileHandle) { @fclose($this->fileHandle); $this->fileHandle = null; } } /** * @task uploader */ public function readBytes($start, $end) { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } $len = ($end - $start); if ($this->data !== null) { return substr($this->data, $start, $len); } $path = $this->path; if ($this->fileHandle === null) { $f = @fopen($path, 'rb'); if (!$f) { throw new Exception( pht( 'Unable to upload file: failed to open path "%s" for reading.', $path)); } $this->fileHandle = $f; } $f = $this->fileHandle; $ok = @fseek($f, $start); if ($ok !== 0) { throw new Exception( pht( 'Unable to upload file: failed to fseek() to offset %d in file '. 'at path "%s".', $start, $path)); } $data = @fread($f, $len); if ($data === false) { throw new Exception( pht( 'Unable to upload file: failed to read %d bytes after offset %d '. 'from file at path "%s".', $len, $start, $path)); } return $data; } } diff --git a/src/upload/ArcanistFileUploader.php b/src/upload/ArcanistFileUploader.php index 60b26442..955cdbdc 100644 --- a/src/upload/ArcanistFileUploader.php +++ b/src/upload/ArcanistFileUploader.php @@ -1,279 +1,303 @@ setConduitClient($conduit); * * // Queue one or more files to be uploaded. * $file = id(new ArcanistFileDataRef()) * ->setName('example.jpg') * ->setPath('/path/to/example.jpg'); * $uploader->addFile($file); * * // Upload the files. * $files = $uploader->uploadFiles(); * * For details about building file references, see @{class:ArcanistFileDataRef}. * * @task config Configuring the Uploader * @task add Adding Files * @task upload Uploading Files * @task internal Internals */ final class ArcanistFileUploader extends Phobject { private $conduit; private $files; /* -( Configuring the Uploader )------------------------------------------- */ /** + * Provide a Conduit client to choose which server to upload files to. + * + * @param ConduitClient Configured client. + * @return this * @task config */ public function setConduitClient(ConduitClient $conduit) { $this->conduit = $conduit; return $this; } /* -( Adding Files )------------------------------------------------------- */ /** + * Add a file to the list of files to be uploaded. + * + * You can optionally provide an explicit key which will be used to identify + * the file. After adding files, upload them with @{method:uploadFiles}. * * @param ArcanistFileDataRef File data to upload. * @param null|string Optional key to use to identify this file. * @return this * @task add */ public function addFile(ArcanistFileDataRef $file, $key = null) { if ($key === null) { $this->files[] = $file; } else { if (isset($this->files[$key])) { throw new Exception( pht( 'Two files were added with identical explicit keys ("%s"); each '. 'explicit key must be unique.', $key)); } $this->files[$key] = $file; } return $this; } /* -( Uploading Files )---------------------------------------------------- */ /** + * Upload files to the server. + * + * This transfers all files which have been queued with @{method:addFiles} + * over the Conduit link configured with @{method:setConduitClient}. + * * This method returns a map of all file data references. If references were * added with an explicit key when @{method:addFile} was called, the key is * retained in the result map. * + * On return, files are either populated with a PHID (indicating a successful + * upload) or a list of errors. See @{class:ArcanistFileDataRef} for + * details. + * * @return map Files with results populated. * @task upload */ public function uploadFiles() { if (!$this->conduit) { throw new PhutilInvalidStateException('setConduitClient'); } $files = $this->files; foreach ($files as $key => $file) { try { $file->willUpload(); } catch (Exception $ex) { $file->didFail($ex->getMessage()); unset($files[$key]); } } $conduit = $this->conduit; $futures = array(); foreach ($files as $key => $file) { $futures[$key] = $conduit->callMethod( 'file.allocate', array( 'name' => $file->getName(), 'contentLength' => $file->getByteSize(), 'contentHash' => $file->getContentHash(), )); } $iterator = id(new FutureIterator($futures))->limit(4); $chunks = array(); foreach ($iterator as $key => $future) { try { $result = $future->resolve(); } catch (Exception $ex) { // The most likely cause for a failure here is that the server does // not support `file.allocate`. In this case, we'll try the older // upload method below. continue; } $phid = $result['filePHID']; $file = $files[$key]; // We don't need to upload any data. Figure out why not: this can either // be because of an error (server can't accept the data) or because the // server already has the data. if (!$result['upload']) { if (!$phid) { $file->didFail( pht( 'Unable to upload file: the server refused to accept file '. '"%s". This usually means it is too large.', $file->getName())); } else { // These server completed the upload by creating a reference to known // file data. We don't need to transfer the actual data, and are all // set. $file->setPHID($phid); } unset($files[$key]); continue; } // The server wants us to do an upload. if ($phid) { $chunks[$key] = array( 'file' => $file, 'phid' => $phid, ); } } foreach ($chunks as $key => $chunk) { $file = $chunk['file']; $phid = $chunk['phid']; try { $this->uploadChunks($file, $phid); $file->setPHID($phid); } catch (Exception $ex) { $file->didFail( pht( 'Unable to upload file chunks: %s', $ex->getMessage())); } unset($files[$key]); } foreach ($files as $key => $file) { try { $phid = $this->uploadData($file); $file->setPHID($phid); } catch (Exception $ex) { $file->didFail( pht( 'Unable to upload file data: %s', $ex->getMessage())); } unset($files[$key]); } foreach ($this->files as $file) { $file->didUpload(); } return $this->files; } /* -( Internals )---------------------------------------------------------- */ /** + * Upload missing chunks of a large file by calling `file.uploadchunk` over + * Conduit. + * * @task internal */ private function uploadChunks(ArcanistFileDataRef $file, $file_phid) { $conduit = $this->conduit; $chunks = $conduit->callMethodSynchronous( 'file.querychunks', array( 'filePHID' => $file_phid, )); $remaining = array(); foreach ($chunks as $chunk) { if (!$chunk['complete']) { $remaining[] = $chunk; } } $done = (count($chunks) - count($remaining)); if ($done) { $this->writeStatus( pht( 'Resuming upload (%d of %d chunks remain).', new PhutilNumber(count($remaining)), new PhutilNumber(count($chunks)))); } else { $this->writeStatus( pht( 'Uploading chunks (%d chunks to upload).', new PhutilNumber(count($remaining)))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($chunks)); for ($ii = 0; $ii < $done; $ii++) { $progress->update(1); } $progress->draw(); // TODO: We could do these in parallel to improve upload performance. foreach ($remaining as $chunk) { $data = $file->readBytes($chunk['byteStart'], $chunk['byteEnd']); $conduit->callMethodSynchronous( 'file.uploadchunk', array( 'filePHID' => $file_phid, 'byteStart' => $chunk['byteStart'], 'dataEncoding' => 'base64', 'data' => base64_encode($data), )); $progress->update(1); } } /** + * Upload an entire file by calling `file.upload` over Conduit. + * * @task internal */ private function uploadData(ArcanistFileDataRef $file) { $conduit = $this->conduit; $data = $file->readBytes(0, $file->getByteSize()); return $conduit->callMethodSynchronous( 'file.upload', array( 'name' => $file->getName(), 'data_base64' => base64_encode($data), )); } /** + * Write a status message. + * * @task internal */ private function writeStatus($message) { - echo $message."\n"; + fwrite(STDERR, $message."\n"); } }