diff --git a/src/upload/ArcanistFileDataRef.php b/src/upload/ArcanistFileDataRef.php index 318418ec..99ed03d9 100644 --- a/src/upload/ArcanistFileDataRef.php +++ b/src/upload/ArcanistFileDataRef.php @@ -1,357 +1,368 @@ name = $name; return $this; } /** * @task config */ public function getName() { return $this->name; } /** * Set the data to upload as a single raw blob. * * You can specify file data by calling this method with a single blob of * data, or by calling @{method:setPath} and providing a path to a file on * disk. * * @param bytes Blob of file data. * @task config */ public function setData($data) { $this->data = $data; return $this; } /** * @task config */ public function getData() { return $this->data; } /** * Set the data to upload by pointing to a file on disk. * * You can specify file data by calling this method with a path, or by * providing a blob of raw data to @{method:setData}. * * The path itself only provides data. If you want to name the file, you * should also call @{method:setName}. * * @param string Path on disk to a file containing data to upload. * @return this * @task config */ public function setPath($path) { $this->path = $path; return $this; } /** * @task config */ public function getPath() { return $this->path; } /** * @task config */ public function setViewPolicy($view_policy) { $this->viewPolicy = $view_policy; return $this; } /** * @task config */ public function getViewPolicy() { return $this->viewPolicy; } /** * Configure a file to be temporary instead of permanent. * * By default, files are retained indefinitely until explicitly deleted. If * you want to upload a temporary file instead, you can specify an epoch * timestamp. The file will be deleted after this time. * * @param int Epoch timestamp to retain the file until. * @return this * @task config */ public function setDeleteAfterEpoch($epoch) { $this->deleteAfterEpoch = $epoch; return $this; } /** * @task config */ public function getDeleteAfterEpoch() { return $this->deleteAfterEpoch; } /* -( Handling Upload Results )-------------------------------------------- */ /** * @task results */ public function getErrors() { return $this->errors; } /** * @task results */ public function getPHID() { return $this->phid; } /* -( Uploader API )------------------------------------------------------- */ /** * @task uploader */ public function willUpload() { $have_data = ($this->data !== null); $have_path = ($this->path !== null); if (!$have_data && !$have_path) { throw new Exception( pht( 'Specify setData() or setPath() when building a file data '. 'reference.')); } if ($have_data && $have_path) { throw new Exception( pht( 'Specify either setData() or setPath() when building a file data '. 'reference, but not both.')); } if ($have_path) { $path = $this->path; if (!Filesystem::pathExists($path)) { throw new Exception( pht( 'Unable to upload file: path "%s" does not exist.', $path)); } try { Filesystem::assertIsFile($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not a file.', $path)); } try { Filesystem::assertReadable($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not readable.', $path)); } - $hash = @sha1_file($path); - if ($hash === false) { - throw new Exception( - pht( - 'Unable to upload file: failed to calculate file data hash for '. - 'path "%s".', - $path)); - } - $size = @filesize($path); if ($size === false) { throw new Exception( pht( 'Unable to upload file: failed to determine filesize of '. 'path "%s".', $path)); } - $this->hash = $hash; + $this->hash = $this->newFileHash($path); $this->size = $size; } else { $data = $this->data; - $this->hash = sha1($data); + $this->hash = $this->newDataHash($data); $this->size = strlen($data); } } /** * @task uploader */ public function didFail($error) { $this->errors[] = $error; return $this; } /** * @task uploader */ public function setPHID($phid) { $this->phid = $phid; return $this; } /** * @task uploader */ public function getByteSize() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->size; } /** * @task uploader */ public function getContentHash() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->hash; } /** * @task uploader */ public function didUpload() { if ($this->fileHandle) { @fclose($this->fileHandle); $this->fileHandle = null; } } /** * @task uploader */ public function readBytes($start, $end) { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } $len = ($end - $start); if ($this->data !== null) { return substr($this->data, $start, $len); } $path = $this->path; if ($this->fileHandle === null) { $f = @fopen($path, 'rb'); if (!$f) { throw new Exception( pht( 'Unable to upload file: failed to open path "%s" for reading.', $path)); } $this->fileHandle = $f; } $f = $this->fileHandle; $ok = @fseek($f, $start); if ($ok !== 0) { throw new Exception( pht( 'Unable to upload file: failed to fseek() to offset %d in file '. 'at path "%s".', $start, $path)); } $data = @fread($f, $len); if ($data === false) { throw new Exception( pht( 'Unable to upload file: failed to read %d bytes after offset %d '. 'from file at path "%s".', $len, $start, $path)); } return $data; } + private function newFileHash($path) { + $hash = hash_file('sha256', $path, $raw_output = false); + + if ($hash === false) { + return null; + } + + return $hash; + } + + private function newDataHash($data) { + $hash = hash('sha256', $data, $raw_output = false); + + if ($hash === false) { + return null; + } + + return $hash; + } + } diff --git a/src/workflow/ArcanistDownloadWorkflow.php b/src/workflow/ArcanistDownloadWorkflow.php index f3eb466b..66fa7331 100644 --- a/src/workflow/ArcanistDownloadWorkflow.php +++ b/src/workflow/ArcanistDownloadWorkflow.php @@ -1,115 +1,272 @@ array( 'conflicts' => array( 'as' => pht( 'Use %s to direct the file to stdout, or %s to direct '. 'it to a named location.', '--show', '--as'), ), 'help' => pht('Write file to stdout instead of to disk.'), ), 'as' => array( 'param' => 'name', 'help' => pht( 'Save the file with a specific name rather than the default.'), ), '*' => 'argv', ); } protected function didParseArguments() { $argv = $this->getArgument('argv'); if (!$argv) { throw new ArcanistUsageException(pht('Specify a file to download.')); } if (count($argv) > 1) { throw new ArcanistUsageException( pht('Specify exactly one file to download.')); } $file = reset($argv); if (!preg_match('/^F?\d+$/', $file)) { throw new ArcanistUsageException( pht('Specify file by ID, e.g. %s.', 'F123')); } $this->id = (int)ltrim($file, 'F'); $this->saveAs = $this->getArgument('as'); $this->show = $this->getArgument('show'); } public function requiresAuthentication() { return true; } public function run() { $conduit = $this->getConduit(); + $id = $this->id; + $display_name = 'F'.$id; + $is_show = $this->show; + $save_as = $this->saveAs; + + try { + $file = $conduit->callMethodSynchronous( + 'file.search', + array( + 'constraints' => array( + 'ids' => array($id), + ), + )); + + $data = $file['data']; + if (!$data) { + throw new ArcanistUsageException( + pht( + 'File "%s" is not a valid file, or not visible.', + $display_name)); + } + + $file = head($data); + $data_uri = idxv($file, array('fields', 'dataURI')); + + if ($data_uri === null) { + throw new ArcanistUsageException( + pht( + 'File "%s" can not be downloaded.', + $display_name)); + } + + if ($is_show) { + // Skip all the file path stuff if we're just going to echo the + // file contents. + } else { + if ($save_as !== null) { + $path = Filesystem::resolvePath($save_as); + + $try_unique = false; + } else { + $path = idxv($file, array('fields', 'name'), $display_name); + $path = basename($path); + $path = Filesystem::resolvePath($path); + + $try_unique = true; + } + + if ($try_unique) { + $path = Filesystem::writeUniqueFile($path, ''); + } else { + if (Filesystem::pathExists($path)) { + throw new ArcanistUsageException( + pht( + 'File "%s" already exists.', + $save_as)); + } + + Filesystem::writeFile($path, ''); + } + + $display_path = Filesystem::readablePath($path); + } + + $size = idxv($file, array('fields', 'size'), 0); + + if ($is_show) { + $file_handle = null; + } else { + $file_handle = fopen($path, 'ab+'); + if ($file_handle === false) { + throw new Exception( + pht( + 'Failed to open file "%s" for writing.', + $path)); + } + + $this->writeInfo( + pht('DATA'), + pht( + 'Downloading "%s" (%s byte(s))...', + $display_name, + new PhutilNumber($size))); + } + + $future = new HTTPSFuture($data_uri); + + // For small files, don't bother drawing a progress bar. + $minimum_bar_bytes = (1024 * 1024 * 4); + + if ($is_show || ($size < $minimum_bar_bytes)) { + $bar = null; + } else { + $bar = id(new PhutilConsoleProgressBar()) + ->setTotal($size); + } + + // TODO: We should stream responses to disk, but cURL gives us the raw + // HTTP response data and BaseHTTPFuture can not currently parse it in + // a stream-oriented way. Until this is resolved, buffer the file data + // in memory and write it to disk in one shot. + + list($status, $data) = $future->resolve(); + if ($status->getStatusCode() !== 200) { + throw new Exception( + pht( + 'Got HTTP %d status response, expected HTTP 200.', + $status)); + } + + if (strlen($data)) { + if ($is_show) { + echo $data; + } else { + $ok = fwrite($file_handle, $data); + if ($ok === false) { + throw new Exception( + pht( + 'Failed to write file data to "%s".', + $path)); + } + } + } + + if ($bar) { + $bar->update(strlen($data)); + } + + if ($bar) { + $bar->done(); + } + + if ($file_handle) { + $ok = fclose($file_handle); + if ($ok === false) { + throw new Exception( + pht( + 'Failed to close file handle for "%s".', + $path)); + } + } + + if (!$is_show) { + $this->writeOkay( + pht('DONE'), + pht( + 'Saved "%s" as "%s".', + $display_name, + $display_path)); + } + + return 0; + } catch (Exception $ex) { + // If we fail for any reason, fall back to the older mechanism using + // "file.info" and "file.download". + } + $this->writeStatusMessage(pht('Getting file information...')."\n"); $info = $conduit->callMethodSynchronous( 'file.info', array( 'id' => $this->id, )); $desc = pht('(%s bytes)', new PhutilNumber($info['byteSize'])); if ($info['name']) { $desc = "'".$info['name']."' ".$desc; } $this->writeStatusMessage(pht('Downloading file %s...', $desc)."\n"); $data = $conduit->callMethodSynchronous( 'file.download', array( 'phid' => $info['phid'], )); $data = base64_decode($data); if ($this->show) { echo $data; } else { $path = Filesystem::writeUniqueFile( nonempty($this->saveAs, $info['name'], 'file'), $data); $this->writeStatusMessage(pht("Saved file as '%s'.", $path)."\n"); } return 0; } }