diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2223,6 +2223,7 @@ 'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php', 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', + 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', @@ -2312,6 +2313,7 @@ 'PhabricatorFileUploadController' => 'applications/files/controller/PhabricatorFileUploadController.php', 'PhabricatorFileUploadDialogController' => 'applications/files/controller/PhabricatorFileUploadDialogController.php', 'PhabricatorFileUploadException' => 'applications/files/exception/PhabricatorFileUploadException.php', + 'PhabricatorFileUploadSource' => 'applications/files/uploadsource/PhabricatorFileUploadSource.php', 'PhabricatorFileinfoSetupCheck' => 'applications/config/check/PhabricatorFileinfoSetupCheck.php', 'PhabricatorFilesApplication' => 'applications/files/application/PhabricatorFilesApplication.php', 'PhabricatorFilesApplicationStorageEnginePanel' => 'applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php', @@ -6504,6 +6506,7 @@ 'PhabricatorEventListener' => 'PhutilEventListener', 'PhabricatorEventType' => 'PhutilEventType', 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', + 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorExternalAccount' => array( @@ -6621,6 +6624,7 @@ 'PhabricatorFileUploadController' => 'PhabricatorFileController', 'PhabricatorFileUploadDialogController' => 'PhabricatorFileController', 'PhabricatorFileUploadException' => 'Exception', + 'PhabricatorFileUploadSource' => 'Phobject', 'PhabricatorFileinfoSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorFilesApplication' => 'PhabricatorApplication', 'PhabricatorFilesApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel', diff --git a/src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php --- a/src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php +++ b/src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php @@ -39,14 +39,20 @@ $file_query->setByteLimit($byte_limit); } - $content = $file_query->execute(); + $file = $file_query->execute(); $too_slow = (bool)$file_query->getExceededTimeLimit(); $too_huge = (bool)$file_query->getExceededByteLimit(); $file_phid = null; if (!$too_slow && !$too_huge) { - $file = $this->newFile($drequest, $content); + $repository = $drequest->getRepository(); + $repository_phid = $repository->getPHID(); + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + $file->attachToObject($repository_phid); + unset($unguarded); + $file_phid = $file->getPHID(); } @@ -57,26 +63,4 @@ ); } - private function newFile(DiffusionRequest $drequest, $content) { - $path = $drequest->getPath(); - $name = basename($path); - - $repository = $drequest->getRepository(); - $repository_phid = $repository->getPHID(); - - $file = PhabricatorFile::buildFromFileDataOrHash( - $content, - array( - 'name' => $name, - 'ttl' => time() + phutil_units('48 hours in seconds'), - 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, - )); - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - $file->attachToObject($repository_phid); - unset($unguarded); - - return $file; - } - } diff --git a/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php b/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php --- a/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php +++ b/src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php @@ -54,23 +54,46 @@ $future->setStdoutSizeLimit($byte_limit + 1); } + $drequest = $this->getRequest(); + + $name = basename($drequest->getPath()); + $ttl = PhabricatorTime::getNow() + phutil_units('48 hours in seconds'); + try { - $file_content = $this->resolveFileContentFuture($future); + $threshold = PhabricatorFileStorageEngine::getChunkThreshold(); + $future->setReadBufferSize($threshold); + + $source = id(new PhabricatorExecFutureFileUploadSource()) + ->setName($name) + ->setTTL($ttl) + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setExecFuture($future); + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + $file = $source->uploadFile(); + unset($unguarded); + } catch (CommandException $ex) { if (!$future->getWasKilledByTimeout()) { throw $ex; } $this->didHitTimeLimit = true; - $file_content = null; + $file = null; } - if ($byte_limit && (strlen($file_content) > $byte_limit)) { + if ($byte_limit && ($file->getByteSize() > $byte_limit)) { $this->didHitByteLimit = true; - $file_content = null; + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + id(new PhabricatorDestructionEngine()) + ->destroyObject($file); + unset($unguarded); + + $file = null; } - return $file_content; + return $file; } } diff --git a/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php b/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php --- a/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php +++ b/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php @@ -26,11 +26,15 @@ $rows = array(); $rowc = array(); foreach ($engines as $key => $engine) { - $limited = $no; + if ($engine->isTestEngine()) { + continue; + } + $limit = null; if ($engine->hasFilesizeLimit()) { - $limited = $yes; $limit = phutil_format_bytes($engine->getFilesizeLimit()); + } else { + $limit = pht('Unlimited'); } if ($engine->canWriteFiles()) { @@ -39,12 +43,6 @@ $writable = $no; } - if ($engine->isTestEngine()) { - $test = $yes; - } else { - $test = $no; - } - if (isset($writable_engines[$key]) || isset($chunk_engines[$key])) { $rowc[] = 'highlighted'; } else { @@ -54,9 +52,7 @@ $rows[] = array( $key, get_class($engine), - $test, $writable, - $limited, $limit, ); } @@ -67,9 +63,7 @@ array( pht('Key'), pht('Class'), - pht('Unit Test'), pht('Writable'), - pht('Has Limit'), pht('Limit'), )) ->setRowClasses($rowc) @@ -78,8 +72,6 @@ '', 'wide', '', - '', - '', 'n', )); diff --git a/src/applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php b/src/applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php new file mode 100644 --- /dev/null +++ b/src/applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php @@ -0,0 +1,28 @@ +future = $future; + return $this; + } + + public function getExecFuture() { + return $this->future; + } + + protected function newDataIterator() { + $future = $this->getExecFuture(); + + return id(new LinesOfALargeExecFuture($future)) + ->setDelimiter(null); + } + + protected function getDataLength() { + return null; + } + +} diff --git a/src/applications/files/uploadsource/PhabricatorFileUploadSource.php b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php new file mode 100644 --- /dev/null +++ b/src/applications/files/uploadsource/PhabricatorFileUploadSource.php @@ -0,0 +1,235 @@ +name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setTTL($ttl) { + $this->ttl = $ttl; + return $this; + } + + public function getTTL() { + return $this->ttl; + } + + public function setViewPolicy($view_policy) { + $this->viewPolicy = $view_policy; + return $this; + } + + public function getViewPolicy() { + return $this->viewPolicy; + } + + public function uploadFile() { + if (!$this->shouldChunkFile()) { + return $this->writeSingleFile(); + } else { + return $this->writeChunkedFile(); + } + } + + private function getDataIterator() { + if (!$this->data) { + $this->data = $this->newDataIterator(); + } + return $this->data; + } + + private function getRope() { + if (!$this->rope) { + $this->rope = new PhutilRope(); + } + return $this->rope; + } + + abstract protected function newDataIterator(); + abstract protected function getDataLength(); + + private function readFileData() { + $data = $this->getDataIterator(); + + if (!$this->didRewind) { + $data->rewind(); + $this->didRewind = true; + } else { + $data->next(); + } + + if (!$data->valid()) { + return false; + } + + $rope = $this->getRope(); + $rope->append($data->current()); + + return true; + } + + private function shouldChunkFile() { + if ($this->shouldChunk !== null) { + return $this->shouldChunk; + } + + $threshold = PhabricatorFileStorageEngine::getChunkThreshold(); + + // If we don't know how large the file is, we're going to read some data + // from it until we know whether it's a small file or not. This will give + // us enough information to make a decision about chunking. + $length = $this->getDataLength(); + if ($length === null) { + $rope = $this->getRope(); + while ($this->readFileData()) { + $length = $rope->getByteLength(); + if ($length > $threshold) { + break; + } + } + } + + $this->shouldChunk = ($length > $threshold); + + return $this->shouldChunk; + } + + private function writeSingleFile() { + while ($this->readFileData()) { + // Read the entire file. + } + + $rope = $this->getRope(); + $data = $rope->getAsString(); + + $parameters = $this->getNewFileParameters(); + + return PhabricatorFile::newFromFileData($data, $parameters); + } + + private function writeChunkedFile() { + $engine = $this->getChunkEngine(); + + $parameters = $this->getNewFileParameters(); + + $parameters = array( + 'isPartial' => true, + ) + $parameters; + + $data_length = $this->getDataLength(); + if ($data_length !== null) { + $length = $data_length; + } else { + $length = 0; + } + + $file = PhabricatorFile::newChunkedFile($engine, $length, $parameters); + $file->save(); + + $rope = $this->getRope(); + + // Read the source, writing chunks as we get enough data. + while ($this->readFileData()) { + while (true) { + $rope_length = $rope->getByteLength(); + if ($rope_length < $engine->getChunkSize()) { + break; + } + $this->writeChunk($file, $engine); + } + } + + // If we have extra bytes at the end, write them. + if ($rope->getByteLength()) { + $this->writeChunk($file, $engine); + } + + $file->setIsPartial(0); + if ($data_length === null) { + $file->setByteSize($this->getTotalBytesWritten()); + } + $file->save(); + + return $file; + } + + private function writeChunk( + PhabricatorFile $file, + PhabricatorFileStorageEngine $engine) { + + $offset = $this->getTotalBytesWritten(); + $max_length = $engine->getChunkSize(); + $rope = $this->getRope(); + + $data = $rope->getPrefixBytes($max_length); + $actual_length = strlen($data); + $rope->removeBytesFromHead($actual_length); + + $chunk_data = PhabricatorFile::newFromFileData( + $data, + array( + 'name' => $file->getMonogram().'.chunk-'.$offset, + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + )); + + $chunk = PhabricatorFileChunk::initializeNewChunk( + $file->getStorageHandle(), + $offset, + $offset + $actual_length); + + $chunk + ->setDataFilePHID($chunk_data->getPHID()) + ->save(); + + $this->setTotalBytesWritten($offset + $actual_length); + + return $chunk; + } + + private function getNewFileParameters() { + return array( + 'name' => $this->getName(), + 'ttl' => $this->getTTL(), + 'viewPolicy' => $this->getViewPolicy(), + ); + } + + private function getChunkEngine() { + $chunk_engines = PhabricatorFileStorageEngine::loadWritableChunkEngines(); + if (!$chunk_engines) { + throw new Exception( + pht( + 'Unable to upload file: this server is not configured with any '. + 'storage engine which can store large files.')); + } + + return head($chunk_engines); + } + + private function setTotalBytesWritten($total_bytes_written) { + $this->totalBytesWritten = $total_bytes_written; + return $this; + } + + private function getTotalBytesWritten() { + return $this->totalBytesWritten; + } + +}