diff --git a/src/applications/config/check/PhabricatorStorageSetupCheck.php b/src/applications/config/check/PhabricatorStorageSetupCheck.php index acdcd49306..dbe79c90d6 100644 --- a/src/applications/config/check/PhabricatorStorageSetupCheck.php +++ b/src/applications/config/check/PhabricatorStorageSetupCheck.php @@ -1,150 +1,143 @@ isChunkEngine()) { - $chunk_engine_active = true; - break; - } - } + $engines = PhabricatorFileStorageEngine::loadWritableChunkEngines(); + $chunk_engine_active = (bool)$engines; if (!$chunk_engine_active) { $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage'); $message = pht( 'Large file storage has not been configured, which will limit '. 'the maximum size of file uploads. See %s for '. 'instructions on configuring uploads and storage.', phutil_tag( 'a', array( 'href' => $doc_href, 'target' => '_blank', ), pht('Configuring File Storage'))); $this ->newIssue('large-files') ->setShortName(pht('Large Files')) ->setName(pht('Large File Storage Not Configured')) ->setMessage($message); } $post_max_size = ini_get('post_max_size'); if ($post_max_size && ((int)$post_max_size > 0)) { $post_max_bytes = phutil_parse_bytes($post_max_size); $post_max_need = (32 * 1024 * 1024); if ($post_max_need > $post_max_bytes) { $summary = pht( 'Set %s in your PHP configuration to at least 32MB '. 'to support large file uploads.', phutil_tag('tt', array(), 'post_max_size')); $message = pht( 'Adjust %s in your PHP configuration to at least 32MB. When '. 'set to smaller value, large file uploads may not work properly.', phutil_tag('tt', array(), 'post_max_size')); $this ->newIssue('php.post_max_size') ->setName(pht('PHP post_max_size Not Configured')) ->setSummary($summary) ->setMessage($message) ->setGroup(self::GROUP_PHP) ->addPHPConfig('post_max_size'); } } // This is somewhat arbitrary, but make sure we have enough headroom to // upload a default file at the chunk threshold (8MB), which may be // base64 encoded, then JSON encoded in the request, and may need to be // held in memory in the raw and as a query string. $need_bytes = (64 * 1024 * 1024); $memory_limit = PhabricatorStartup::getOldMemoryLimit(); if ($memory_limit && ((int)$memory_limit > 0)) { $memory_limit_bytes = phutil_parse_bytes($memory_limit); $memory_usage_bytes = memory_get_usage(); $available_bytes = ($memory_limit_bytes - $memory_usage_bytes); if ($need_bytes > $available_bytes) { $summary = pht( 'Your PHP memory limit is configured in a way that may prevent '. 'you from uploading large files or handling large requests.'); $message = pht( 'When you upload a file via drag-and-drop or the API, chunks must '. 'be buffered into memory before being written to permanent '. 'storage. Phabricator needs memory available to store these '. 'chunks while they are uploaded, but PHP is currently configured '. 'to severly limit the available memory.'. "\n\n". 'PHP processes currently have very little free memory available '. '(%s). To work well, processes should have at least %s.'. "\n\n". '(Note that the application itself must also fit in available '. 'memory, so not all of the memory under the memory limit is '. 'available for running workloads.)'. "\n\n". "The easiest way to resolve this issue is to set %s to %s in your ". "PHP configuration, to disable the memory limit. There is ". "usually little or no value to using this option to limit ". "Phabricator process memory.". "\n\n". "You can also increase the limit or ignore this issue and accept ". "that you may encounter problems uploading large files and ". "processing large requests.", phutil_format_bytes($available_bytes), phutil_format_bytes($need_bytes), phutil_tag('tt', array(), 'memory_limit'), phutil_tag('tt', array(), '-1')); $this ->newIssue('php.memory_limit.upload') ->setName(pht('Memory Limit Restricts File Uploads')) ->setSummary($summary) ->setMessage($message) ->setGroup(self::GROUP_PHP) ->addPHPConfig('memory_limit') ->addPHPConfigOriginalValue('memory_limit', $memory_limit); } } $local_path = PhabricatorEnv::getEnvConfig('storage.local-disk.path'); if (!$local_path) { return; } if (!Filesystem::pathExists($local_path) || !is_readable($local_path) || !is_writable($local_path)) { $message = pht( 'Configured location for storing uploaded files on disk ("%s") does '. 'not exist, or is not readable or writable. Verify the directory '. 'exists and is readable and writable by the webserver.', $local_path); $this ->newIssue('config.storage.local-disk.path') ->setShortName(pht('Local Disk Storage')) ->setName(pht('Local Disk Storage Not Readable/Writable')) ->setMessage($message) ->addPhabricatorConfig('storage.local-disk.path'); } } } diff --git a/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php b/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php index 0b2999a66d..a174102c18 100644 --- a/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php +++ b/src/applications/files/applicationpanel/PhabricatorFilesApplicationStorageEnginePanel.php @@ -1,100 +1,101 @@ getViewer(); $application = $this->getApplication(); $engines = PhabricatorFileStorageEngine::loadAllEngines(); $writable_engines = PhabricatorFileStorageEngine::loadWritableEngines(); + $chunk_engines = PhabricatorFileStorageEngine::loadWritableChunkEngines(); $yes = pht('Yes'); $no = pht('No'); $rows = array(); $rowc = array(); foreach ($engines as $key => $engine) { $limited = $no; $limit = null; if ($engine->hasFilesizeLimit()) { $limited = $yes; $limit = phutil_format_bytes($engine->getFilesizeLimit()); } if ($engine->canWriteFiles()) { $writable = $yes; } else { $writable = $no; } if ($engine->isTestEngine()) { $test = $yes; } else { $test = $no; } - if (isset($writable_engines[$key])) { + if (isset($writable_engines[$key]) || isset($chunk_engines[$key])) { $rowc[] = 'highlighted'; } else { $rowc[] = null; } $rows[] = array( $key, get_class($engine), $test, $writable, $limited, $limit, ); } $table = $table = id(new AphrontTableView($rows)) ->setNoDataString(pht('No storage engines available.')) ->setHeaders( array( pht('Key'), pht('Class'), pht('Unit Test'), pht('Writable'), pht('Has Limit'), pht('Limit'), )) ->setRowClasses($rowc) ->setColumnClasses( array( '', 'wide', '', '', '', 'n', )); $box = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Storage Engines')) ->appendChild($table); return $box; } public function handlePanelRequest( AphrontRequest $request, PhabricatorController $controller) { return new Aphront404Response(); } } diff --git a/src/applications/files/conduit/FileAllocateConduitAPIMethod.php b/src/applications/files/conduit/FileAllocateConduitAPIMethod.php index ae6c16ae03..80f1cad7e1 100644 --- a/src/applications/files/conduit/FileAllocateConduitAPIMethod.php +++ b/src/applications/files/conduit/FileAllocateConduitAPIMethod.php @@ -1,141 +1,137 @@ 'string', 'contentLength' => 'int', 'contentHash' => 'optional string', 'viewPolicy' => 'optional string', ); } public function defineReturnType() { return 'map'; } public function defineErrorTypes() { return array(); } protected function execute(ConduitAPIRequest $request) { $viewer = $request->getUser(); $hash = $request->getValue('contentHash'); $name = $request->getValue('name'); $view_policy = $request->getValue('viewPolicy'); $length = $request->getValue('contentLength'); $properties = array( 'name' => $name, 'authorPHID' => $viewer->getPHID(), 'viewPolicy' => $view_policy, 'isExplicitUpload' => true, ); $file = null; if ($hash) { $file = PhabricatorFile::newFileFromContentHash( $hash, $properties); } if ($hash && !$file) { $chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash( $viewer, $hash); $file = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withContentHashes(array($chunked_hash)) ->executeOne(); } if (strlen($name) && !$hash && !$file) { if ($length > PhabricatorFileStorageEngine::getChunkThreshold()) { // If we don't have a hash, but this file is large enough to store in // chunks and thus may be resumable, try to find a partially uploaded // file by the same author with the same name and same length. This // allows us to resume uploads in Javascript where we can't efficiently // compute file hashes. $file = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withAuthorPHIDs(array($viewer->getPHID())) ->withNames(array($name)) ->withLengthBetween($length, $length) ->withIsPartial(true) ->setLimit(1) ->executeOne(); } } if ($file) { return array( 'upload' => (bool)$file->getIsPartial(), 'filePHID' => $file->getPHID(), ); } + // If there are any non-chunk engines which this file can fit into, + // just tell the client to upload the file. $engines = PhabricatorFileStorageEngine::loadStorageEngines($length); if ($engines) { + return array( + 'upload' => true, + 'filePHID' => null, + ); + } - // Pick the first engine. If the file is small enough to fit into a - // single engine without chunking, this will be a non-chunk engine and - // we'll just tell the client to upload the file. - $engine = head($engines); - if ($engine) { - if (!$engine->isChunkEngine()) { - return array( - 'upload' => true, - 'filePHID' => null, - ); - } - - // Otherwise, this is a large file and we need to perform a chunked - // upload. - - $chunk_properties = $properties; - - if ($hash) { - $chunk_properties += array( - 'chunkedHash' => $chunked_hash, - ); - } - - $file = $engine->allocateChunks($length, $chunk_properties); - - return array( - 'upload' => true, - 'filePHID' => $file->getPHID(), + // Otherwise, this is a large file and we want to perform a chunked + // upload if we have a chunk engine available. + $chunk_engines = PhabricatorFileStorageEngine::loadWritableChunkEngines(); + if ($chunk_engines) { + $chunk_properties = $properties; + + if ($hash) { + $chunk_properties += array( + 'chunkedHash' => $chunked_hash, ); } + + $chunk_engine = head($chunk_engines); + $file = $chunk_engine->allocateChunks($length, $chunk_properties); + + return array( + 'upload' => true, + 'filePHID' => $file->getPHID(), + ); } // None of the storage engines can accept this file. if (PhabricatorFileStorageEngine::loadWritableEngines()) { $error = pht( 'Unable to upload file: this file is too large for any '. 'configured storage engine.'); } else { $error = pht( 'Unable to upload file: the server is not configured with any '. 'writable storage engines.'); } return array( 'upload' => false, 'filePHID' => null, 'error' => $error, ); } } diff --git a/src/applications/files/engine/PhabricatorFileStorageEngine.php b/src/applications/files/engine/PhabricatorFileStorageEngine.php index 6d0826c95c..be2fb2bbd3 100644 --- a/src/applications/files/engine/PhabricatorFileStorageEngine.php +++ b/src/applications/files/engine/PhabricatorFileStorageEngine.php @@ -1,319 +1,369 @@ } /* -( Engine Metadata )---------------------------------------------------- */ /** * Return a unique, nonempty string which identifies this storage engine. * This is used to look up the storage engine when files needs to be read or * deleted. For instance, if you store files by giving them to a duck for * safe keeping in his nest down by the pond, you might return 'duck' from * this method. * * @return string Unique string for this engine, max length 32. * @task meta */ abstract public function getEngineIdentifier(); /** * Prioritize this engine relative to other engines. * * Engines with a smaller priority number get an opportunity to write files * first. Generally, lower-latency filestores should have lower priority * numbers, and higher-latency filestores should have higher priority * numbers. Setting priority to approximately the number of milliseconds of * read latency will generally produce reasonable results. * * In conjunction with filesize limits, the goal is to store small files like * profile images, thumbnails, and text snippets in lower-latency engines, * and store large files in higher-capacity engines. * * @return float Engine priority. * @task meta */ abstract public function getEnginePriority(); /** * Return `true` if the engine is currently writable. * * Engines that are disabled or missing configuration should return `false` * to prevent new writes. If writes were made with this engine in the past, * the application may still try to perform reads. * * @return bool True if this engine can support new writes. * @task meta */ abstract public function canWriteFiles(); /** * Return `true` if the engine has a filesize limit on storable files. * * The @{method:getFilesizeLimit} method can retrieve the actual limit. This * method just removes the ambiguity around the meaning of a `0` limit. * * @return bool `true` if the engine has a filesize limit. * @task meta */ public function hasFilesizeLimit() { return true; } /** * Return maximum storable file size, in bytes. * * Not all engines have a limit; use @{method:getFilesizeLimit} to check if * an engine has a limit. Engines without a limit can store files of any * size. * * By default, engines define a limit which supports chunked storage of * large files. In most cases, you should not change this limit, even if an * engine has vast storage capacity: chunked storage makes large files more * manageable and enables features like resumable uploads. * * @return int Maximum storable file size, in bytes. * @task meta */ public function getFilesizeLimit() { // NOTE: This 8MB limit is selected to be larger than the 4MB chunk size, // but not much larger. Files between 0MB and 8MB will be stored normally; // files larger than 8MB will be chunked. return (1024 * 1024 * 8); } /** * Identifies storage engines that support unit tests. * * These engines are not used for production writes. * * @return bool True if this is a test engine. * @task meta */ public function isTestEngine() { return false; } /** * Identifies chunking storage engines. * * If this is a storage engine which splits files into chunks and stores the * chunks in other engines, it can return `true` to signal that other * chunking engines should not try to store data here. * * @return bool True if this is a chunk engine. * @task meta */ public function isChunkEngine() { return false; } /* -( Managing File Data )------------------------------------------------- */ /** * Write file data to the backing storage and return a handle which can later * be used to read or delete it. For example, if the backing storage is local * disk, the handle could be the path to the file. * * The caller will provide a $params array, which may be empty or may have * some metadata keys (like "name" and "author") in it. You should be prepared * to handle writes which specify no metadata, but might want to optionally * use some keys in this array for debugging or logging purposes. This is * the same dictionary passed to @{method:PhabricatorFile::newFromFileData}, * so you could conceivably do custom things with it. * * If you are unable to write for whatever reason (e.g., the disk is full), * throw an exception. If there are other satisfactory but less-preferred * storage engines available, they will be tried. * * @param string The file data to write. * @param array File metadata (name, author), if available. * @return string Unique string which identifies the stored file, max length * 255. * @task file */ abstract public function writeFile($data, array $params); /** * Read the contents of a file previously written by @{method:writeFile}. * * @param string The handle returned from @{method:writeFile} when the * file was written. * @return string File contents. * @task file */ abstract public function readFile($handle); /** * Delete the data for a file previously written by @{method:writeFile}. * * @param string The handle returned from @{method:writeFile} when the * file was written. * @return void * @task file */ abstract public function deleteFile($handle); + +/* -( Loading Storage Engines )-------------------------------------------- */ + + /** * Select viable default storage engines according to configuration. We'll * select the MySQL and Local Disk storage engines if they are configured * to allow a given file. * * @param int File size in bytes. + * @task load */ public static function loadStorageEngines($length) { $engines = self::loadWritableEngines(); $writable = array(); foreach ($engines as $key => $engine) { if ($engine->hasFilesizeLimit()) { $limit = $engine->getFilesizeLimit(); if ($limit < $length) { continue; } } $writable[$key] = $engine; } return $writable; } + + /** + * @task load + */ public static function loadAllEngines() { static $engines; if ($engines === null) { $objects = id(new PhutilSymbolLoader()) ->setAncestorClass(__CLASS__) ->loadObjects(); $map = array(); foreach ($objects as $engine) { $key = $engine->getEngineIdentifier(); if (empty($map[$key])) { $map[$key] = $engine; } else { throw new Exception( pht( 'Storage engines "%s" and "%s" have the same engine '. 'identifier "%s". Each storage engine must have a unique '. 'identifier.', get_class($engine), get_class($map[$key]), $key)); } } $map = msort($map, 'getEnginePriority'); $engines = $map; } return $engines; } - public static function loadWritableEngines() { + + /** + * @task load + */ + private static function loadProductionEngines() { $engines = self::loadAllEngines(); - $writable = array(); + $active = array(); foreach ($engines as $key => $engine) { if ($engine->isTestEngine()) { continue; } + $active[$key] = $engine; + } + + return $active; + } + + + /** + * @task load + */ + public static function loadWritableEngines() { + $engines = self::loadProductionEngines(); + + $writable = array(); + foreach ($engines as $key => $engine) { if (!$engine->canWriteFiles()) { continue; } + if ($engine->isChunkEngine()) { + // Don't select chunk engines as writable. + continue; + } $writable[$key] = $engine; } return $writable; } + /** + * @task load + */ + public static function loadWritableChunkEngines() { + $engines = self::loadProductionEngines(); + + $chunk = array(); + foreach ($engines as $key => $engine) { + if (!$engine->canWriteFiles()) { + continue; + } + if (!$engine->isChunkEngine()) { + continue; + } + $chunk[$key] = $engine; + } + + return $chunk; + } + + /** - * Return the largest file size which can be uploaded without chunking. + * Return the largest file size which can not be uploaded in chunks. * * Files smaller than this will always upload in one request, so clients * can safely skip the allocation step. * * @return int|null Byte size, or `null` if there is no chunk support. */ public static function getChunkThreshold() { - $engines = self::loadWritableEngines(); + $engines = self::loadWritableChunkEngines(); $min = null; foreach ($engines as $engine) { - if (!$engine->isChunkEngine()) { - continue; - } - if (!$min) { $min = $engine; continue; } if ($min->getChunkSize() > $engine->getChunkSize()) { $min = $engine->getChunkSize(); } } if (!$min) { return null; } return $engine->getChunkSize(); } public function getFileDataIterator(PhabricatorFile $file, $begin, $end) { // The default implementation is trivial and just loads the entire file // upfront. $data = $file->loadFileData(); if ($begin !== null && $end !== null) { $data = substr($data, $begin, ($end - $begin)); } else if ($begin !== null) { $data = substr($data, $begin); } else if ($end !== null) { $data = substr($data, 0, $end); } return array($data); } }