diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php --- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php +++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php @@ -214,6 +214,9 @@ 'storage.engine-selector' => pht( 'Phabricator now automatically discovers available storage engines '. 'at runtime.'), + 'storage.upload-size-limit' => pht( + 'Phabricator now supports arbitrarily large files. Consult the '. + 'documentation for configuration details.'), ); return $ancient_config; diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php --- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php +++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php @@ -21,10 +21,13 @@ protected function executeChecks() { $max_allowed_packet = self::loadRawConfigValue('max_allowed_packet'); - $recommended_minimum = 1024 * 1024; + + // This primarily supports setting the filesize limit for MySQL to 8MB, + // which may produce a >16MB packet after escaping. + $recommended_minimum = (32 * 1024 * 1024); if ($max_allowed_packet < $recommended_minimum) { $message = pht( - "MySQL is configured with a very small 'max_allowed_packet' (%d), ". + "MySQL is configured with a small 'max_allowed_packet' (%d), ". "which may cause some large writes to fail. Strongly consider raising ". "this to at least %d in your MySQL configuration.", $max_allowed_packet, diff --git a/src/applications/config/check/PhabricatorStorageSetupCheck.php b/src/applications/config/check/PhabricatorStorageSetupCheck.php --- a/src/applications/config/check/PhabricatorStorageSetupCheck.php +++ b/src/applications/config/check/PhabricatorStorageSetupCheck.php @@ -10,73 +10,116 @@ * @phutil-external-symbol class PhabricatorStartup */ protected function executeChecks() { - $upload_limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit'); - if (!$upload_limit) { + $chunk_engine_active = false; + + $engines = PhabricatorFileStorageEngine::loadWritableEngines(); + foreach ($engines as $engine) { + if ($engine->isChunkEngine()) { + $chunk_engine_active = true; + break; + } + } + + if (!$chunk_engine_active) { + $doc_href = PhabricatorEnv::getDocLink('Configuring File Storage'); + $message = pht( - 'The Phabricator file upload limit is not configured. You may only '. - 'be able to upload very small files until you configure it, because '. - 'some PHP default limits are very low (as low as 2MB).'); + 'Large file storage has not been configured, which will limit '. + 'the maximum size of file uploads. See %s for '. + 'instructions on configuring uploads and storage.', + phutil_tag( + 'a', + array( + 'href' => $doc_href, + 'target' => '_blank', + ), + pht('Configuring File Storage'))); $this - ->newIssue('config.storage.upload-size-limit') - ->setShortName(pht('Upload Limit')) - ->setName(pht('Upload Limit Not Yet Configured')) - ->setMessage($message) - ->addPhabricatorConfig('storage.upload-size-limit'); - } else { - $memory_limit = PhabricatorStartup::getOldMemoryLimit(); - if ($memory_limit && ((int)$memory_limit > 0)) { - $memory_limit_bytes = phutil_parse_bytes($memory_limit); - $memory_usage_bytes = memory_get_usage(); - $upload_limit_bytes = phutil_parse_bytes($upload_limit); - - $available_bytes = ($memory_limit_bytes - $memory_usage_bytes); - - if ($upload_limit_bytes > $available_bytes) { - $summary = pht( - 'Your PHP memory limit is configured in a way that may prevent '. - 'you from uploading large files.'); - - $message = pht( - 'When you upload a file via drag-and-drop or the API, the entire '. - 'file is buffered into memory before being written to permanent '. - 'storage. Phabricator needs memory available to store these '. - 'files while they are uploaded, but PHP is currently configured '. - 'to limit the available memory.'. - "\n\n". - 'Your Phabricator %s is currently set to a larger value (%s) than '. - 'the amount of available memory (%s) that a PHP process has '. - 'available to use, so uploads via drag-and-drop and the API will '. - 'hit the memory limit before they hit other limits.'. - "\n\n". - '(Note that the application itself must also fit in available '. - 'memory, so not all of the memory under the memory limit is '. - 'available for buffering file uploads.)'. - "\n\n". - "The easiest way to resolve this issue is to set %s to %s in your ". - "PHP configuration, to disable the memory limit. There is ". - "usually little or no value to using this option to limit ". - "Phabricator process memory.". - "\n\n". - "You can also increase the limit, or decrease %s, or ignore this ". - "issue and accept that these upload mechanisms will be limited ". - "in the size of files they can handle.", - phutil_tag('tt', array(), 'storage.upload-size-limit'), - phutil_format_bytes($upload_limit_bytes), - phutil_format_bytes($available_bytes), - phutil_tag('tt', array(), 'memory_limit'), - phutil_tag('tt', array(), '-1'), - phutil_tag('tt', array(), 'storage.upload-size-limit')); - - $this - ->newIssue('php.memory_limit.upload') - ->setName(pht('Memory Limit Restricts File Uploads')) - ->setSummary($summary) - ->setMessage($message) - ->addPHPConfig('memory_limit') - ->addPHPConfigOriginalValue('memory_limit', $memory_limit) - ->addPhabricatorConfig('storage.upload-size-limit'); - } + ->newIssue('large-files') + ->setShortName(pht('Large Files')) + ->setName(pht('Large File Storage Not Configured')) + ->setMessage($message); + } + + $post_max_size = ini_get('post_max_size'); + if ($post_max_size && ((int)$post_max_size > 0)) { + $post_max_bytes = phutil_parse_bytes($post_max_size); + $post_max_need = (32 * 1024 * 1024) * 100; + if ($post_max_need > $post_max_bytes) { + $summary = pht( + 'Set %s in your PHP configuration to at least 32MB '. + 'to support large file uploads.', + phutil_tag('tt', array(), 'post_max_size')); + + $message = pht( + 'Adjust %s in your PHP configuration to at least 32MB. When '. + 'set to smaller value, large file uploads may not work properly.', + phutil_tag('tt', array(), 'post_max_size')); + + $this + ->newIssue('php.post_max_size') + ->setName(pht('PHP post_max_size Not Configured')) + ->setSummary($summary) + ->setMessage($message) + ->setGroup(self::GROUP_PHP) + ->addPHPConfig('post_max_size'); + } + } + + // This is somewhat arbitrary, but make sure we have enough headroom to + // upload a default file at the chunk threshold (8MB), which may be + // base64 encoded, then JSON encoded in the request, and may need to be + // held in memory in the raw and as a query string. + $need_bytes = (64 * 1024 * 1024); + + $memory_limit = PhabricatorStartup::getOldMemoryLimit(); + if ($memory_limit && ((int)$memory_limit > 0)) { + $memory_limit_bytes = phutil_parse_bytes($memory_limit); + $memory_usage_bytes = memory_get_usage(); + + $available_bytes = ($memory_limit_bytes - $memory_usage_bytes); + + if ($need_bytes > $available_bytes) { + $summary = pht( + 'Your PHP memory limit is configured in a way that may prevent '. + 'you from uploading large files or handling large requests.'); + + $message = pht( + 'When you upload a file via drag-and-drop or the API, chunks must '. + 'be buffered into memory before being written to permanent '. + 'storage. Phabricator needs memory available to store these '. + 'chunks while they are uploaded, but PHP is currently configured '. + 'to severly limit the available memory.'. + "\n\n". + 'PHP processes currently have very little free memory available '. + '(%s). To work well, processes should have at least %s.'. + "\n\n". + '(Note that the application itself must also fit in available '. + 'memory, so not all of the memory under the memory limit is '. + 'available for running workloads.)'. + "\n\n". + "The easiest way to resolve this issue is to set %s to %s in your ". + "PHP configuration, to disable the memory limit. There is ". + "usually little or no value to using this option to limit ". + "Phabricator process memory.". + "\n\n". + "You can also increase the limit or ignore this issue and accept ". + "that you may encounter problems uploading large files and ". + "processing large requests.", + phutil_format_bytes($available_bytes), + phutil_format_bytes($need_bytes), + phutil_tag('tt', array(), 'memory_limit'), + phutil_tag('tt', array(), '-1')); + + $this + ->newIssue('php.memory_limit.upload') + ->setName(pht('Memory Limit Restricts File Uploads')) + ->setSummary($summary) + ->setMessage($message) + ->setGroup(self::GROUP_PHP) + ->addPHPConfig('memory_limit') + ->addPHPConfigOriginalValue('memory_limit', $memory_limit); } } diff --git a/src/applications/files/config/PhabricatorFilesConfigOptions.php b/src/applications/files/config/PhabricatorFilesConfigOptions.php --- a/src/applications/files/config/PhabricatorFilesConfigOptions.php +++ b/src/applications/files/config/PhabricatorFilesConfigOptions.php @@ -147,26 +147,6 @@ "Set this to a valid Amazon S3 bucket to store files there. You ". "must also configure S3 access keys in the 'Amazon Web Services' ". "group.")), - $this->newOption('storage.upload-size-limit', 'string', null) - ->setSummary( - pht('Limit to users in interfaces which allow uploading.')) - ->setDescription( - pht( - "Set the size of the largest file a user may upload. This is ". - "used to render text like 'Maximum file size: 10MB' on ". - "interfaces where users can upload files, and files larger than ". - "this size will be rejected. \n\n". - "NOTE: **Setting this to a large size is NOT sufficient to ". - "allow users to upload large files. You must also configure a ". - "number of other settings.** To configure file upload limits, ". - "consult the article 'Configuring File Upload Limits' in the ". - "documentation. Once you've configured some limit across all ". - "levels of the server, you can set this limit to an appropriate ". - "value and the UI will then reflect the actual configured ". - "limit.\n\n". - "Specify this limit in bytes, or using a 'K', 'M', or 'G' ". - "suffix.")) - ->addExample('10M', pht('Allow Uploads 10MB or Smaller')), $this->newOption( 'metamta.files.public-create-email', 'string', diff --git a/src/applications/files/controller/PhabricatorFileUploadController.php b/src/applications/files/controller/PhabricatorFileUploadController.php --- a/src/applications/files/controller/PhabricatorFileUploadController.php +++ b/src/applications/files/controller/PhabricatorFileUploadController.php @@ -57,8 +57,7 @@ id(new AphrontFormFileControl()) ->setLabel(pht('File')) ->setName('file') - ->setError($e_file) - ->setCaption($this->renderUploadLimit())) + ->setError($e_file)) ->appendChild( id(new AphrontFormTextControl()) ->setLabel(pht('Name')) @@ -102,25 +101,4 @@ )); } - private function renderUploadLimit() { - $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit'); - $limit = phutil_parse_bytes($limit); - if ($limit) { - $formatted = phutil_format_bytes($limit); - return 'Maximum file size: '.$formatted; - } - - $doc_href = PhabricatorEnv::getDocLink( - 'Configuring File Upload Limits'); - $doc_link = phutil_tag( - 'a', - array( - 'href' => $doc_href, - 'target' => '_blank', - ), - 'Configuring File Upload Limits'); - - return hsprintf('Upload limit is not configured, see %s.', $doc_link); - } - } diff --git a/src/applications/files/exception/PhabricatorFileUploadException.php b/src/applications/files/exception/PhabricatorFileUploadException.php --- a/src/applications/files/exception/PhabricatorFileUploadException.php +++ b/src/applications/files/exception/PhabricatorFileUploadException.php @@ -20,11 +20,6 @@ 'Unable to write file: failed to write to temporary directory.', UPLOAD_ERR_EXTENSION => 'Unable to upload: a PHP extension stopped the upload.', - - -1000 => - pht("Uploaded file is too large: current limit is %s. To adjust this ". - "limit change 'storage.upload-size-limit' in the Phabricator config.", - PhabricatorEnv::getEnvConfig('storage.upload-size-limit')), ); $message = idx($map, $code, 'Upload failed: unknown error.'); diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -162,8 +162,6 @@ throw new Exception('File size disagrees with uploaded size.'); } - self::validateFileSize(strlen($file_data)); - return $file_data; } @@ -181,22 +179,9 @@ } public static function newFromXHRUpload($data, array $params = array()) { - self::validateFileSize(strlen($data)); return self::newFromFileData($data, $params); } - private static function validateFileSize($size) { - $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit'); - if (!$limit) { - return; - } - - $limit = phutil_parse_bytes($limit); - if ($size > $limit) { - throw new PhabricatorFileUploadException(-1000); - } - } - /** * Given a block of data, try to load an existing file with the same content diff --git a/src/docs/user/configuration/configuring_file_storage.diviner b/src/docs/user/configuration/configuring_file_storage.diviner --- a/src/docs/user/configuration/configuring_file_storage.diviner +++ b/src/docs/user/configuration/configuring_file_storage.diviner @@ -1,45 +1,134 @@ @title Configuring File Storage @group config -Setup how Phabricator will store files. +Setup file storage and support for large files. Overview ======== -Phabricator allows users to upload files, and several applications use file -storage (for instance, Maniphest allows you to attach files to tasks). You can -configure several different storage systems. +This document describes how to configure Phabricator to support large file +uploads, and how to choose where Phabricator stores files. -| System | Setup | Cost | Notes | +There are two major things to configure: + + - set up PHP and your HTTP server to accept large requests; + - choose and configure a storage engine. + +The following sections will guide you through this configuration. + + +How Phabricator Stores Files +============================ + +Phabricator stores files in "storage engines", which are modular backends +that implement access to some storage system (like MySQL, the filesystem, or +a cloud storage service like Amazon S3). + +Phabricator stores large files by breaking them up into many chunks (a few +megabytes in size) and storing the chunks in an underlying storage engine. +This makes it easier to implement new storage engines and gives Phabricator +more flexibility in managing file data. + +The first section of this document discusses configuring your install so that +PHP and your HTTP server will accept requests which are larger than the size of +one file chunk. Without this configuration, file chunk data will be rejected. + +The second section discusses choosing and configuring storage engines, so data +is stored where you want it to be. + + +Configuring Upload Limits +========================= + +File uploads are limited by several pieces of configuration at different layers +of the stack. Generally, the minimum value of all the limits is the effective +one. + +To upload large files, you need to increase all the limits to at least +**32MB**. This will allow you to upload file chunks, which will let Phabricator +store arbitrarily large files. + +The settings which limit file uploads are: + +**HTTP Server**: The HTTP server may set a limit on the maximum request size. +If you exceed this limit, you'll see a default server page with an HTTP error. +These directives limit the total size of the request body, so they must be +somewhat larger than the desired maximum filesize. + + - **Apache**: Apache limits requests with the Apache `LimitRequestBody` + directive. + - **nginx**: nginx limits requests with the nginx `client_max_body_size` + directive. This often defaults to `1M`. + - **lighttpd**: lighttpd limits requests with the lighttpd + `server.max-request-size` directive. + +Set the applicable limit to at least **32MB**. Phabricator can not read these +settings, so it can not raise setup warnings if they are misconfigured. + +**PHP**: PHP has several directives which limit uploads. These directives are +found in `php.ini`. + + - **post_max_size**: Maximum POST request size PHP will accept. If you + exceed this, Phabricator will give you a useful error. This often defaults + to `8M`. Set this to at least `32MB`. Phabricator will give you a setup + warning about this if it is set too low. + - **memory_limit**: For some uploads, file data will be read into memory + before Phabricator can adjust the memory limit. If you exceed this, PHP + may give you a useful error, depending on your configuration. It is + recommended that you set this to `-1` to disable it. Phabricator will + give you a setup warning about this if it is set too low. + +You may also want to configure these PHP options: + + - **max_input_vars**: When files are uploaded via HTML5 drag and drop file + upload APIs, PHP parses the file body as though it contained normal POST + parameters, and may trigger `max_input_vars` if a file has a lot of + brackets in it. You may need to set it to some astronomically high value. + - **upload_max_filesize**: Maximum file size PHP will accept in a raw file + upload. This is not normally used when uploading files via drag-and-drop, + but affects some other kinds of file uploads. If you exceed this, + Phabricator will give you a useful error. This often defaults to `2M`. Set + this to at least `32MB`. + +Once you've adjusted all this configuration, your server will be able to +receive chunk uploads. As long as you have somewhere to store them, this will +enable you to store arbitrarily large files. + + +Storage Engines +=============== + +Phabricator supports several different file storage engines: + +| Engine | Setup | Cost | Notes | |========|=======|======|=======| | MySQL | Automatic | Free | May not scale well. | | Local Disk | Easy | Free | Does not scale well. | | Amazon S3 | Easy | Cheap | Scales well. | | Custom | Hard | Varies | Implement a custom storage engine. | +You can review available storage engines and their configuration by navigating +to {nav Applications > Files > Help/Options > Storage Engines} in the web UI. + By default, Phabricator is configured to store files up to 1MB in MySQL, and reject files larger than 1MB. To store larger files, you can either: - - configure local disk storage; or - - configure Amazon S3 storage; or - - raise the limits on MySQL. + - increase the MySQL limit to at least 8MB; or + - configure another storage engine. -See the rest of this document for some additional discussion of engines. +Doing either of these will enable the chunk storage engine and support for +arbitrarily large files. -You don't have to fully configure this immediately, the defaults are okay until -you need to upload larger files and it's relatively easy to port files between -storage engines later. - -Storage Engines -=============== +The remaining sections of this document discuss the available storage engines +and how to configure them. -Builtin storage engines and information on how to configure them. -== MySQL == +Engine: MySQL +============= - - **Pros**: Fast, no setup required. - - **Cons**: Storing files in a database is a classic bad idea. Does not scale - well. Maximum file size is limited. + - **Pros**: Low latency, no setup required. + - **Cons**: Storing files in a database is a classic bad idea. May become + difficult to administrate if you have a large amount of data. MySQL storage is configured by default, for files up to (just under) 1MB. You can configure it with these keys: @@ -49,37 +138,43 @@ For most installs, it is reasonable to leave this engine as-is and let small files (like thumbnails and profile images) be stored in MySQL, which is usually -the lowest-latency filestore. +the lowest-latency filestore, even if you configure another storage engine. -To support larger files, configure another engine or increase this limit. +To support large files, increase this limit to at least **8MB**. This will +activate chunk storage in MySQL. -== Local Disk == +Engine: Local Disk +================== - - **Pros**: Very simple. Almost no setup required. + - **Pros**: Simple to setup. - **Cons**: Doesn't scale to multiple web frontends without NFS. -To upload larger files: +To configure file storage on the local disk, set: - `storage.local-disk.path`: Set to some writable directory on local disk. Make that directory. -== Amazon S3 == +Engine: Amazon S3 +================= - **Pros**: Scales well. - - **Cons**: More complicated and expensive than other approaches. + - **Cons**: Slightly more complicated than other engines, not free. -To enable file storage in S3, set these key: +To enable file storage in S3, set these keys: - - ##amazon-s3.access-key## Your AWS access key. - - ##amazon-s3.secret-key## Your AWS secret key. - - ##storage.s3.bucket## S3 bucket name where files should be stored. + - `amazon-s3.access-key`: Your AWS access key. + - `amazon-s3.secret-key`: Your AWS secret key. + - `storage.s3.bucket`: S3 bucket name where files should be stored. -= Testing Storage Engines = +Testing Storage Engines +======================= -You can test that things are correctly configured by going to the Files -application (##/file/##) and uploading files. +You can test that things are correctly configured by dragging and dropping +a file onto the Phabricator home page. If engines have been configured +properly, the file should upload. -= Migrating Files Between Engines = +Migrating Files Between Engines +=============================== If you want to move files between storage engines, you can use the `bin/files` script to perform migrations. For example, suppose you previously used MySQL but @@ -95,10 +190,9 @@ You can use `--dry-run` to show which migrations would be performed without taking any action. Run `bin/files help` for more options and information. -= Next Steps = +Next Steps +========== Continue by: - - configuring file size upload limits with - @{article:Configuring File Upload Limits}; or - returning to the @{article:Configuration Guide}. diff --git a/src/docs/user/configuration/configuring_file_upload_limits.diviner b/src/docs/user/configuration/configuring_file_upload_limits.diviner deleted file mode 100644 --- a/src/docs/user/configuration/configuring_file_upload_limits.diviner +++ /dev/null @@ -1,77 +0,0 @@ -@title Configuring File Upload Limits -@group config - -Explains limits on file upload sizes. - -= Overview = - -File uploads are limited by a large number of pieces of configuration, at -multiple layers of the application. Generally, the minimum value of all the -limits is the effective one. To upload large files, you need to increase all -the limits above the maximum file size you want to support. The settings which -limit uploads are: - - - **HTTP Server**: The HTTP server may set a limit on the maximum request - size. If you exceed this limit, you'll see a default server page with an - HTTP error. These directives limit the total size of the request body, - so they must be somewhat larger than the desired maximum filesize. - - **Apache**: Apache limits requests with the Apache `LimitRequestBody` - directive. - - **nginx**: nginx limits requests with the nginx `client_max_body_size` - directive. This often defaults to `1M`. - - **lighttpd**: lighttpd limits requests with the lighttpd - `server.max-request-size` directive. - - **PHP**: PHP has several directives which limit uploads. These directives - are found in `php.ini`. - - **upload_max_filesize**: Maximum file size PHP will accept in a file - upload. If you exceed this, Phabricator will give you a useful error. This - often defaults to `2M`. - - **post_max_size**: Maximum POST request size PHP will accept. If you - exceed this, Phabricator will give you a useful error. This often defaults - to `8M`. - - **memory_limit**: For some uploads, file data will be read into memory - before Phabricator can adjust the memory limit. If you exceed this, PHP - may give you a useful error, depending on your configuration. - - **max_input_vars**: When files are uploaded via HTML5 drag and drop file - upload APIs, PHP parses the file body as though it contained normal POST - parameters, and may trigger `max_input_vars` if a file has a lot of - brackets in it. You may need to set it to some astronomically high value. - - **Storage Engines**: Some storage engines can be configured not to accept - files over a certain size. To upload a file, you must have at least one - configured storage engine which can accept it. Phabricator should give you - useful errors if any of these fail. - - **MySQL Engine**: Upload size is limited by the Phabricator setting - `storage.mysql-engine.max-size`. - - **Amazon S3**: Upload size is limited by Phabricator's implementation to - `5G`. - - **Local Disk**: Upload size is limited only by free disk space. - - **Resource Constraints**: File uploads are limited by resource constraints - on the application server. In particular, some uploaded files are written - to disk in their entirety before being moved to storage engines, and all - uploaded files are read into memory before being moved. These hard limits - should be large for most servers, but will fundamentally prevent Phabricator - from processing truly enormous files (GB/TB scale). Phabricator is probably - not the best application for this in any case. - - **Phabricator Master Limit**: The master limit, `storage.upload-size-limit`, - is used to show upload limits in the UI. - -Phabricator can't read some of these settings, so it can't figure out what the -current limit is or be much help at all in configuring it. Thus, you need to -manually configure all of these limits and then tell Phabricator what you set -them to. Follow these steps: - - - Pick some limit you want to set, like `100M`. - - Configure all of the settings mentioned above to be a bit bigger than the - limit you want to enforce (**note that there are some security implications - to raising these limits**; principally, your server may become easier to - attack with a denial-of-service). - - Set `storage.upload-size-limit` to the limit you want. - - The UI should now show your limit. - - Upload a big file to make sure it works. - -= Next Steps = - -Continue by: - - - configuring file storage with @{article:Configuring File Storage}; or - - returning to the @{article:Configuration Guide}.