diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2490,12 +2490,16 @@ 'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php', 'PhabricatorFileListController' => 'applications/files/controller/PhabricatorFileListController.php', 'PhabricatorFileQuery' => 'applications/files/query/PhabricatorFileQuery.php', + 'PhabricatorFileROT13StorageFormat' => 'applications/files/format/PhabricatorFileROT13StorageFormat.php', + 'PhabricatorFileRawStorageFormat' => 'applications/files/format/PhabricatorFileRawStorageFormat.php', 'PhabricatorFileSchemaSpec' => 'applications/files/storage/PhabricatorFileSchemaSpec.php', 'PhabricatorFileSearchEngine' => 'applications/files/query/PhabricatorFileSearchEngine.php', 'PhabricatorFileStorageBlob' => 'applications/files/storage/PhabricatorFileStorageBlob.php', 'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php', 'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php', 'PhabricatorFileStorageEngineTestCase' => 'applications/files/engine/__tests__/PhabricatorFileStorageEngineTestCase.php', + 'PhabricatorFileStorageFormat' => 'applications/files/format/PhabricatorFileStorageFormat.php', + 'PhabricatorFileStorageFormatTestCase' => 'applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php', 'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php', 'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php', 'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php', @@ -7134,12 +7138,16 @@ 'PhabricatorFileLinkView' => 'AphrontView', 'PhabricatorFileListController' => 'PhabricatorFileController', 'PhabricatorFileQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', + 'PhabricatorFileROT13StorageFormat' => 'PhabricatorFileStorageFormat', + 'PhabricatorFileRawStorageFormat' => 'PhabricatorFileStorageFormat', 'PhabricatorFileSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorFileSearchEngine' => 'PhabricatorApplicationSearchEngine', 'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO', 'PhabricatorFileStorageConfigurationException' => 'Exception', 'PhabricatorFileStorageEngine' => 'Phobject', 'PhabricatorFileStorageEngineTestCase' => 'PhabricatorTestCase', + 'PhabricatorFileStorageFormat' => 'Phobject', + 'PhabricatorFileStorageFormatTestCase' => 'PhabricatorTestCase', 'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorFileTestCase' => 'PhabricatorTestCase', 'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator', diff --git a/src/applications/files/controller/PhabricatorFileInfoController.php b/src/applications/files/controller/PhabricatorFileInfoController.php --- a/src/applications/files/controller/PhabricatorFileInfoController.php +++ b/src/applications/files/controller/PhabricatorFileInfoController.php @@ -256,8 +256,10 @@ $types[] = pht('Profile'); } - $types = implode(', ', $types); - $finfo->addProperty(pht('Attributes'), $types); + if ($types) { + $types = implode(', ', $types); + $finfo->addProperty(pht('Attributes'), $types); + } $storage_properties = new PHUIPropertyListView(); $box->addPropertyList($storage_properties, pht('Storage')); @@ -266,9 +268,14 @@ pht('Engine'), $file->getStorageEngine()); - $storage_properties->addProperty( - pht('Format'), - $file->getStorageFormat()); + $format_key = $file->getStorageFormat(); + $format = PhabricatorFileStorageFormat::getFormat($format_key); + if ($format) { + $format_name = $format->getStorageFormatName(); + } else { + $format_name = pht('Unknown ("%s")', $format_key); + } + $storage_properties->addProperty(pht('Format'), $format_name); $storage_properties->addProperty( pht('Handle'), diff --git a/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php --- a/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php +++ b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php @@ -174,7 +174,7 @@ return (4 * 1024 * 1024); } - public function getFileDataIterator(PhabricatorFile $file, $begin, $end) { + public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) { $chunks = id(new PhabricatorFileChunkQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withChunkHandles(array($file->getStorageHandle())) diff --git a/src/applications/files/engine/PhabricatorFileStorageEngine.php b/src/applications/files/engine/PhabricatorFileStorageEngine.php --- a/src/applications/files/engine/PhabricatorFileStorageEngine.php +++ b/src/applications/files/engine/PhabricatorFileStorageEngine.php @@ -325,10 +325,10 @@ return $engine->getChunkSize(); } - public function getFileDataIterator(PhabricatorFile $file, $begin, $end) { + public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) { // The default implementation is trivial and just loads the entire file // upfront. - $data = $file->loadFileData(); + $data = $this->readFile($file->getStorageHandle()); if ($begin !== null && $end !== null) { $data = substr($data, $begin, ($end - $begin)); diff --git a/src/applications/files/format/PhabricatorFileROT13StorageFormat.php b/src/applications/files/format/PhabricatorFileROT13StorageFormat.php new file mode 100644 --- /dev/null +++ b/src/applications/files/format/PhabricatorFileROT13StorageFormat.php @@ -0,0 +1,44 @@ +getFile(); + $iterations = $file->getStorageProperty('iterations', 1); + + $value = $file->loadDataFromIterator($raw_iterator); + for ($ii = 0; $ii < $iterations; $ii++) { + $value = str_rot13($value); + } + + return array($value); + } + + public function newWriteIterator($raw_iterator) { + return $this->newReadIterator($raw_iterator); + } + + public function newStorageProperties() { + // For extreme security, repeatedly encode the data using a random (odd) + // number of iterations. + return array( + 'iterations' => (mt_rand(1, 3) * 2) - 1, + ); + } + +} diff --git a/src/applications/files/format/PhabricatorFileRawStorageFormat.php b/src/applications/files/format/PhabricatorFileRawStorageFormat.php new file mode 100644 --- /dev/null +++ b/src/applications/files/format/PhabricatorFileRawStorageFormat.php @@ -0,0 +1,20 @@ +file = $file; + return $this; + } + + final public function getFile() { + if (!$this->file) { + throw new PhutilInvalidStateException('setFile'); + } + return $this->file; + } + + abstract public function getStorageFormatName(); + + abstract public function newReadIterator($raw_iterator); + abstract public function newWriteIterator($raw_iterator); + + public function newStorageProperties() { + return array(); + } + + final public function getStorageFormatKey() { + return $this->getPhobjectClassConstant('FORMATKEY'); + } + + final public static function getAllFormats() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getStorageFormatKey') + ->execute(); + } + + final public static function getFormat($key) { + $formats = self::getAllFormats(); + return idx($formats, $key); + } + + final public static function requireFormat($key) { + $format = self::getFormat($key); + + if (!$format) { + throw new Exception( + pht( + 'No file storage format with key "%s" exists.', + $key)); + } + + return $format; + } + +} diff --git a/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php new file mode 100644 --- /dev/null +++ b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php @@ -0,0 +1,38 @@ + true, + ); + } + + public function testRot13Storage() { + $engine = new PhabricatorTestStorageEngine(); + $rot13_format = PhabricatorFileROT13StorageFormat::FORMATKEY; + + $data = 'The cow jumped over the full moon.'; + $expect = 'Gur pbj whzcrq bire gur shyy zbba.'; + + $params = array( + 'name' => 'test.dat', + 'storageEngines' => array( + $engine, + ), + 'format' => $rot13_format, + ); + + $file = PhabricatorFile::newFromFileData($data, $params); + + // We should have a file stored as rot13, which reads back the input + // data correctly. + $this->assertEqual($rot13_format, $file->getStorageFormat()); + $this->assertEqual($data, $file->loadFileData()); + + // The actual raw data in the storage engine should be encoded. + $raw_data = $engine->readFile($file->getStorageHandle()); + $this->assertEqual($expect, $raw_data); + } + +} diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -26,14 +26,13 @@ PhabricatorPolicyInterface, PhabricatorDestructibleInterface { - const STORAGE_FORMAT_RAW = 'raw'; - const METADATA_IMAGE_WIDTH = 'width'; const METADATA_IMAGE_HEIGHT = 'height'; const METADATA_CAN_CDN = 'canCDN'; const METADATA_BUILTIN = 'builtin'; const METADATA_PARTIAL = 'partial'; const METADATA_PROFILE = 'profile'; + const METADATA_STORAGE = 'storage'; protected $name; protected $mimeType; @@ -233,10 +232,10 @@ $hash); if ($file) { - // copy storageEngine, storageHandle, storageFormat $copy_of_storage_engine = $file->getStorageEngine(); $copy_of_storage_handle = $file->getStorageHandle(); $copy_of_storage_format = $file->getStorageFormat(); + $copy_of_storage_properties = $file->getStorageProperties(); $copy_of_byte_size = $file->getByteSize(); $copy_of_mime_type = $file->getMimeType(); @@ -248,6 +247,7 @@ $new_file->setStorageEngine($copy_of_storage_engine); $new_file->setStorageHandle($copy_of_storage_handle); $new_file->setStorageFormat($copy_of_storage_format); + $new_file->setStorageProperties($copy_of_storage_properties); $new_file->setMimeType($copy_of_mime_type); $new_file->copyDimensions($file); @@ -290,7 +290,11 @@ $file->setStorageEngine($engine->getEngineIdentifier()); $file->setStorageHandle(PhabricatorFileChunk::newChunkHandle()); - $file->setStorageFormat(self::STORAGE_FORMAT_RAW); + + // Chunked files are always stored raw because they do not actually store + // data. The chunks do, and can be individually formatted. + $file->setStorageFormat(PhabricatorFileRawStorageFormat::FORMATKEY); + $file->setIsPartial(1); $file->readPropertiesFromParameters($params); @@ -322,6 +326,16 @@ $file = self::initializeNewFile(); + $default_key = PhabricatorFileRawStorageFormat::FORMATKEY; + $format_key = idx($params, 'format', $default_key); + + $format = id(clone PhabricatorFileStorageFormat::requireFormat($format_key)) + ->setFile($file); + + $properties = $format->newStorageProperties(); + $file->setStorageFormat($format->getStorageFormatKey()); + $file->setStorageProperties($properties); + $data_handle = null; $engine_identifier = null; $exceptions = array(); @@ -361,10 +375,6 @@ $file->setStorageEngine($engine_identifier); $file->setStorageHandle($data_handle); - // TODO: This is probably YAGNI, but allows for us to do encryption or - // compression later if we want. - $file->setStorageFormat(self::STORAGE_FORMAT_RAW); - $file->readPropertiesFromParameters($params); if (!$file->getMimeType()) { @@ -434,7 +444,15 @@ $engine_class = get_class($engine); - $data_handle = $engine->writeFile($data, $params); + $key = $this->getStorageFormat(); + $format = id(clone PhabricatorFileStorageFormat::requireFormat($key)) + ->setFile($this); + + $data_iterator = array($data); + $formatted_iterator = $format->newWriteIterator($data_iterator); + $formatted_data = $this->loadDataFromIterator($formatted_iterator); + + $data_handle = $engine->writeFile($formatted_data, $params); if (!$data_handle || strlen($data_handle) > 255) { // This indicates an improperly implemented storage engine. @@ -663,19 +681,8 @@ } public function loadFileData() { - - $engine = $this->instantiateStorageEngine(); - $data = $engine->readFile($this->getStorageHandle()); - - switch ($this->getStorageFormat()) { - case self::STORAGE_FORMAT_RAW: - $data = $data; - break; - default: - throw new Exception(pht('Unknown storage format.')); - } - - return $data; + $iterator = $this->getFileDataIterator(); + return $this->loadDataFromIterator($iterator); } @@ -688,7 +695,14 @@ */ public function getFileDataIterator($begin = null, $end = null) { $engine = $this->instantiateStorageEngine(); - return $engine->getFileDataIterator($this, $begin, $end); + $raw_iterator = $engine->getRawFileDataIterator($this, $begin, $end); + + $key = $this->getStorageFormat(); + + $format = id(clone PhabricatorFileStorageFormat::requireFormat($key)) + ->setFile($this); + + return $format->newReadIterator($raw_iterator); } @@ -917,6 +931,30 @@ return Filesystem::readRandomCharacters(20); } + public function setStorageProperties(array $properties) { + $this->metadata[self::METADATA_STORAGE] = $properties; + return $this; + } + + public function getStorageProperties() { + return idx($this->metadata, self::METADATA_STORAGE, array()); + } + + public function getStorageProperty($key, $default = null) { + $properties = $this->getStorageProperties(); + return idx($properties, $key, $default); + } + + public function loadDataFromIterator($iterator) { + $result = ''; + + foreach ($iterator as $chunk) { + $result .= $chunk; + } + + return $result; + } + public function updateDimensions($save = true) { if (!$this->isViewableImage()) { throw new Exception(pht('This file is not a viewable image.'));