Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F13185351
D16122.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
16 KB
Referenced Files
None
Subscribers
None
D16122.diff
View Options
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2490,12 +2490,16 @@
'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php',
'PhabricatorFileListController' => 'applications/files/controller/PhabricatorFileListController.php',
'PhabricatorFileQuery' => 'applications/files/query/PhabricatorFileQuery.php',
+ 'PhabricatorFileROT13StorageFormat' => 'applications/files/format/PhabricatorFileROT13StorageFormat.php',
+ 'PhabricatorFileRawStorageFormat' => 'applications/files/format/PhabricatorFileRawStorageFormat.php',
'PhabricatorFileSchemaSpec' => 'applications/files/storage/PhabricatorFileSchemaSpec.php',
'PhabricatorFileSearchEngine' => 'applications/files/query/PhabricatorFileSearchEngine.php',
'PhabricatorFileStorageBlob' => 'applications/files/storage/PhabricatorFileStorageBlob.php',
'PhabricatorFileStorageConfigurationException' => 'applications/files/exception/PhabricatorFileStorageConfigurationException.php',
'PhabricatorFileStorageEngine' => 'applications/files/engine/PhabricatorFileStorageEngine.php',
'PhabricatorFileStorageEngineTestCase' => 'applications/files/engine/__tests__/PhabricatorFileStorageEngineTestCase.php',
+ 'PhabricatorFileStorageFormat' => 'applications/files/format/PhabricatorFileStorageFormat.php',
+ 'PhabricatorFileStorageFormatTestCase' => 'applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php',
'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php',
'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php',
'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php',
@@ -7134,12 +7138,16 @@
'PhabricatorFileLinkView' => 'AphrontView',
'PhabricatorFileListController' => 'PhabricatorFileController',
'PhabricatorFileQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+ 'PhabricatorFileROT13StorageFormat' => 'PhabricatorFileStorageFormat',
+ 'PhabricatorFileRawStorageFormat' => 'PhabricatorFileStorageFormat',
'PhabricatorFileSchemaSpec' => 'PhabricatorConfigSchemaSpec',
'PhabricatorFileSearchEngine' => 'PhabricatorApplicationSearchEngine',
'PhabricatorFileStorageBlob' => 'PhabricatorFileDAO',
'PhabricatorFileStorageConfigurationException' => 'Exception',
'PhabricatorFileStorageEngine' => 'Phobject',
'PhabricatorFileStorageEngineTestCase' => 'PhabricatorTestCase',
+ 'PhabricatorFileStorageFormat' => 'Phobject',
+ 'PhabricatorFileStorageFormatTestCase' => 'PhabricatorTestCase',
'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector',
'PhabricatorFileTestCase' => 'PhabricatorTestCase',
'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator',
diff --git a/src/applications/files/controller/PhabricatorFileInfoController.php b/src/applications/files/controller/PhabricatorFileInfoController.php
--- a/src/applications/files/controller/PhabricatorFileInfoController.php
+++ b/src/applications/files/controller/PhabricatorFileInfoController.php
@@ -256,8 +256,10 @@
$types[] = pht('Profile');
}
- $types = implode(', ', $types);
- $finfo->addProperty(pht('Attributes'), $types);
+ if ($types) {
+ $types = implode(', ', $types);
+ $finfo->addProperty(pht('Attributes'), $types);
+ }
$storage_properties = new PHUIPropertyListView();
$box->addPropertyList($storage_properties, pht('Storage'));
@@ -266,9 +268,14 @@
pht('Engine'),
$file->getStorageEngine());
- $storage_properties->addProperty(
- pht('Format'),
- $file->getStorageFormat());
+ $format_key = $file->getStorageFormat();
+ $format = PhabricatorFileStorageFormat::getFormat($format_key);
+ if ($format) {
+ $format_name = $format->getStorageFormatName();
+ } else {
+ $format_name = pht('Unknown ("%s")', $format_key);
+ }
+ $storage_properties->addProperty(pht('Format'), $format_name);
$storage_properties->addProperty(
pht('Handle'),
diff --git a/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
--- a/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
+++ b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
@@ -174,7 +174,7 @@
return (4 * 1024 * 1024);
}
- public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
+ public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
$chunks = id(new PhabricatorFileChunkQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withChunkHandles(array($file->getStorageHandle()))
diff --git a/src/applications/files/engine/PhabricatorFileStorageEngine.php b/src/applications/files/engine/PhabricatorFileStorageEngine.php
--- a/src/applications/files/engine/PhabricatorFileStorageEngine.php
+++ b/src/applications/files/engine/PhabricatorFileStorageEngine.php
@@ -325,10 +325,10 @@
return $engine->getChunkSize();
}
- public function getFileDataIterator(PhabricatorFile $file, $begin, $end) {
+ public function getRawFileDataIterator(PhabricatorFile $file, $begin, $end) {
// The default implementation is trivial and just loads the entire file
// upfront.
- $data = $file->loadFileData();
+ $data = $this->readFile($file->getStorageHandle());
if ($begin !== null && $end !== null) {
$data = substr($data, $begin, ($end - $begin));
diff --git a/src/applications/files/format/PhabricatorFileROT13StorageFormat.php b/src/applications/files/format/PhabricatorFileROT13StorageFormat.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/format/PhabricatorFileROT13StorageFormat.php
@@ -0,0 +1,44 @@
+<?php
+
+/**
+ * Trivial example of a file storage format for at-rest encryption.
+ *
+ * This format applies ROT13 encoding to file data as it is stored and
+ * reverses it on the way out. This encoding is trivially reversible. This
+ * format is for testing, developing, and understanding encoding formats and
+ * is not intended for production use.
+ */
+final class PhabricatorFileROT13StorageFormat
+ extends PhabricatorFileStorageFormat {
+
+ const FORMATKEY = 'rot13';
+
+ public function getStorageFormatName() {
+ return pht('Encoded (ROT13)');
+ }
+
+ public function newReadIterator($raw_iterator) {
+ $file = $this->getFile();
+ $iterations = $file->getStorageProperty('iterations', 1);
+
+ $value = $file->loadDataFromIterator($raw_iterator);
+ for ($ii = 0; $ii < $iterations; $ii++) {
+ $value = str_rot13($value);
+ }
+
+ return array($value);
+ }
+
+ public function newWriteIterator($raw_iterator) {
+ return $this->newReadIterator($raw_iterator);
+ }
+
+ public function newStorageProperties() {
+ // For extreme security, repeatedly encode the data using a random (odd)
+ // number of iterations.
+ return array(
+ 'iterations' => (mt_rand(1, 3) * 2) - 1,
+ );
+ }
+
+}
diff --git a/src/applications/files/format/PhabricatorFileRawStorageFormat.php b/src/applications/files/format/PhabricatorFileRawStorageFormat.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/format/PhabricatorFileRawStorageFormat.php
@@ -0,0 +1,20 @@
+<?php
+
+final class PhabricatorFileRawStorageFormat
+ extends PhabricatorFileStorageFormat {
+
+ const FORMATKEY = 'raw';
+
+ public function getStorageFormatName() {
+ return pht('Raw Data');
+ }
+
+ public function newReadIterator($raw_iterator) {
+ return $raw_iterator;
+ }
+
+ public function newWriteIterator($raw_iterator) {
+ return $raw_iterator;
+ }
+
+}
diff --git a/src/applications/files/format/PhabricatorFileStorageFormat.php b/src/applications/files/format/PhabricatorFileStorageFormat.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/format/PhabricatorFileStorageFormat.php
@@ -0,0 +1,58 @@
+<?php
+
+abstract class PhabricatorFileStorageFormat
+ extends Phobject {
+
+ private $file;
+
+ final public function setFile(PhabricatorFile $file) {
+ $this->file = $file;
+ return $this;
+ }
+
+ final public function getFile() {
+ if (!$this->file) {
+ throw new PhutilInvalidStateException('setFile');
+ }
+ return $this->file;
+ }
+
+ abstract public function getStorageFormatName();
+
+ abstract public function newReadIterator($raw_iterator);
+ abstract public function newWriteIterator($raw_iterator);
+
+ public function newStorageProperties() {
+ return array();
+ }
+
+ final public function getStorageFormatKey() {
+ return $this->getPhobjectClassConstant('FORMATKEY');
+ }
+
+ final public static function getAllFormats() {
+ return id(new PhutilClassMapQuery())
+ ->setAncestorClass(__CLASS__)
+ ->setUniqueMethod('getStorageFormatKey')
+ ->execute();
+ }
+
+ final public static function getFormat($key) {
+ $formats = self::getAllFormats();
+ return idx($formats, $key);
+ }
+
+ final public static function requireFormat($key) {
+ $format = self::getFormat($key);
+
+ if (!$format) {
+ throw new Exception(
+ pht(
+ 'No file storage format with key "%s" exists.',
+ $key));
+ }
+
+ return $format;
+ }
+
+}
diff --git a/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php
@@ -0,0 +1,38 @@
+<?php
+
+final class PhabricatorFileStorageFormatTestCase extends PhabricatorTestCase {
+
+ protected function getPhabricatorTestCaseConfiguration() {
+ return array(
+ self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
+ );
+ }
+
+ public function testRot13Storage() {
+ $engine = new PhabricatorTestStorageEngine();
+ $rot13_format = PhabricatorFileROT13StorageFormat::FORMATKEY;
+
+ $data = 'The cow jumped over the full moon.';
+ $expect = 'Gur pbj whzcrq bire gur shyy zbba.';
+
+ $params = array(
+ 'name' => 'test.dat',
+ 'storageEngines' => array(
+ $engine,
+ ),
+ 'format' => $rot13_format,
+ );
+
+ $file = PhabricatorFile::newFromFileData($data, $params);
+
+ // We should have a file stored as rot13, which reads back the input
+ // data correctly.
+ $this->assertEqual($rot13_format, $file->getStorageFormat());
+ $this->assertEqual($data, $file->loadFileData());
+
+ // The actual raw data in the storage engine should be encoded.
+ $raw_data = $engine->readFile($file->getStorageHandle());
+ $this->assertEqual($expect, $raw_data);
+ }
+
+}
diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -26,14 +26,13 @@
PhabricatorPolicyInterface,
PhabricatorDestructibleInterface {
- const STORAGE_FORMAT_RAW = 'raw';
-
const METADATA_IMAGE_WIDTH = 'width';
const METADATA_IMAGE_HEIGHT = 'height';
const METADATA_CAN_CDN = 'canCDN';
const METADATA_BUILTIN = 'builtin';
const METADATA_PARTIAL = 'partial';
const METADATA_PROFILE = 'profile';
+ const METADATA_STORAGE = 'storage';
protected $name;
protected $mimeType;
@@ -233,10 +232,10 @@
$hash);
if ($file) {
- // copy storageEngine, storageHandle, storageFormat
$copy_of_storage_engine = $file->getStorageEngine();
$copy_of_storage_handle = $file->getStorageHandle();
$copy_of_storage_format = $file->getStorageFormat();
+ $copy_of_storage_properties = $file->getStorageProperties();
$copy_of_byte_size = $file->getByteSize();
$copy_of_mime_type = $file->getMimeType();
@@ -248,6 +247,7 @@
$new_file->setStorageEngine($copy_of_storage_engine);
$new_file->setStorageHandle($copy_of_storage_handle);
$new_file->setStorageFormat($copy_of_storage_format);
+ $new_file->setStorageProperties($copy_of_storage_properties);
$new_file->setMimeType($copy_of_mime_type);
$new_file->copyDimensions($file);
@@ -290,7 +290,11 @@
$file->setStorageEngine($engine->getEngineIdentifier());
$file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
- $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
+
+ // Chunked files are always stored raw because they do not actually store
+ // data. The chunks do, and can be individually formatted.
+ $file->setStorageFormat(PhabricatorFileRawStorageFormat::FORMATKEY);
+
$file->setIsPartial(1);
$file->readPropertiesFromParameters($params);
@@ -322,6 +326,16 @@
$file = self::initializeNewFile();
+ $default_key = PhabricatorFileRawStorageFormat::FORMATKEY;
+ $format_key = idx($params, 'format', $default_key);
+
+ $format = id(clone PhabricatorFileStorageFormat::requireFormat($format_key))
+ ->setFile($file);
+
+ $properties = $format->newStorageProperties();
+ $file->setStorageFormat($format->getStorageFormatKey());
+ $file->setStorageProperties($properties);
+
$data_handle = null;
$engine_identifier = null;
$exceptions = array();
@@ -361,10 +375,6 @@
$file->setStorageEngine($engine_identifier);
$file->setStorageHandle($data_handle);
- // TODO: This is probably YAGNI, but allows for us to do encryption or
- // compression later if we want.
- $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
-
$file->readPropertiesFromParameters($params);
if (!$file->getMimeType()) {
@@ -434,7 +444,15 @@
$engine_class = get_class($engine);
- $data_handle = $engine->writeFile($data, $params);
+ $key = $this->getStorageFormat();
+ $format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
+ ->setFile($this);
+
+ $data_iterator = array($data);
+ $formatted_iterator = $format->newWriteIterator($data_iterator);
+ $formatted_data = $this->loadDataFromIterator($formatted_iterator);
+
+ $data_handle = $engine->writeFile($formatted_data, $params);
if (!$data_handle || strlen($data_handle) > 255) {
// This indicates an improperly implemented storage engine.
@@ -663,19 +681,8 @@
}
public function loadFileData() {
-
- $engine = $this->instantiateStorageEngine();
- $data = $engine->readFile($this->getStorageHandle());
-
- switch ($this->getStorageFormat()) {
- case self::STORAGE_FORMAT_RAW:
- $data = $data;
- break;
- default:
- throw new Exception(pht('Unknown storage format.'));
- }
-
- return $data;
+ $iterator = $this->getFileDataIterator();
+ return $this->loadDataFromIterator($iterator);
}
@@ -688,7 +695,14 @@
*/
public function getFileDataIterator($begin = null, $end = null) {
$engine = $this->instantiateStorageEngine();
- return $engine->getFileDataIterator($this, $begin, $end);
+ $raw_iterator = $engine->getRawFileDataIterator($this, $begin, $end);
+
+ $key = $this->getStorageFormat();
+
+ $format = id(clone PhabricatorFileStorageFormat::requireFormat($key))
+ ->setFile($this);
+
+ return $format->newReadIterator($raw_iterator);
}
@@ -917,6 +931,30 @@
return Filesystem::readRandomCharacters(20);
}
+ public function setStorageProperties(array $properties) {
+ $this->metadata[self::METADATA_STORAGE] = $properties;
+ return $this;
+ }
+
+ public function getStorageProperties() {
+ return idx($this->metadata, self::METADATA_STORAGE, array());
+ }
+
+ public function getStorageProperty($key, $default = null) {
+ $properties = $this->getStorageProperties();
+ return idx($properties, $key, $default);
+ }
+
+ public function loadDataFromIterator($iterator) {
+ $result = '';
+
+ foreach ($iterator as $chunk) {
+ $result .= $chunk;
+ }
+
+ return $result;
+ }
+
public function updateDimensions($save = true) {
if (!$this->isViewableImage()) {
throw new Exception(pht('This file is not a viewable image.'));
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, May 12, 2:49 AM (3 w, 2 h ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
6287433
Default Alt Text
D16122.diff (16 KB)
Attached To
Mode
D16122: Add support to Files for file storage formats, to support encryption-at-rest
Attached
Detach File
Event Timeline
Log In to Comment