Page MenuHomePhabricator
Files F14042457

D12060.diff
No OneTemporary
Actions

D12060.diff
View Options

diff --git a/resources/sql/autopatches/20150312.filechunk.1.sql b/resources/sql/autopatches/20150312.filechunk.1.sql
new file mode 100644
--- /dev/null
+++ b/resources/sql/autopatches/20150312.filechunk.1.sql
@@ -0,0 +1,9 @@
+CREATE TABLE {$NAMESPACE}_file.file_chunk (
+ id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ chunkHandle BINARY(12) NOT NULL,
+ byteStart BIGINT UNSIGNED NOT NULL,
+ byteEnd BIGINT UNSIGNED NOT NULL,
+ dataFilePHID VARBINARY(64),
+ KEY `key_file` (chunkhandle, byteStart, byteEnd),
+ KEY `key_data` (dataFilePHID)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -747,12 +747,15 @@
'FeedPublisherWorker' => 'applications/feed/worker/FeedPublisherWorker.php',
'FeedPushWorker' => 'applications/feed/worker/FeedPushWorker.php',
'FeedQueryConduitAPIMethod' => 'applications/feed/conduit/FeedQueryConduitAPIMethod.php',
+ 'FileAllocateConduitAPIMethod' => 'applications/files/conduit/FileAllocateConduitAPIMethod.php',
'FileConduitAPIMethod' => 'applications/files/conduit/FileConduitAPIMethod.php',
'FileCreateMailReceiver' => 'applications/files/mail/FileCreateMailReceiver.php',
'FileDownloadConduitAPIMethod' => 'applications/files/conduit/FileDownloadConduitAPIMethod.php',
'FileInfoConduitAPIMethod' => 'applications/files/conduit/FileInfoConduitAPIMethod.php',
'FileMailReceiver' => 'applications/files/mail/FileMailReceiver.php',
+ 'FileQueryChunksConduitAPIMethod' => 'applications/files/conduit/FileQueryChunksConduitAPIMethod.php',
'FileReplyHandler' => 'applications/files/mail/FileReplyHandler.php',
+ 'FileUploadChunkConduitAPIMethod' => 'applications/files/conduit/FileUploadChunkConduitAPIMethod.php',
'FileUploadConduitAPIMethod' => 'applications/files/conduit/FileUploadConduitAPIMethod.php',
'FileUploadHashConduitAPIMethod' => 'applications/files/conduit/FileUploadHashConduitAPIMethod.php',
'FilesDefaultViewCapability' => 'applications/files/capability/FilesDefaultViewCapability.php',
@@ -1485,6 +1488,7 @@
'PhabricatorChatLogDAO' => 'applications/chatlog/storage/PhabricatorChatLogDAO.php',
'PhabricatorChatLogEvent' => 'applications/chatlog/storage/PhabricatorChatLogEvent.php',
'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
+ 'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
'PhabricatorCommitBranchesField' => 'applications/repository/customfield/PhabricatorCommitBranchesField.php',
'PhabricatorCommitCustomField' => 'applications/repository/customfield/PhabricatorCommitCustomField.php',
@@ -1790,6 +1794,8 @@
'PhabricatorFeedStoryReference' => 'applications/feed/storage/PhabricatorFeedStoryReference.php',
'PhabricatorFile' => 'applications/files/storage/PhabricatorFile.php',
'PhabricatorFileBundleLoader' => 'applications/files/query/PhabricatorFileBundleLoader.php',
+ 'PhabricatorFileChunk' => 'applications/files/storage/PhabricatorFileChunk.php',
+ 'PhabricatorFileChunkQuery' => 'applications/files/query/PhabricatorFileChunkQuery.php',
'PhabricatorFileCommentController' => 'applications/files/controller/PhabricatorFileCommentController.php',
'PhabricatorFileComposeController' => 'applications/files/controller/PhabricatorFileComposeController.php',
'PhabricatorFileController' => 'applications/files/controller/PhabricatorFileController.php',
@@ -3908,12 +3914,15 @@
'FeedPublisherWorker' => 'FeedPushWorker',
'FeedPushWorker' => 'PhabricatorWorker',
'FeedQueryConduitAPIMethod' => 'FeedConduitAPIMethod',
+ 'FileAllocateConduitAPIMethod' => 'FileConduitAPIMethod',
'FileConduitAPIMethod' => 'ConduitAPIMethod',
'FileCreateMailReceiver' => 'PhabricatorMailReceiver',
'FileDownloadConduitAPIMethod' => 'FileConduitAPIMethod',
'FileInfoConduitAPIMethod' => 'FileConduitAPIMethod',
'FileMailReceiver' => 'PhabricatorObjectMailReceiver',
+ 'FileQueryChunksConduitAPIMethod' => 'FileConduitAPIMethod',
'FileReplyHandler' => 'PhabricatorMailReplyHandler',
+ 'FileUploadChunkConduitAPIMethod' => 'FileConduitAPIMethod',
'FileUploadConduitAPIMethod' => 'FileConduitAPIMethod',
'FileUploadHashConduitAPIMethod' => 'FileConduitAPIMethod',
'FilesDefaultViewCapability' => 'PhabricatorPolicyCapability',
@@ -4748,6 +4757,7 @@
'PhabricatorPolicyInterface',
),
'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
+ 'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorCommitBranchesField' => 'PhabricatorCommitCustomField',
'PhabricatorCommitCustomField' => 'PhabricatorCustomField',
@@ -5081,6 +5091,12 @@
'PhabricatorPolicyInterface',
'PhabricatorDestructibleInterface',
),
+ 'PhabricatorFileChunk' => array(
+ 'PhabricatorFileDAO',
+ 'PhabricatorPolicyInterface',
+ 'PhabricatorDestructibleInterface',
+ ),
+ 'PhabricatorFileChunkQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'PhabricatorFileCommentController' => 'PhabricatorFileController',
'PhabricatorFileComposeController' => 'PhabricatorFileController',
'PhabricatorFileController' => 'PhabricatorController',
diff --git a/src/applications/files/conduit/FileAllocateConduitAPIMethod.php b/src/applications/files/conduit/FileAllocateConduitAPIMethod.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/conduit/FileAllocateConduitAPIMethod.php
@@ -0,0 +1,131 @@
+<?php
+
+final class FileAllocateConduitAPIMethod
+ extends FileConduitAPIMethod {
+
+ public function getAPIMethodName() {
+ return 'file.allocate';
+ }
+
+ public function getMethodDescription() {
+ return pht('Prepare to upload a file.');
+ }
+
+ public function defineParamTypes() {
+ return array(
+ 'name' => 'string',
+ 'contentLength' => 'int',
+ 'contentHash' => 'optional string',
+ 'viewPolicy' => 'optional string',
+
+ // TODO: Remove this, it's just here to make testing easier.
+ 'forceChunking' => 'optional bool',
+ );
+ }
+
+ public function defineReturnType() {
+ return 'map<string, wild>';
+ }
+
+ public function defineErrorTypes() {
+ return array();
+ }
+
+ protected function execute(ConduitAPIRequest $request) {
+ $viewer = $request->getUser();
+
+ $hash = $request->getValue('contentHash');
+ $name = $request->getValue('name');
+ $view_policy = $request->getValue('viewPolicy');
+ $content_length = $request->getValue('contentLength');
+
+ $force_chunking = $request->getValue('forceChunking');
+
+ $properties = array(
+ 'name' => $name,
+ 'authorPHID' => $viewer->getPHID(),
+ 'viewPolicy' => $view_policy,
+ 'isExplicitUpload' => true,
+ );
+
+ if ($hash) {
+ $file = PhabricatorFile::newFileFromContentHash(
+ $hash,
+ $properties);
+
+ if ($file && !$force_chunking) {
+ return array(
+ 'upload' => false,
+ 'filePHID' => $file->getPHID(),
+ );
+ }
+
+ $chunked_hash = PhabricatorChunkedFileStorageEngine::getChunkedHash(
+ $viewer,
+ $hash);
+ $file = id(new PhabricatorFileQuery())
+ ->setViewer($viewer)
+ ->withContentHashes(array($chunked_hash))
+ ->executeOne();
+
+ if ($file) {
+ return array(
+ 'upload' => $file->isPartial(),
+ 'filePHID' => $file->getPHID(),
+ );
+ }
+ }
+
+ $engines = PhabricatorFileStorageEngine::loadStorageEngines(
+ $content_length);
+ if ($engines) {
+
+ if ($force_chunking) {
+ foreach ($engines as $key => $engine) {
+ if (!$engine->isChunkEngine()) {
+ unset($engines[$key]);
+ }
+ }
+ }
+
+ // Pick the first engine. If the file is small enough to fit into a
+ // single engine without chunking, this will be a non-chunk engine and
+ // we'll just tell the client to upload the file.
+ $engine = head($engines);
+ if ($engine) {
+ if (!$engine->isChunkEngine()) {
+ return array(
+ 'upload' => true,
+ 'filePHID' => null,
+ );
+ }
+
+ // Otherwise, this is a large file and we need to perform a chunked
+ // upload.
+
+ $chunk_properties = array();
+
+ if ($hash) {
+ $chunk_properties += array(
+ 'chunkedHash' => $chunked_hash,
+ );
+ }
+
+ $file = $engine->allocateChunks($content_length, $chunk_properties);
+
+ return array(
+ 'upload' => true,
+ 'filePHID' => $file->getPHID(),
+ );
+ }
+ }
+
+ // None of the storage engines can accept this file.
+
+ return array(
+ 'upload' => false,
+ 'filePHID' => null,
+ );
+ }
+
+}
diff --git a/src/applications/files/conduit/FileConduitAPIMethod.php b/src/applications/files/conduit/FileConduitAPIMethod.php
--- a/src/applications/files/conduit/FileConduitAPIMethod.php
+++ b/src/applications/files/conduit/FileConduitAPIMethod.php
@@ -6,4 +6,104 @@
return PhabricatorApplication::getByClass('PhabricatorFilesApplication');
}
+ protected function loadFileByPHID(PhabricatorUser $viewer, $file_phid) {
+ $file = id(new PhabricatorFileQuery())
+ ->setViewer($viewer)
+ ->withPHIDs(array($file_phid))
+ ->executeOne();
+ if (!$file) {
+ throw new Exception(pht('No such file "%s"!', $file_phid));
+ }
+
+ return $file;
+ }
+
+ protected function loadFileChunks(
+ PhabricatorUser $viewer,
+ PhabricatorFile $file) {
+ return $this->newChunkQuery($viewer, $file)
+ ->execute();
+ }
+
+ protected function loadFileChunkForUpload(
+ PhabricatorUser $viewer,
+ PhabricatorFile $file,
+ $start,
+ $end) {
+
+ $start = (int)$start;
+ $end = (int)$end;
+
+ $chunks = $this->newChunkQuery($viewer, $file)
+ ->withByteRange($start, $end)
+ ->execute();
+
+ if (!$chunks) {
+ throw new Exception(
+ pht(
+ 'There are no file data chunks in byte range %d - %d.',
+ $start,
+ $end));
+ }
+
+ if (count($chunks) !== 1) {
+ phlog($chunks);
+ throw new Exception(
+ pht(
+ 'There are multiple chunks in byte range %d - %d.',
+ $start,
+ $end));
+ }
+
+ $chunk = head($chunks);
+ if ($chunk->getByteStart() != $start) {
+ throw new Exception(
+ pht(
+ 'Chunk start byte is %d, not %d.',
+ $chunk->getByteStart(),
+ $start));
+ }
+
+ if ($chunk->getByteEnd() != $end) {
+ throw new Exception(
+ pht(
+ 'Chunk end byte is %d, not %d.',
+ $chunk->getByteEnd(),
+ $end));
+ }
+
+ if ($chunk->getDataFilePHID()) {
+ throw new Exception(
+ pht(
+ 'Chunk has already been uploaded.'));
+ }
+
+ return $chunk;
+ }
+
+ protected function decodeBase64($data) {
+ $data = base64_decode($data, $strict = true);
+ if ($data === false) {
+ throw new Exception(pht('Unable to decode base64 data!'));
+ }
+ return $data;
+ }
+
+ private function newChunkQuery(
+ PhabricatorUser $viewer,
+ PhabricatorFile $file) {
+
+ $engine = $file->instantiateStorageEngine();
+ if (!$engine->isChunkEngine()) {
+ throw new Exception(
+ pht(
+ 'File "%s" does not have chunks!',
+ $file->getPHID()));
+ }
+
+ return id(new PhabricatorFileChunkQuery())
+ ->setViewer($viewer)
+ ->withChunkHandles(array($file->getStorageHandle()));
+ }
+
}
diff --git a/src/applications/files/conduit/FileQueryChunksConduitAPIMethod.php b/src/applications/files/conduit/FileQueryChunksConduitAPIMethod.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/conduit/FileQueryChunksConduitAPIMethod.php
@@ -0,0 +1,47 @@
+<?php
+
+final class FileQueryChunksConduitAPIMethod
+ extends FileConduitAPIMethod {
+
+ public function getAPIMethodName() {
+ return 'file.querychunks';
+ }
+
+ public function getMethodDescription() {
+ return pht('Get information about file chunks.');
+ }
+
+ public function defineParamTypes() {
+ return array(
+ 'filePHID' => 'phid',
+ );
+ }
+
+ public function defineReturnType() {
+ return 'list<wild>';
+ }
+
+ public function defineErrorTypes() {
+ return array();
+ }
+
+ protected function execute(ConduitAPIRequest $request) {
+ $viewer = $request->getUser();
+
+ $file_phid = $request->getValue('filePHID');
+ $file = $this->loadFileByPHID($viewer, $file_phid);
+ $chunks = $this->loadFileChunks($viewer, $file);
+
+ $results = array();
+ foreach ($chunks as $chunk) {
+ $results[] = array(
+ 'byteStart' => $chunk->getByteStart(),
+ 'byteEnd' => $chunk->getByteEnd(),
+ 'complete' => (bool)$chunk->getDataFilePHID(),
+ );
+ }
+
+ return $results;
+ }
+
+}
diff --git a/src/applications/files/conduit/FileUploadChunkConduitAPIMethod.php b/src/applications/files/conduit/FileUploadChunkConduitAPIMethod.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/conduit/FileUploadChunkConduitAPIMethod.php
@@ -0,0 +1,74 @@
+<?php
+
+final class FileUploadChunkConduitAPIMethod
+ extends FileConduitAPIMethod {
+
+ public function getAPIMethodName() {
+ return 'file.uploadchunk';
+ }
+
+ public function getMethodDescription() {
+ return pht('Upload a chunk of file data to the server.');
+ }
+
+ public function defineParamTypes() {
+ return array(
+ 'filePHID' => 'phid',
+ 'byteStart' => 'int',
+ 'data' => 'string',
+ 'dataEncoding' => 'string',
+ );
+ }
+
+ public function defineReturnType() {
+ return 'void';
+ }
+
+ public function defineErrorTypes() {
+ return array();
+ }
+
+ protected function execute(ConduitAPIRequest $request) {
+ $viewer = $request->getUser();
+
+ $file_phid = $request->getValue('filePHID');
+ $file = $this->loadFileByPHID($viewer, $file_phid);
+
+ $start = $request->getValue('byteStart');
+
+ $data = $request->getValue('data');
+ $encoding = $request->getValue('dataEncoding');
+ switch ($encoding) {
+ case 'base64':
+ $data = $this->decodeBase64($data);
+ break;
+ case null:
+ break;
+ default:
+ throw new Exception(pht('Unsupported data encoding.'));
+ }
+ $length = strlen($data);
+
+ $chunk = $this->loadFileChunkForUpload(
+ $viewer,
+ $file,
+ $start,
+ $start + $length);
+
+ // NOTE: These files have a view policy which prevents normal access. They
+ // are only accessed through the storage engine.
+ $file = PhabricatorFile::newFromFileData(
+ $data,
+ array(
+ 'name' => $file->getMonogram().'.chunk-'.$chunk->getID(),
+ 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
+ ));
+
+ $chunk->setDataFilePHID($file->getPHID())->save();
+
+ // TODO: If all chunks are up, mark the file as complete.
+
+ return null;
+ }
+
+}
diff --git a/src/applications/files/conduit/FileUploadHashConduitAPIMethod.php b/src/applications/files/conduit/FileUploadHashConduitAPIMethod.php
--- a/src/applications/files/conduit/FileUploadHashConduitAPIMethod.php
+++ b/src/applications/files/conduit/FileUploadHashConduitAPIMethod.php
@@ -3,6 +3,7 @@
final class FileUploadHashConduitAPIMethod extends FileConduitAPIMethod {
public function getAPIMethodName() {
+ // TODO: Deprecate this in favor of `file.allocate`.
return 'file.uploadhash';
}
diff --git a/src/applications/files/controller/PhabricatorFileInfoController.php b/src/applications/files/controller/PhabricatorFileInfoController.php
--- a/src/applications/files/controller/PhabricatorFileInfoController.php
+++ b/src/applications/files/controller/PhabricatorFileInfoController.php
@@ -295,6 +295,67 @@
$box->addPropertyList($media);
}
+
+ $engine = null;
+ try {
+ $engine = $file->instantiateStorageEngine();
+ } catch (Exception $ex) {
+ // Don't bother raising this anywhere for now.
+ }
+
+ if ($engine) {
+ if ($engine->isChunkEngine()) {
+ $chunkinfo = new PHUIPropertyListView();
+ $box->addPropertyList($chunkinfo, pht('Chunks'));
+
+ $chunks = id(new PhabricatorFileChunkQuery())
+ ->setViewer($user)
+ ->withChunkHandles(array($file->getStorageHandle()))
+ ->execute();
+ $chunks = msort($chunks, 'getByteStart');
+
+ $rows = array();
+ $completed = array();
+ foreach ($chunks as $chunk) {
+ $is_complete = $chunk->getDataFilePHID();
+
+ $rows[] = array(
+ $chunk->getByteStart(),
+ $chunk->getByteEnd(),
+ ($is_complete ? pht('Yes') : pht('No')),
+ );
+
+ if ($is_complete) {
+ $completed[] = $chunk;
+ }
+ }
+
+ $table = id(new AphrontTableView($rows))
+ ->setHeaders(
+ array(
+ pht('Offset'),
+ pht('End'),
+ pht('Complete'),
+ ))
+ ->setColumnClasses(
+ array(
+ '',
+ '',
+ 'wide',
+ ));
+
+ $chunkinfo->addProperty(
+ pht('Total Chunks'),
+ count($chunks));
+
+ $chunkinfo->addProperty(
+ pht('Completed Chunks'),
+ count($completed));
+
+ $chunkinfo->addRawContent($table);
+ }
+ }
+
}
}
diff --git a/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/engine/PhabricatorChunkedFileStorageEngine.php
@@ -0,0 +1,171 @@
+<?php
+
+final class PhabricatorChunkedFileStorageEngine
+ extends PhabricatorFileStorageEngine {
+
+ public function getEngineIdentifier() {
+ return 'chunks';
+ }
+
+ public function getEnginePriority() {
+ return 60000;
+ }
+
+ /**
+ * We can write chunks if we have at least one valid storage engine
+ * underneath us.
+ *
+ * This engine must not also be a chunk engine.
+ */
+ public function canWriteFiles() {
+ return (bool)$this->getWritableEngine();
+ }
+
+ public function hasFilesizeLimit() {
+ return false;
+ }
+
+ public function isChunkEngine() {
+ return true;
+ }
+
+ public function isTestEngine() {
+ // TODO: For now, prevent this from actually being selected.
+ return true;
+ }
+
+ public function writeFile($data, array $params) {
+ // The chunk engine does not support direct writes.
+ throw new PhutilMethodNotImplementedException();
+ }
+
+ public function readFile($handle) {
+ // This is inefficient, but makes the API work as expected.
+ $chunks = $this->loadAllChunks($handle, true);
+
+ $buffer = '';
+ foreach ($chunks as $chunk) {
+ $data_file = $chunk->getDataFile();
+ if (!$data_file) {
+ throw new Exception(pht('This file data is incomplete!'));
+ }
+
+ $buffer .= $chunk->getDataFile()->loadFileData();
+ }
+
+ return $buffer;
+ }
+
+ public function deleteFile($handle) {
+ $engine = new PhabricatorDestructionEngine();
+ $chunks = $this->loadAllChunks($handle);
+ foreach ($chunks as $chunk) {
+ $engine->destroyObject($chunk);
+ }
+ }
+
+ private function loadAllChunks($handle, $need_files) {
+ $chunks = id(new PhabricatorFileChunkQuery())
+ ->setViewer(PhabricatorUser::getOmnipotentUser())
+ ->withChunkHandles(array($handle))
+ ->needDataFiles($need_files)
+ ->execute();
+
+ $chunks = msort($chunks, 'getByteStart');
+
+ return $chunks;
+ }
+
+ /**
+ * Compute a chunked file hash for the viewer.
+ *
+ * We can not currently compute a real hash for chunked file uploads (because
+ * no process sees all of the file data).
+ *
+ * We also can not trust the hash that the user claims to have computed. If
+ * we trust the user, they can upload some `evil.exe` and claim it has the
+ * same file hash as `good.exe`. When another user later uploads the real
+ * `good.exe`, we'll just create a reference to the existing `evil.exe`. Users
+ * who download `good.exe` will then receive `evil.exe`.
+ *
+ * Instead, we rehash the user's claimed hash with account secrets. This
+ * allows users to resume file uploads, but not collide with other users.
+ *
+ * Ideally, we'd like to be able to verify hashes, but this is complicated
+ * and time consuming and gives us a fairly small benefit.
+ *
+ * @param PhabricatorUser Viewing user.
+ * @param string Claimed file hash.
+ * @return string Rehashed file hash.
+ */
+ public static function getChunkedHash(PhabricatorUser $viewer, $hash) {
+ if (!$viewer->getPHID()) {
+ throw new Exception(
+ pht('Unable to compute chunked hash without real viewer!'));
+ }
+
+ $input = $viewer->getAccountSecret().':'.$hash.':'.$viewer->getPHID();
+ return PhabricatorHash::digest($input);
+ }
+
+ public function allocateChunks($length, array $properties) {
+ $file = PhabricatorFile::newChunkedFile($this, $length, $properties);
+
+ $chunk_size = $this->getChunkSize();
+
+ $handle = $file->getStorageHandle();
+
+ $chunks = array();
+ for ($ii = 0; $ii < $length; $ii += $chunk_size) {
+ $chunks[] = PhabricatorFileChunk::initializeNewChunk(
+ $handle,
+ $ii,
+ min($ii + $chunk_size, $length));
+ }
+
+ $file->openTransaction();
+ foreach ($chunks as $chunk) {
+ $chunk->save();
+ }
+ $file->save();
+ $file->saveTransaction();
+
+ return $file;
+ }
+
+ private function getWritableEngine() {
+ // NOTE: We can't just load writable engines or we'll loop forever.
+ $engines = PhabricatorFileStorageEngine::loadAllEngines();
+
+ foreach ($engines as $engine) {
+ if ($engine->isChunkEngine()) {
+ continue;
+ }
+
+ if ($engine->isTestEngine()) {
+ continue;
+ }
+
+ if (!$engine->canWriteFiles()) {
+ continue;
+ }
+
+ if ($engine->hasFilesizeLimit()) {
+ if ($engine->getFilesizeLimit() < $this->getChunkSize()) {
+ continue;
+ }
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ private function getChunkSize() {
+ // TODO: This is an artificially small size to make it easier to
+ // test chunking.
+ return 32;
+ }
+
+}
diff --git a/src/applications/files/engine/PhabricatorFileStorageEngine.php b/src/applications/files/engine/PhabricatorFileStorageEngine.php
--- a/src/applications/files/engine/PhabricatorFileStorageEngine.php
+++ b/src/applications/files/engine/PhabricatorFileStorageEngine.php
@@ -113,6 +113,21 @@
}
+ /**
+ * Identifies chunking storage engines.
+ *
+ * If this is a storage engine which splits files into chunks and stores the
+ * chunks in other engines, it can return `true` to signal that other
+ * chunking engines should not try to store data here.
+ *
+ * @return bool True if this is a chunk engine.
+ * @task meta
+ */
+ public function isChunkEngine() {
+ return false;
+ }
+
+
/* -( Managing File Data )------------------------------------------------- */
diff --git a/src/applications/files/query/PhabricatorFileChunkQuery.php b/src/applications/files/query/PhabricatorFileChunkQuery.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/query/PhabricatorFileChunkQuery.php
@@ -0,0 +1,116 @@
+<?php
+
+final class PhabricatorFileChunkQuery
+ extends PhabricatorCursorPagedPolicyAwareQuery {
+
+ private $chunkHandles;
+ private $rangeStart;
+ private $rangeEnd;
+ private $needDataFiles;
+
+ public function withChunkHandles(array $handles) {
+ $this->chunkHandles = $handles;
+ return $this;
+ }
+
+ public function withByteRange($start, $end) {
+ $this->rangeStart = $start;
+ $this->rangeEnd = $end;
+ return $this;
+ }
+
+ public function needDataFiles($need) {
+ $this->needDataFiles = $need;
+ return $this;
+ }
+
+ protected function loadPage() {
+ $table = new PhabricatorFileChunk();
+ $conn_r = $table->establishConnection('r');
+
+ $data = queryfx_all(
+ $conn_r,
+ 'SELECT * FROM %T %Q %Q %Q',
+ $table->getTableName(),
+ $this->buildWhereClause($conn_r),
+ $this->buildOrderClause($conn_r),
+ $this->buildLimitClause($conn_r));
+
+ return $table->loadAllFromArray($data);
+ }
+
+ protected function willFilterPage(array $chunks) {
+
+ if ($this->needDataFiles) {
+ $file_phids = mpull($chunks, 'getDataFilePHID');
+ $file_phids = array_filter($file_phids);
+ if ($file_phids) {
+ $files = id(new PhabricatorFileQuery())
+ ->setViewer($this->getViewer())
+ ->setParentQuery($this)
+ ->withPHIDs($file_phids)
+ ->execute();
+ $files = mpull($files, null, 'getPHID');
+ } else {
+ $files = array();
+ }
+
+ foreach ($chunks as $key => $chunk) {
+ $data_phid = $chunk->getDataFilePHID();
+ if (!$data_phid) {
+ $chunk->attachDataFile(null);
+ continue;
+ }
+
+ $file = idx($files, $data_phid);
+ if (!$file) {
+ unset($chunks[$key]);
+ $this->didRejectResult($chunk);
+ continue;
+ }
+
+ $chunk->attachDataFile($file);
+ }
+
+ if (!$chunks) {
+ return $chunks;
+ }
+ }
+
+ return $chunks;
+ }
+
+ private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
+ $where = array();
+
+ if ($this->chunkHandles !== null) {
+ $where[] = qsprintf(
+ $conn_r,
+ 'chunkHandle IN (%Ls)',
+ $this->chunkHandles);
+ }
+
+ if ($this->rangeStart !== null) {
+ $where[] = qsprintf(
+ $conn_r,
+ 'byteEnd > %d',
+ $this->rangeStart);
+ }
+
+ if ($this->rangeEnd !== null) {
+ $where[] = qsprintf(
+ $conn_r,
+ 'byteStart < %d',
+ $this->rangeEnd);
+ }
+
+ $where[] = $this->buildPagingClause($conn_r);
+
+ return $this->formatWhereClause($where);
+ }
+
+ public function getQueryApplicationClass() {
+ return 'PhabricatorFilesApplication';
+ }
+
+}
diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -33,6 +33,7 @@
const METADATA_IMAGE_HEIGHT = 'height';
const METADATA_CAN_CDN = 'canCDN';
const METADATA_BUILTIN = 'builtin';
+ const METADATA_PARTIAL = 'partial';
protected $name;
protected $mimeType;
@@ -264,6 +265,41 @@
return $file;
}
+ public static function newChunkedFile(
+ PhabricatorFileStorageEngine $engine,
+ $length,
+ array $params) {
+
+ $file = PhabricatorFile::initializeNewFile();
+
+ $file->setByteSize($length);
+
+ // TODO: We might be able to test the first chunk in order to figure
+ // this out more reliably, since MIME detection usually examines headers.
+ // However, enormous files are probably always either actually raw data
+ // or reasonable to treat like raw data.
+ $file->setMimeType('application/octet-stream');
+
+ $chunked_hash = idx($params, 'chunkedHash');
+ if ($chunked_hash) {
+ $file->setContentHash($chunked_hash);
+ } else {
+ // See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some
+ // discussion of this.
+ $file->setContentHash(
+ PhabricatorHash::digest(
+ Filesystem::readRandomBytes(64)));
+ }
+
+ $file->setStorageEngine($engine->getEngineIdentifier());
+ $file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
+ $file->setStorageFormat(self::STORAGE_FORMAT_RAW);
+
+ $file->readPropertiesFromParameters($params);
+
+ return $file;
+ }
+
private static function buildFromFileData($data, array $params = array()) {
if (isset($params['storageEngines'])) {
@@ -1134,6 +1170,11 @@
->setURI($uri);
}
+ public function isPartial() {
+ // TODO: Placeholder for resumable uploads.
+ return false;
+ }
+
/* -( PhabricatorApplicationTransactionInterface )------------------------- */
diff --git a/src/applications/files/storage/PhabricatorFileChunk.php b/src/applications/files/storage/PhabricatorFileChunk.php
new file mode 100644
--- /dev/null
+++ b/src/applications/files/storage/PhabricatorFileChunk.php
@@ -0,0 +1,105 @@
+<?php
+
+final class PhabricatorFileChunk extends PhabricatorFileDAO
+ implements
+ PhabricatorPolicyInterface,
+ PhabricatorDestructibleInterface {
+
+ protected $chunkHandle;
+ protected $byteStart;
+ protected $byteEnd;
+ protected $dataFilePHID;
+
+ private $dataFile = self::ATTACHABLE;
+
+ protected function getConfiguration() {
+ return array(
+ self::CONFIG_TIMESTAMPS => false,
+ self::CONFIG_COLUMN_SCHEMA => array(
+ 'chunkHandle' => 'bytes12',
+ 'byteStart' => 'uint64',
+ 'byteEnd' => 'uint64',
+ 'dataFilePHID' => 'phid?',
+ ),
+ self::CONFIG_KEY_SCHEMA => array(
+ 'key_file' => array(
+ 'columns' => array('chunkHandle', 'byteStart', 'byteEnd'),
+ ),
+ 'key_data' => array(
+ 'columns' => array('dataFilePHID'),
+ ),
+ ),
+ ) + parent::getConfiguration();
+ }
+
+ public static function newChunkHandle() {
+ $seed = Filesystem::readRandomBytes(64);
+ return PhabricatorHash::digestForIndex($seed);
+ }
+
+ public static function initializeNewChunk($handle, $start, $end) {
+ return id(new PhabricatorFileChunk())
+ ->setChunkHandle($handle)
+ ->setByteStart($start)
+ ->setByteEnd($end);
+ }
+
+ public function attachDataFile(PhabricatorFile $file = null) {
+ $this->dataFile = $file;
+ return $this;
+ }
+
+ public function getDataFile() {
+ return $this->assertAttached($this->dataFile);
+ }
+
+
+/* -( PhabricatorPolicyInterface )----------------------------------------- */
+
+
+ public function getCapabilities() {
+ return array(
+ PhabricatorPolicyCapability::CAN_VIEW,
+ );
+ }
+
+
+ public function getPolicy($capability) {
+ // These objects are low-level and only accessed through the storage
+ // engine, so policies are mostly just in place to let us use the common
+ // query infrastructure.
+ return PhabricatorPolicies::getMostOpenPolicy();
+ }
+
+
+ public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
+ return false;
+ }
+
+
+ public function describeAutomaticCapability($capability) {
+ return null;
+ }
+
+
+/* -( PhabricatorDestructibleInterface )----------------------------------- */
+
+
+ public function destroyObjectPermanently(
+ PhabricatorDestructionEngine $engine) {
+
+ $data_phid = $this->getDataFilePHID();
+ if ($data_phid) {
+ $data_file = id(new PhabricatorFileQuery())
+ ->setViewer(PhabricatorUser::getOmnipotentUser())
+ ->withPHIDs(array($data_phid))
+ ->executeOne();
+ if ($data_file) {
+ $engine->destroyObject($data_file);
+ }
+ }
+
+ $this->delete();
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Wed, Nov 13, 3:40 AM (5 d, 15 h ago)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/5p/rl/wng45knfc4taul5a
Default Alt Text
D12060.diff (31 KB)

Event Timeline

Phabricator · Built by Phacility