diff --git a/resources/sql/autopatches/20160921.fileexternalrequest.sql b/resources/sql/autopatches/20160921.fileexternalrequest.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160921.fileexternalrequest.sql @@ -0,0 +1,14 @@ +CREATE TABLE {$NAMESPACE}_file.file_externalrequest ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + filePHID VARBINARY(64), + ttl INT UNSIGNED NOT NULL, + uri LONGTEXT NOT NULL, + uriIndex BINARY(12) NOT NULL, + isSuccessful BOOL NOT NULL, + responseMessage LONGTEXT, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_uriindex` (uriIndex), + KEY `key_ttl` (ttl), + KEY `key_file` (filePHID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2553,10 +2553,13 @@ 'PhabricatorFileDropUploadController' => 'applications/files/controller/PhabricatorFileDropUploadController.php', 'PhabricatorFileEditController' => 'applications/files/controller/PhabricatorFileEditController.php', 'PhabricatorFileEditor' => 'applications/files/editor/PhabricatorFileEditor.php', + 'PhabricatorFileExternalRequest' => 'applications/files/storage/PhabricatorFileExternalRequest.php', + 'PhabricatorFileExternalRequestGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php', 'PhabricatorFileFilePHIDType' => 'applications/files/phid/PhabricatorFileFilePHIDType.php', 'PhabricatorFileHasObjectEdgeType' => 'applications/files/edge/PhabricatorFileHasObjectEdgeType.php', 'PhabricatorFileIconSetSelectController' => 'applications/files/controller/PhabricatorFileIconSetSelectController.php', 'PhabricatorFileImageMacro' => 'applications/macro/storage/PhabricatorFileImageMacro.php', + 'PhabricatorFileImageProxyController' => 'applications/files/controller/PhabricatorFileImageProxyController.php', 'PhabricatorFileImageTransform' => 'applications/files/transform/PhabricatorFileImageTransform.php', 'PhabricatorFileInfoController' => 'applications/files/controller/PhabricatorFileInfoController.php', 'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php', @@ -7368,6 +7371,11 @@ 'PhabricatorFileDropUploadController' => 'PhabricatorFileController', 'PhabricatorFileEditController' => 'PhabricatorFileController', 'PhabricatorFileEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhabricatorFileExternalRequest' => array( + 'PhabricatorFileDAO', + 'PhabricatorDestructibleInterface', + ), + 'PhabricatorFileExternalRequestGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorFileFilePHIDType' => 'PhabricatorPHIDType', 'PhabricatorFileHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorFileIconSetSelectController' => 'PhabricatorFileController', @@ -7379,6 +7387,7 @@ 'PhabricatorTokenReceiverInterface', 'PhabricatorPolicyInterface', ), + 'PhabricatorFileImageProxyController' => 'PhabricatorFileController', 'PhabricatorFileImageTransform' => 'PhabricatorFileTransform', 'PhabricatorFileInfoController' => 'PhabricatorFileController', 'PhabricatorFileLinkView' => 'AphrontView', diff --git a/src/applications/files/application/PhabricatorFilesApplication.php b/src/applications/files/application/PhabricatorFilesApplication.php --- a/src/applications/files/application/PhabricatorFilesApplication.php +++ b/src/applications/files/application/PhabricatorFilesApplication.php @@ -78,7 +78,7 @@ 'delete/(?P[1-9]\d*)/' => 'PhabricatorFileDeleteController', 'edit/(?P[1-9]\d*)/' => 'PhabricatorFileEditController', 'info/(?P[^/]+)/' => 'PhabricatorFileInfoController', - 'proxy/' => 'PhabricatorFileProxyController', + 'imageproxy/' => 'PhabricatorFileImageProxyController', 'transforms/(?P[1-9]\d*)/' => 'PhabricatorFileTransformListController', 'uploaddialog/(?Psingle/)?' diff --git a/src/applications/files/controller/PhabricatorFileImageProxyController.php b/src/applications/files/controller/PhabricatorFileImageProxyController.php new file mode 100644 --- /dev/null +++ b/src/applications/files/controller/PhabricatorFileImageProxyController.php @@ -0,0 +1,118 @@ +getViewer(); + $img_uri = $request->getStr('uri'); + + // Validate the URI before doing anything + PhabricatorEnv::requireValidRemoteURIForLink($img_uri); + $uri = new PhutilURI($img_uri); + $proto = $uri->getProtocol(); + if (!in_array($proto, array('http', 'https'))) { + throw new Exception( + pht('The provided image URI must be either http or https')); + } + + // Check if we already have the specified image URI downloaded + $cached_request = id(new PhabricatorFileExternalRequest())->loadOneWhere( + 'uriIndex = %s', + PhabricatorHash::digestForIndex($img_uri)); + + if ($cached_request) { + return $this->getExternalResponse($cached_request); + } + + $ttl = PhabricatorTime::getNow() + phutil_units('7 days in seconds'); + $external_request = id(new PhabricatorFileExternalRequest()) + ->setURI($img_uri) + ->setTTL($ttl); + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + // Cache missed so we'll need to validate and download the image + try { + // Rate limit outbound fetches to make this mechanism less useful for + // scanning networks and ports. + PhabricatorSystemActionEngine::willTakeAction( + array($viewer->getPHID()), + new PhabricatorFilesOutboundRequestAction(), + 1); + + $file = PhabricatorFile::newFromFileDownload( + $uri, + array( + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + 'canCDN' => true, + )); + if (!$file->isViewableImage()) { + $mime_type = $file->getMimeType(); + $engine = new PhabricatorDestructionEngine(); + $engine->destroyObject($file); + $file = null; + throw new Exception( + pht( + 'The URI "%s" does not correspond to a valid image file, got '. + 'a file with MIME type "%s". You must specify the URI of a '. + 'valid image file.', + $uri, + $mime_type)); + } else { + $file->save(); + } + + $external_request->setIsSuccessful(true) + ->setFilePHID($file->getPHID()) + ->save(); + unset($unguarded); + return $this->getExternalResponse($external_request); + } catch (HTTPFutureHTTPResponseStatus $status) { + $external_request->setIsSuccessful(false) + ->setResponseMessage($status->getMessage()) + ->save(); + return $this->getExternalResponse($external_request); + } catch (Exception $ex) { + // Not actually saving the request in this case + $external_request->setResponseMessage($ex->getMessage()); + return $this->getExternalResponse($external_request); + } + } + + private function getExternalResponse( + PhabricatorFileExternalRequest $request) { + if ($request->getIsSuccessful()) { + $file = id(new PhabricatorFileQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($request->getFilePHID())) + ->executeOne(); + if (!file) { + throw new Exception(pht( + 'The underlying file does not exist, but the cached request was '. + 'successful. This likely means the file record was manually deleted '. + 'by an administrator.')); + } + return id(new AphrontRedirectResponse()) + ->setIsExternal(true) + ->setURI($file->getViewURI()); + } else { + throw new Exception(pht( + "The request to get the external file from '%s' was unsuccessful:\n %s", + $request->getURI(), + $request->getResponseMessage())); + } + } +} diff --git a/src/applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php b/src/applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php new file mode 100644 --- /dev/null +++ b/src/applications/files/garbagecollector/PhabricatorFileExternalRequestGarbageCollector.php @@ -0,0 +1,28 @@ +loadAllWhere( + 'ttl < %d LIMIT 100', + PhabricatorTime::getNow()); + $engine = new PhabricatorDestructionEngine(); + foreach ($file_requests as $request) { + $engine->destroyObject($request); + } + + return (count($file_requests) == 100); + } + +} diff --git a/src/applications/files/storage/PhabricatorFileExternalRequest.php b/src/applications/files/storage/PhabricatorFileExternalRequest.php new file mode 100644 --- /dev/null +++ b/src/applications/files/storage/PhabricatorFileExternalRequest.php @@ -0,0 +1,63 @@ + array( + 'uri' => 'text', + 'uriIndex' => 'bytes12', + 'ttl' => 'epoch', + 'filePHID' => 'phid?', + 'isSuccessful' => 'bool', + 'responseMessage' => 'text?', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_uriindex' => array( + 'columns' => array('uriIndex'), + 'unique' => true, + ), + 'key_ttl' => array( + 'columns' => array('ttl'), + ), + 'key_file' => array( + 'columns' => array('filePHID'), + ), + ), + ) + parent::getConfiguration(); + } + + public function save() { + $hash = PhabricatorHash::digestForIndex($this->getURI()); + $this->setURIIndex($hash); + return parent::save(); + } + +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + + $file_phid = $this->getFilePHID(); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($engine->getViewer()) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + $engine->destroyObject($file); + } + } + $this->delete(); + } + +}