diff --git a/resources/sql/autopatches/20160317.lfs.01.ref.sql b/resources/sql/autopatches/20160317.lfs.01.ref.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160317.lfs.01.ref.sql @@ -0,0 +1,11 @@ +CREATE TABLE {$NAMESPACE}_repository.repository_gitlfsref ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + repositoryPHID VARBINARY(64) NOT NULL, + objectHash BINARY(64) NOT NULL, + byteSize BIGINT UNSIGNED NOT NULL, + authorPHID VARBINARY(64) NOT NULL, + filePHID VARBINARY(64) NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_hash` (repositoryPHID, objectHash) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3100,6 +3100,8 @@ 'PhabricatorRepositoryEngine' => 'applications/repository/engine/PhabricatorRepositoryEngine.php', 'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php', 'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php', + 'PhabricatorRepositoryGitLFSRef' => 'applications/repository/storage/PhabricatorRepositoryGitLFSRef.php', + 'PhabricatorRepositoryGitLFSRefQuery' => 'applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php', 'PhabricatorRepositoryGraphCache' => 'applications/repository/graphcache/PhabricatorRepositoryGraphCache.php', 'PhabricatorRepositoryGraphStream' => 'applications/repository/daemon/PhabricatorRepositoryGraphStream.php', 'PhabricatorRepositoryManagementCacheWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementCacheWorkflow.php', @@ -7665,6 +7667,11 @@ 'PhabricatorRepositoryEngine' => 'Phobject', 'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', 'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker', + 'PhabricatorRepositoryGitLFSRef' => array( + 'PhabricatorRepositoryDAO', + 'PhabricatorPolicyInterface', + ), + 'PhabricatorRepositoryGitLFSRefQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorRepositoryGraphCache' => 'Phobject', 'PhabricatorRepositoryGraphStream' => 'Phobject', 'PhabricatorRepositoryManagementCacheWorkflow' => 'PhabricatorRepositoryManagementWorkflow', diff --git a/src/applications/diffusion/controller/DiffusionServeController.php b/src/applications/diffusion/controller/DiffusionServeController.php --- a/src/applications/diffusion/controller/DiffusionServeController.php +++ b/src/applications/diffusion/controller/DiffusionServeController.php @@ -9,6 +9,8 @@ private $gitLFSToken; public function setServiceViewer(PhabricatorUser $viewer) { + $this->getRequest()->setUser($viewer); + $this->serviceViewer = $viewer; return $this; } @@ -42,6 +44,7 @@ $content_type = $request->getHTTPHeader('Content-Type'); $user_agent = idx($_SERVER, 'HTTP_USER_AGENT'); + $request_type = $request->getHTTPHeader('X-Phabricator-Request-Type'); // This may have a "charset" suffix, so only match the prefix. $lfs_pattern = '(^application/vnd\\.git-lfs\\+json(;|\z))'; @@ -64,6 +67,10 @@ // This is a Git LFS HTTP API request. $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; $this->isGitLFSRequest = true; + } else if ($request_type == 'git-lfs') { + // This is a Git LFS object content request. + $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; + $this->isGitLFSRequest = true; } else if ($request->getExists('cmd')) { // Mercurial also sends an Accept header like // "application/mercurial-0.1", and a User-Agent like @@ -884,12 +891,140 @@ } $path = $this->getGitLFSRequestPath($repository); + if ($path == 'objects/batch') { + return $this->serveGitLFSBatchRequest($repository, $viewer); + } else { + return DiffusionGitLFSResponse::newErrorResponse( + 404, + pht( + 'Git LFS operation "%s" is not supported by this server.', + $path)); + } + } + + private function serveGitLFSBatchRequest( + PhabricatorRepository $repository, + PhabricatorUser $viewer) { + + $input = PhabricatorStartup::getRawInput(); + $input = phutil_json_decode($input); + + $operation = idx($input, 'operation'); + switch ($operation) { + case 'upload': + $want_upload = true; + break; + case 'download': + $want_upload = false; + break; + default: + return DiffusionGitLFSResponse::newErrorResponse( + 404, + pht( + 'Git LFS batch operation "%s" is not supported by this server.', + $operation)); + } + + $objects = idx($input, 'objects', array()); + + $hashes = array(); + foreach ($objects as $object) { + $hashes[] = idx($object, 'oid'); + } + + if ($hashes) { + $refs = id(new PhabricatorRepositoryGitLFSRefQuery()) + ->setViewer($viewer) + ->withRepositoryPHIDs(array($repository->getPHID())) + ->withObjectHashes($hashes) + ->execute(); + $refs = mpull($refs, null, 'getObjectHash'); + } else { + $refs = array(); + } + + $file_phids = mpull($refs, 'getFilePHID'); + if ($file_phids) { + $files = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phids)) + ->execute(); + $files = mpull($files, null, 'getPHID'); + } else { + $files = array(); + } + + $authorization = null; + $output = array(); + foreach ($objects as $object) { + $oid = idx($object, 'oid'); + $size = idx($object, 'size'); + $ref = idx($refs, $oid); + + // NOTE: If we already have a ref for this object, we only emit a + // "download" action. The client should not upload the file again. + + $actions = array(); + if ($ref) { + $file = idx($files, $ref->getFilePHID()); + if ($file) { + $get_uri = $file->getCDNURIWithToken(); + $actions['download'] = array( + 'href' => $get_uri, + ); + } + } else if ($want_upload) { + if (!$authorization) { + // Here, we could reuse the existing authorization if we have one, + // but it's a little simpler to just generate a new one + // unconditionally. + $authorization = $this->newGitLFSHTTPAuthorization( + $repository, + $viewer, + $operation); + } + + $put_uri = $repository->getGitLFSURI("info/lfs/upload/{$oid}"); + + $actions['upload'] = array( + 'href' => $put_uri, + 'header' => array( + 'Authorization' => $authorization, + 'X-Phabricator-Request-Type' => 'git-lfs', + ), + ); + } + + $output[] = array( + 'oid' => $oid, + 'size' => $size, + 'actions' => $actions, + ); + } + + $output = array( + 'objects' => $output, + ); + + return id(new DiffusionGitLFSResponse()) + ->setContent($output); + } + + private function newGitLFSHTTPAuthorization( + PhabricatorRepository $repository, + PhabricatorUser $viewer, + $operation) { + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + + $authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization( + $repository, + $viewer, + $operation); + + unset($unguarded); - return DiffusionGitLFSResponse::newErrorResponse( - 404, - pht( - 'Git LFS operation "%s" is not supported by this server.', - $path)); + return $authorization; } private function getGitLFSRequestPath(PhabricatorRepository $repository) { diff --git a/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php b/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php --- a/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php +++ b/src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php @@ -83,25 +83,15 @@ // on this host, and does not require the user to have a VCS password. $user = $this->getUser(); - $headers = array(); - $lfs_user = DiffusionGitLFSTemporaryTokenType::HTTP_USERNAME; - $lfs_pass = Filesystem::readRandomCharacters(32); - $lfs_hash = PhabricatorHash::digest($lfs_pass); + $authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization( + $repository, + $user, + $operation); - $ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds'); - - $token = id(new PhabricatorAuthTemporaryToken()) - ->setTokenResource($repository->getPHID()) - ->setTokenType(DiffusionGitLFSTemporaryTokenType::TOKENTYPE) - ->setTokenCode($lfs_hash) - ->setUserPHID($user->getPHID()) - ->setTemporaryTokenProperty('lfs.operation', $operation) - ->setTokenExpires($ttl) - ->save(); - - $authorization_header = base64_encode($lfs_user.':'.$lfs_pass); - $headers['Authorization'] = 'Basic '.$authorization_header; + $headers = array( + 'authorization' => $authorization, + ); $result = array( 'header' => $headers, diff --git a/src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php b/src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php --- a/src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php +++ b/src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php @@ -15,4 +15,28 @@ return pht('Git LFS Token'); } + public static function newHTTPAuthorization( + PhabricatorRepository $repository, + PhabricatorUser $viewer, + $operation) { + + $lfs_user = self::HTTP_USERNAME; + $lfs_pass = Filesystem::readRandomCharacters(32); + $lfs_hash = PhabricatorHash::digest($lfs_pass); + + $ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds'); + + $token = id(new PhabricatorAuthTemporaryToken()) + ->setTokenResource($repository->getPHID()) + ->setTokenType(self::TOKENTYPE) + ->setTokenCode($lfs_hash) + ->setUserPHID($viewer->getPHID()) + ->setTemporaryTokenProperty('lfs.operation', $operation) + ->setTokenExpires($ttl) + ->save(); + + $authorization_header = base64_encode($lfs_user.':'.$lfs_pass); + return 'Basic '.$authorization_header; + } + } diff --git a/src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php b/src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php new file mode 100644 --- /dev/null +++ b/src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php @@ -0,0 +1,64 @@ +ids = $ids; + return $this; + } + + public function withRepositoryPHIDs(array $phids) { + $this->repositoryPHIDs = $phids; + return $this; + } + + public function withObjectHashes(array $hashes) { + $this->objectHashes = $hashes; + return $this; + } + + public function newResultObject() { + return new PhabricatorRepositoryGitLFSRef(); + } + + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); + + if ($this->ids !== null) { + $where[] = qsprintf( + $conn, + 'id IN (%Ld)', + $this->ids); + } + + if ($this->repositoryPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'repositoryPHID IN (%Ls)', + $this->repositoryPHIDs); + } + + if ($this->objectHashes !== null) { + $where[] = qsprintf( + $conn, + 'objectHash IN (%Ls)', + $this->objectHashes); + } + + return $where; + } + + public function getQueryApplicationClass() { + return 'PhabricatorDiffusionApplication'; + } + +} diff --git a/src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php b/src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php new file mode 100644 --- /dev/null +++ b/src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php @@ -0,0 +1,51 @@ + array( + 'objectHash' => 'bytes64', + 'byteSize' => 'uint64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_hash' => array( + 'columns' => array('repositoryPHID', 'objectHash'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + +/* -( PhabricatorPolicyInterface )----------------------------------------- */ + + + public function getCapabilities() { + return array( + PhabricatorPolicyCapability::CAN_VIEW, + ); + } + + public function getPolicy($capability) { + return PhabricatorPolicies::getMostOpenPolicy(); + } + + public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { + return false; + } + + public function describeAutomaticCapability($capability) { + return null; + } + + +}