diff --git a/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php b/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php index 9b6f4eee63..b450fd09d3 100644 --- a/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryCreateController.php @@ -1,903 +1,903 @@ getUser(); $this->edit = $request->getURIData('edit'); // NOTE: We can end up here via either "Create Repository", or via // "Import Repository", or via "Edit Remote", or via "Edit Policies". In // the latter two cases, we show only a few of the pages. $repository = null; $service = null; switch ($this->edit) { case 'remote': case 'policy': $repository = $this->getDiffusionRequest()->getRepository(); // Make sure we have CAN_EDIT. PhabricatorPolicyFilter::requireCapability( $viewer, $repository, PhabricatorPolicyCapability::CAN_EDIT); $this->setRepository($repository); $cancel_uri = $this->getRepositoryControllerURI($repository, 'edit/'); break; case 'import': case 'create': $this->requireApplicationCapability( DiffusionCreateRepositoriesCapability::CAPABILITY); // Pick a random open service to allocate this repository on, if any // exist. If there are no services, we aren't in cluster mode and // will allocate locally. If there are services but none permit // allocations, we fail. $services = id(new AlmanacServiceQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withServiceClasses( array( 'AlmanacClusterRepositoryServiceType', )) ->execute(); if ($services) { // Filter out services which do not permit new allocations. foreach ($services as $key => $possible_service) { if ($possible_service->getAlmanacPropertyValue('closed')) { unset($services[$key]); } } if (!$services) { throw new Exception( pht( 'This install is configured in cluster mode, but all '. 'available repository cluster services are closed to new '. 'allocations. At least one service must be open to allow '. 'new allocations to take place.')); } shuffle($services); $service = head($services); } $cancel_uri = $this->getApplicationURI('new/'); break; default: throw new Exception(pht('Invalid edit operation!')); } $form = id(new PHUIPagedFormView()) ->setUser($viewer) ->setCancelURI($cancel_uri); switch ($this->edit) { case 'remote': $title = pht('Edit Remote'); $form ->addPage('remote-uri', $this->buildRemoteURIPage()) ->addPage('auth', $this->buildAuthPage()); break; case 'policy': $title = pht('Edit Policies'); $form ->addPage('policy', $this->buildPolicyPage()); break; case 'create': $title = pht('Create Repository'); $form ->addPage('vcs', $this->buildVCSPage()) ->addPage('name', $this->buildNamePage()) ->addPage('policy', $this->buildPolicyPage()) ->addPage('done', $this->buildDonePage()); break; case 'import': $title = pht('Import Repository'); $form ->addPage('vcs', $this->buildVCSPage()) ->addPage('name', $this->buildNamePage()) ->addPage('remote-uri', $this->buildRemoteURIPage()) ->addPage('auth', $this->buildAuthPage()) ->addPage('policy', $this->buildPolicyPage()) ->addPage('done', $this->buildDonePage()); break; } if ($request->isFormPost()) { $form->readFromRequest($request); if ($form->isComplete()) { $is_create = ($this->edit === 'import' || $this->edit === 'create'); $is_auth = ($this->edit == 'import' || $this->edit == 'remote'); $is_policy = ($this->edit != 'remote'); $is_init = ($this->edit == 'create'); if ($is_create) { $repository = PhabricatorRepository::initializeNewRepository( $viewer); } $template = id(new PhabricatorRepositoryTransaction()); $type_name = PhabricatorRepositoryTransaction::TYPE_NAME; $type_vcs = PhabricatorRepositoryTransaction::TYPE_VCS; $type_activate = PhabricatorRepositoryTransaction::TYPE_ACTIVATE; $type_local_path = PhabricatorRepositoryTransaction::TYPE_LOCAL_PATH; $type_remote_uri = PhabricatorRepositoryTransaction::TYPE_REMOTE_URI; $type_hosting = PhabricatorRepositoryTransaction::TYPE_HOSTING; $type_http = PhabricatorRepositoryTransaction::TYPE_PROTOCOL_HTTP; $type_ssh = PhabricatorRepositoryTransaction::TYPE_PROTOCOL_SSH; $type_credential = PhabricatorRepositoryTransaction::TYPE_CREDENTIAL; $type_view = PhabricatorTransactions::TYPE_VIEW_POLICY; $type_edit = PhabricatorTransactions::TYPE_EDIT_POLICY; $type_push = PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY; $type_service = PhabricatorRepositoryTransaction::TYPE_SERVICE; $xactions = array(); // If we're creating a new repository, set all this core stuff. if ($is_create) { $callsign = $form->getPage('name') ->getControl('callsign')->getValue(); // We must set this to a unique value to save the repository // initially, and it's immutable, so we don't bother using // transactions to apply this change. $repository->setCallsign($callsign); $xactions[] = id(clone $template) ->setTransactionType($type_name) ->setNewValue( $form->getPage('name')->getControl('name')->getValue()); $xactions[] = id(clone $template) ->setTransactionType($type_vcs) ->setNewValue( $form->getPage('vcs')->getControl('vcs')->getValue()); $activate = $form->getPage('done') ->getControl('activate')->getValue(); $xactions[] = id(clone $template) ->setTransactionType($type_activate) ->setNewValue(($activate == 'start')); if ($service) { $xactions[] = id(clone $template) ->setTransactionType($type_service) ->setNewValue($service->getPHID()); } $default_local_path = PhabricatorEnv::getEnvConfig( 'repository.default-local-path'); $default_local_path = rtrim($default_local_path, '/'); $default_local_path = $default_local_path.'/'.$callsign.'/'; $xactions[] = id(clone $template) ->setTransactionType($type_local_path) ->setNewValue($default_local_path); } if ($is_init) { $xactions[] = id(clone $template) ->setTransactionType($type_hosting) ->setNewValue(true); $vcs = $form->getPage('vcs')->getControl('vcs')->getValue(); if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) { if (PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth')) { $v_http_mode = PhabricatorRepository::SERVE_READWRITE; } else { $v_http_mode = PhabricatorRepository::SERVE_OFF; } $xactions[] = id(clone $template) ->setTransactionType($type_http) ->setNewValue($v_http_mode); } if (PhabricatorEnv::getEnvConfig('diffusion.ssh-user')) { $v_ssh_mode = PhabricatorRepository::SERVE_READWRITE; } else { $v_ssh_mode = PhabricatorRepository::SERVE_OFF; } $xactions[] = id(clone $template) ->setTransactionType($type_ssh) ->setNewValue($v_ssh_mode); } if ($is_auth) { $xactions[] = id(clone $template) ->setTransactionType($type_remote_uri) ->setNewValue( $form->getPage('remote-uri')->getControl('remoteURI') ->getValue()); $xactions[] = id(clone $template) ->setTransactionType($type_credential) ->setNewValue( $form->getPage('auth')->getControl('credential')->getValue()); } if ($is_policy) { $xactions[] = id(clone $template) ->setTransactionType($type_view) ->setNewValue( $form->getPage('policy')->getControl('viewPolicy')->getValue()); $xactions[] = id(clone $template) ->setTransactionType($type_edit) ->setNewValue( $form->getPage('policy')->getControl('editPolicy')->getValue()); if ($is_init || $repository->isHosted()) { $xactions[] = id(clone $template) ->setTransactionType($type_push) ->setNewValue( $form->getPage('policy')->getControl('pushPolicy')->getValue()); } } id(new PhabricatorRepositoryEditor()) ->setContinueOnNoEffect(true) ->setContentSourceFromRequest($request) ->setActor($viewer) ->applyTransactions($repository, $xactions); $repo_uri = $this->getRepositoryControllerURI($repository, 'edit/'); return id(new AphrontRedirectResponse())->setURI($repo_uri); } } else { $dict = array(); if ($repository) { $dict = array( 'remoteURI' => $repository->getRemoteURI(), 'credential' => $repository->getCredentialPHID(), 'viewPolicy' => $repository->getViewPolicy(), 'editPolicy' => $repository->getEditPolicy(), 'pushPolicy' => $repository->getPushPolicy(), ); } $form->readFromObject($dict); } $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb($title); return $this->buildApplicationPage( array( $crumbs, $form, ), array( 'title' => $title, )); } /* -( Page: VCS Type )----------------------------------------------------- */ private function buildVCSPage() { $is_import = ($this->edit == 'import'); if ($is_import) { $git_str = pht( 'Import a Git repository (for example, a repository hosted '. 'on GitHub).'); $hg_str = pht( 'Import a Mercurial repository (for example, a repository '. 'hosted on Bitbucket).'); $svn_str = pht('Import a Subversion repository.'); } else { $git_str = pht('Create a new, empty Git repository.'); $hg_str = pht('Create a new, empty Mercurial repository.'); $svn_str = pht('Create a new, empty Subversion repository.'); } $control = id(new AphrontFormRadioButtonControl()) ->setName('vcs') ->setLabel(pht('Type')) ->addButton( PhabricatorRepositoryType::REPOSITORY_TYPE_GIT, pht('Git'), $git_str) ->addButton( PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL, pht('Mercurial'), $hg_str) ->addButton( PhabricatorRepositoryType::REPOSITORY_TYPE_SVN, pht('Subversion'), $svn_str); if ($is_import) { $control->addButton( PhabricatorRepositoryType::REPOSITORY_TYPE_PERFORCE, pht('Perforce'), pht( 'Perforce is not directly supported, but you can import '. 'a Perforce repository as a Git repository using %s.', phutil_tag( 'a', array( 'href' => 'http://www.perforce.com/product/components/git-fusion', 'target' => '_blank', ), pht('Perforce Git Fusion'))), 'disabled', $disabled = true); } return id(new PHUIFormPageView()) ->setPageName(pht('Repository Type')) ->setUser($this->getRequest()->getUser()) ->setValidateFormPageCallback(array($this, 'validateVCSPage')) ->addControl($control); } public function validateVCSPage(PHUIFormPageView $page) { $valid = array( PhabricatorRepositoryType::REPOSITORY_TYPE_GIT => true, PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL => true, PhabricatorRepositoryType::REPOSITORY_TYPE_SVN => true, ); $c_vcs = $page->getControl('vcs'); $v_vcs = $c_vcs->getValue(); if (!$v_vcs) { $c_vcs->setError(pht('Required')); $page->addPageError( pht('You must select a version control system.')); } else if (empty($valid[$v_vcs])) { $c_vcs->setError(pht('Invalid')); $page->addPageError( pht('You must select a valid version control system.')); } return $c_vcs->isValid(); } /* -( Page: Name and Callsign )-------------------------------------------- */ private function buildNamePage() { return id(new PHUIFormPageView()) ->setUser($this->getRequest()->getUser()) ->setPageName(pht('Repository Name and Location')) ->setValidateFormPageCallback(array($this, 'validateNamePage')) ->addRemarkupInstructions( pht( '**Choose a human-readable name for this repository**, like '. '"CompanyName Mobile App" or "CompanyName Backend Server". You '. 'can change this later.')) ->addControl( id(new AphrontFormTextControl()) ->setName('name') ->setLabel(pht('Name')) ->setCaption(pht('Human-readable repository name.'))) ->addRemarkupInstructions( pht( '**Choose a "Callsign" for the repository.** This is a short, '. 'unique string which identifies commits elsewhere in Phabricator. '. 'For example, you might use `M` for your mobile app repository '. 'and `B` for your backend repository.'. "\n\n". '**Callsigns must be UPPERCASE**, and can not be edited after the '. 'repository is created. Generally, you should choose short '. 'callsigns.')) ->addControl( id(new AphrontFormTextControl()) ->setName('callsign') ->setLabel(pht('Callsign')) ->setCaption(pht('Short UPPERCASE identifier.'))); } public function validateNamePage(PHUIFormPageView $page) { $c_name = $page->getControl('name'); $v_name = $c_name->getValue(); if (!strlen($v_name)) { $c_name->setError(pht('Required')); $page->addPageError( pht('You must choose a name for this repository.')); } $c_call = $page->getControl('callsign'); $v_call = $c_call->getValue(); if (!strlen($v_call)) { $c_call->setError(pht('Required')); $page->addPageError( pht('You must choose a callsign for this repository.')); } else if (!preg_match('/^[A-Z]+\z/', $v_call)) { $c_call->setError(pht('Invalid')); $page->addPageError( pht('The callsign must contain only UPPERCASE letters.')); } else { $exists = false; try { $repo = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getRequest()->getUser()) ->withCallsigns(array($v_call)) ->executeOne(); $exists = (bool)$repo; } catch (PhabricatorPolicyException $ex) { $exists = true; } if ($exists) { $c_call->setError(pht('Not Unique')); $page->addPageError( pht( 'Another repository already uses that callsign. You must choose '. 'a unique callsign.')); } } return $c_name->isValid() && $c_call->isValid(); } /* -( Page: Remote URI )--------------------------------------------------- */ private function buildRemoteURIPage() { return id(new PHUIFormPageView()) ->setUser($this->getRequest()->getUser()) ->setPageName(pht('Repository Remote URI')) ->setValidateFormPageCallback(array($this, 'validateRemoteURIPage')) ->setAdjustFormPageCallback(array($this, 'adjustRemoteURIPage')) ->addControl( id(new AphrontFormTextControl()) ->setName('remoteURI')); } public function adjustRemoteURIPage(PHUIFormPageView $page) { $form = $page->getForm(); $is_git = false; $is_svn = false; $is_mercurial = false; if ($this->getRepository()) { $vcs = $this->getRepository()->getVersionControlSystem(); } else { $vcs = $form->getPage('vcs')->getControl('vcs')->getValue(); } switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $is_git = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $is_svn = true; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $is_mercurial = true; break; default: throw new Exception(pht('Unsupported VCS!')); } $has_local = ($is_git || $is_mercurial); if ($is_git) { $uri_label = pht('Remote URI'); $instructions = pht( 'Enter the URI to clone this Git repository from. It should usually '. 'look like one of these examples:'. "\n\n". "| Example Git Remote URIs |\n". "| ----------------------- |\n". "| `git@github.com:example/example.git` |\n". "| `ssh://user@host.com/git/example.git` |\n". "| `https://example.com/repository.git` |\n"); } else if ($is_mercurial) { $uri_label = pht('Remote URI'); $instructions = pht( 'Enter the URI to clone this Mercurial repository from. It should '. 'usually look like one of these examples:'. "\n\n". "| Example Mercurial Remote URIs |\n". "| ----------------------- |\n". "| `ssh://hg@bitbucket.org/example/repository` |\n". "| `https://bitbucket.org/example/repository` |\n"); } else if ($is_svn) { $uri_label = pht('Repository Root'); $instructions = pht( 'Enter the **Repository Root** for this Subversion repository. '. 'You can figure this out by running `svn info` in a working copy '. 'and looking at the value in the `Repository Root` field. It '. 'should be a URI and will usually look like these:'. "\n\n". "| Example Subversion Repository Root URIs |\n". "| ------------------------------ |\n". "| `http://svn.example.org/svnroot/` |\n". "| `svn+ssh://svn.example.com/svnroot/` |\n". "| `svn://svn.example.net/svnroot/` |\n". "\n\n". "You **MUST** specify the root of the repository, not a ". "subdirectory. (If you want to import only part of a Subversion ". "repository, use the //Import Only// option at the end of this ". "workflow.)"); } else { throw new Exception(pht('Unsupported VCS!')); } $page->addRemarkupInstructions($instructions, 'remoteURI'); $page->getControl('remoteURI')->setLabel($uri_label); } public function validateRemoteURIPage(PHUIFormPageView $page) { $c_remote = $page->getControl('remoteURI'); $v_remote = $c_remote->getValue(); if (!strlen($v_remote)) { $c_remote->setError(pht('Required')); $page->addPageError( pht('You must specify a URI.')); } else { try { PhabricatorRepository::assertValidRemoteURI($v_remote); } catch (Exception $ex) { $c_remote->setError(pht('Invalid')); $page->addPageError($ex->getMessage()); } } return $c_remote->isValid(); } /* -( Page: Authentication )----------------------------------------------- */ public function buildAuthPage() { return id(new PHUIFormPageView()) ->setPageName(pht('Authentication')) ->setUser($this->getRequest()->getUser()) ->setAdjustFormPageCallback(array($this, 'adjustAuthPage')) ->addControl( id(new PassphraseCredentialControl()) ->setName('credential')); } public function adjustAuthPage($page) { $form = $page->getForm(); if ($this->getRepository()) { $vcs = $this->getRepository()->getVersionControlSystem(); } else { $vcs = $form->getPage('vcs')->getControl('vcs')->getValue(); } $remote_uri = $form->getPage('remote-uri') ->getControl('remoteURI') ->getValue(); $proto = PhabricatorRepository::getRemoteURIProtocol($remote_uri); $remote_user = $this->getRemoteURIUser($remote_uri); $c_credential = $page->getControl('credential'); $c_credential->setDefaultUsername($remote_user); if ($this->isSSHProtocol($proto)) { $c_credential->setLabel(pht('SSH Key')); $c_credential->setCredentialType( PassphraseSSHPrivateKeyTextCredentialType::CREDENTIAL_TYPE); $provides_type = PassphraseSSHPrivateKeyCredentialType::PROVIDES_TYPE; $page->addRemarkupInstructions( pht( - 'Choose or add the SSH credentials to use to connect to the the '. + 'Choose or add the SSH credentials to use to connect to the '. 'repository hosted at:'. "\n\n". " lang=text\n". " %s", $remote_uri), 'credential'); } else if ($this->isUsernamePasswordProtocol($proto)) { $c_credential->setLabel(pht('Password')); $c_credential->setAllowNull(true); $c_credential->setCredentialType( PassphrasePasswordCredentialType::CREDENTIAL_TYPE); $provides_type = PassphrasePasswordCredentialType::PROVIDES_TYPE; $page->addRemarkupInstructions( pht( 'Choose the username and password used to connect to the '. 'repository hosted at:'. "\n\n". " lang=text\n". " %s". "\n\n". "If this repository does not require a username or password, ". "you can continue to the next step.", $remote_uri), 'credential'); } else { throw new Exception(pht('Unknown URI protocol!')); } if ($provides_type) { $viewer = $this->getRequest()->getUser(); $options = id(new PassphraseCredentialQuery()) ->setViewer($viewer) ->withIsDestroyed(false) ->withProvidesTypes(array($provides_type)) ->execute(); $c_credential->setOptions($options); } } public function validateAuthPage(PHUIFormPageView $page) { $form = $page->getForm(); $remote_uri = $form->getPage('remote')->getControl('remoteURI')->getValue(); $proto = $this->getRemoteURIProtocol($remote_uri); $c_credential = $page->getControl('credential'); $v_credential = $c_credential->getValue(); // NOTE: We're using the omnipotent user here because the viewer might be // editing a repository they're allowed to edit which uses a credential they // are not allowed to see. This is fine, as long as they don't change it. $credential = id(new PassphraseCredentialQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($v_credential)) ->executeOne(); if ($this->isSSHProtocol($proto)) { if (!$credential) { $c_credential->setError(pht('Required')); $page->addPageError( pht('You must choose an SSH credential to connect over SSH.')); } $ssh_type = PassphraseSSHPrivateKeyCredentialType::PROVIDES_TYPE; if ($credential->getProvidesType() !== $ssh_type) { $c_credential->setError(pht('Invalid')); $page->addPageError( pht( 'You must choose an SSH credential, not some other type '. 'of credential.')); } } else if ($this->isUsernamePasswordProtocol($proto)) { if ($credential) { $password_type = PassphrasePasswordCredentialType::PROVIDES_TYPE; if ($credential->getProvidesType() !== $password_type) { $c_credential->setError(pht('Invalid')); $page->addPageError( pht( 'You must choose a username/password credential, not some other '. 'type of credential.')); } } return $c_credential->isValid(); } else { return true; } } /* -( Page: Policy )------------------------------------------------------- */ private function buildPolicyPage() { $viewer = $this->getRequest()->getUser(); if ($this->getRepository()) { $repository = $this->getRepository(); } else { $repository = PhabricatorRepository::initializeNewRepository($viewer); } $policies = id(new PhabricatorPolicyQuery()) ->setViewer($viewer) ->setObject($repository) ->execute(); $view_policy = id(new AphrontFormPolicyControl()) ->setUser($viewer) ->setCapability(PhabricatorPolicyCapability::CAN_VIEW) ->setPolicyObject($repository) ->setPolicies($policies) ->setName('viewPolicy'); $edit_policy = id(new AphrontFormPolicyControl()) ->setUser($viewer) ->setCapability(PhabricatorPolicyCapability::CAN_EDIT) ->setPolicyObject($repository) ->setPolicies($policies) ->setName('editPolicy'); $push_policy = id(new AphrontFormPolicyControl()) ->setUser($viewer) ->setCapability(DiffusionPushCapability::CAPABILITY) ->setPolicyObject($repository) ->setPolicies($policies) ->setName('pushPolicy'); return id(new PHUIFormPageView()) ->setPageName(pht('Policies')) ->setValidateFormPageCallback(array($this, 'validatePolicyPage')) ->setAdjustFormPageCallback(array($this, 'adjustPolicyPage')) ->setUser($viewer) ->addRemarkupInstructions( pht('Select access policies for this repository.')) ->addControl($view_policy) ->addControl($edit_policy) ->addControl($push_policy); } public function adjustPolicyPage(PHUIFormPageView $page) { if ($this->getRepository()) { $repository = $this->getRepository(); $show_push = $repository->isHosted(); } else { $show_push = ($this->edit == 'create'); } if (!$show_push) { $c_push = $page->getControl('pushPolicy'); $c_push->setHidden(true); } } public function validatePolicyPage(PHUIFormPageView $page) { $form = $page->getForm(); $viewer = $this->getRequest()->getUser(); $c_view = $page->getControl('viewPolicy'); $c_edit = $page->getControl('editPolicy'); $c_push = $page->getControl('pushPolicy'); $v_view = $c_view->getValue(); $v_edit = $c_edit->getValue(); $v_push = $c_push->getValue(); if ($this->getRepository()) { $repository = $this->getRepository(); } else { $repository = PhabricatorRepository::initializeNewRepository($viewer); } $proxy = clone $repository; $proxy->setViewPolicy($v_view); $proxy->setEditPolicy($v_edit); $can_view = PhabricatorPolicyFilter::hasCapability( $viewer, $proxy, PhabricatorPolicyCapability::CAN_VIEW); $can_edit = PhabricatorPolicyFilter::hasCapability( $viewer, $proxy, PhabricatorPolicyCapability::CAN_EDIT); if (!$can_view) { $c_view->setError(pht('Invalid')); $page->addPageError( pht( 'You can not use the selected policy, because you would be unable '. 'to see the repository.')); } if (!$can_edit) { $c_edit->setError(pht('Invalid')); $page->addPageError( pht( 'You can not use the selected edit policy, because you would be '. 'unable to edit the repository.')); } return $c_view->isValid() && $c_edit->isValid(); } /* -( Page: Done )--------------------------------------------------------- */ private function buildDonePage() { $is_create = ($this->edit == 'create'); if ($is_create) { $now_label = pht('Create Repository Now'); $now_caption = pht( 'Create the repository right away. This will create the repository '. 'using default settings.'); $wait_label = pht('Configure More Options First'); $wait_caption = pht( 'Configure more options before creating the repository. '. 'This will let you fine-tune settings. You can create the repository '. 'whenever you are ready.'); } else { $now_label = pht('Start Import Now'); $now_caption = pht( 'Start importing the repository right away. This will import '. 'the entire repository using default settings.'); $wait_label = pht('Configure More Options First'); $wait_caption = pht( 'Configure more options before beginning the repository '. 'import. This will let you fine-tune settings. You can '. 'start the import whenever you are ready.'); } return id(new PHUIFormPageView()) ->setPageName(pht('Repository Ready!')) ->setValidateFormPageCallback(array($this, 'validateDonePage')) ->setUser($this->getRequest()->getUser()) ->addControl( id(new AphrontFormRadioButtonControl()) ->setName('activate') ->setLabel(pht('Start Now')) ->addButton( 'start', $now_label, $now_caption) ->addButton( 'wait', $wait_label, $wait_caption)); } public function validateDonePage(PHUIFormPageView $page) { $c_activate = $page->getControl('activate'); $v_activate = $c_activate->getValue(); if ($v_activate != 'start' && $v_activate != 'wait') { $c_activate->setError(pht('Required')); $page->addPageError( pht('Make a choice about repository activation.')); } return $c_activate->isValid(); } /* -( Internal )----------------------------------------------------------- */ private function getRemoteURIUser($raw_uri) { $uri = new PhutilURI($raw_uri); if ($uri->getUser()) { return $uri->getUser(); } $git_uri = new PhutilGitURI($raw_uri); if (strlen($git_uri->getDomain()) && strlen($git_uri->getPath())) { return $git_uri->getUser(); } return null; } private function isSSHProtocol($proto) { return ($proto == 'git' || $proto == 'ssh' || $proto == 'svn+ssh'); } private function isUsernamePasswordProtocol($proto) { return ($proto == 'http' || $proto == 'https' || $proto == 'svn'); } private function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } private function getRepository() { return $this->repository; } } diff --git a/src/applications/diviner/query/DivinerAtomQuery.php b/src/applications/diviner/query/DivinerAtomQuery.php index c2a5247d78..5f4900d4dd 100644 --- a/src/applications/diviner/query/DivinerAtomQuery.php +++ b/src/applications/diviner/query/DivinerAtomQuery.php @@ -1,511 +1,511 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withBookPHIDs(array $phids) { $this->bookPHIDs = $phids; return $this; } public function withTypes(array $types) { $this->types = $types; return $this; } public function withNames(array $names) { $this->names = $names; return $this; } public function withContexts(array $contexts) { $this->contexts = $contexts; return $this; } public function withIndexes(array $indexes) { $this->indexes = $indexes; return $this; } public function withNodeHashes(array $hashes) { $this->nodeHashes = $hashes; return $this; } public function withTitles($titles) { $this->titles = $titles; return $this; } public function withNameContains($text) { $this->nameContains = $text; return $this; } public function needAtoms($need) { $this->needAtoms = $need; return $this; } public function needChildren($need) { $this->needChildren = $need; return $this; } /** * Include or exclude "ghosts", which are symbols which used to exist but do * not exist currently (for example, a function which existed in an older * version of the codebase but was deleted). * * These symbols had PHIDs assigned to them, and may have other sorts of * metadata that we don't want to lose (like comments or flags), so we don't * delete them outright. They might also come back in the future: the change * which deleted the symbol might be reverted, or the documentation might * have been generated incorrectly by accident. In these cases, we can * restore the original data. * * @param bool * @return this */ public function withGhosts($ghosts) { $this->isGhost = $ghosts; return $this; } public function needExtends($need) { $this->needExtends = $need; return $this; } public function withIsDocumentable($documentable) { $this->isDocumentable = $documentable; return $this; } public function withRepositoryPHIDs(array $repository_phids) { $this->repositoryPHIDs = $repository_phids; return $this; } public function needRepositories($need_repositories) { $this->needRepositories = $need_repositories; return $this; } protected function loadPage() { $table = new DivinerLiveSymbol(); $conn_r = $table->establishConnection('r'); $data = queryfx_all( $conn_r, 'SELECT * FROM %T %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r)); return $table->loadAllFromArray($data); } protected function willFilterPage(array $atoms) { assert_instances_of($atoms, 'DivinerLiveSymbol'); $books = array_unique(mpull($atoms, 'getBookPHID')); $books = id(new DivinerBookQuery()) ->setViewer($this->getViewer()) ->withPHIDs($books) ->execute(); $books = mpull($books, null, 'getPHID'); foreach ($atoms as $key => $atom) { $book = idx($books, $atom->getBookPHID()); if (!$book) { $this->didRejectResult($atom); unset($atoms[$key]); continue; } $atom->attachBook($book); } if ($this->needAtoms) { $atom_data = id(new DivinerLiveAtom())->loadAllWhere( 'symbolPHID IN (%Ls)', mpull($atoms, 'getPHID')); $atom_data = mpull($atom_data, null, 'getSymbolPHID'); foreach ($atoms as $key => $atom) { $data = idx($atom_data, $atom->getPHID()); $atom->attachAtom($data); } } // Load all of the symbols this symbol extends, recursively. Commonly, // this means all the ancestor classes and interfaces it extends and // implements. if ($this->needExtends) { // First, load all the matching symbols by name. This does 99% of the // work in most cases, assuming things are named at all reasonably. $names = array(); foreach ($atoms as $atom) { if (!$atom->getAtom()) { continue; } foreach ($atom->getAtom()->getExtends() as $xref) { $names[] = $xref->getName(); } } if ($names) { $xatoms = id(new DivinerAtomQuery()) ->setViewer($this->getViewer()) ->withNames($names) ->withGhosts(false) ->needExtends(true) ->needAtoms(true) ->needChildren($this->needChildren) ->execute(); $xatoms = mgroup($xatoms, 'getName', 'getType', 'getBookPHID'); } else { $xatoms = array(); } foreach ($atoms as $atom) { $atom_lang = null; $atom_extends = array(); if ($atom->getAtom()) { $atom_lang = $atom->getAtom()->getLanguage(); $atom_extends = $atom->getAtom()->getExtends(); } $extends = array(); foreach ($atom_extends as $xref) { // If there are no symbols of the matching name and type, we can't // resolve this. if (empty($xatoms[$xref->getName()][$xref->getType()])) { continue; } // If we found matches in the same documentation book, prefer them - // over other matches. Otherwise, look at all the the matches. + // over other matches. Otherwise, look at all the matches. $matches = $xatoms[$xref->getName()][$xref->getType()]; if (isset($matches[$atom->getBookPHID()])) { $maybe = $matches[$atom->getBookPHID()]; } else { $maybe = array_mergev($matches); } if (!$maybe) { continue; } // Filter out matches in a different language, since, e.g., PHP // classes can not implement JS classes. $same_lang = array(); foreach ($maybe as $xatom) { if ($xatom->getAtom()->getLanguage() == $atom_lang) { $same_lang[] = $xatom; } } if (!$same_lang) { continue; } // If we have duplicates remaining, just pick the first one. There's // nothing more we can do to figure out which is the real one. $extends[] = head($same_lang); } $atom->attachExtends($extends); } } if ($this->needChildren) { $child_hashes = $this->getAllChildHashes($atoms, $this->needExtends); if ($child_hashes) { $children = id(new DivinerAtomQuery()) ->setViewer($this->getViewer()) ->withNodeHashes($child_hashes) ->needAtoms($this->needAtoms) ->execute(); $children = mpull($children, null, 'getNodeHash'); } else { $children = array(); } $this->attachAllChildren($atoms, $children, $this->needExtends); } if ($this->needRepositories) { $repositories = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()) ->withPHIDs(mpull($atoms, 'getRepositoryPHID')) ->execute(); $repositories = mpull($repositories, null, 'getPHID'); foreach ($atoms as $key => $atom) { if ($atom->getRepositoryPHID() === null) { $atom->attachRepository(null); continue; } $repository = idx($repositories, $atom->getRepositoryPHID()); if (!$repository) { $this->didRejectResult($atom); unset($atom[$key]); continue; } $atom->attachRepository($repository); } } return $atoms; } protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->ids) { $where[] = qsprintf( $conn_r, 'id IN (%Ld)', $this->ids); } if ($this->phids) { $where[] = qsprintf( $conn_r, 'phid IN (%Ls)', $this->phids); } if ($this->bookPHIDs) { $where[] = qsprintf( $conn_r, 'bookPHID IN (%Ls)', $this->bookPHIDs); } if ($this->types) { $where[] = qsprintf( $conn_r, 'type IN (%Ls)', $this->types); } if ($this->names) { $where[] = qsprintf( $conn_r, 'name IN (%Ls)', $this->names); } if ($this->titles) { $hashes = array(); foreach ($this->titles as $title) { $slug = DivinerAtomRef::normalizeTitleString($title); $hash = PhabricatorHash::digestForIndex($slug); $hashes[] = $hash; } $where[] = qsprintf( $conn_r, 'titleSlugHash in (%Ls)', $hashes); } if ($this->contexts) { $with_null = false; $contexts = $this->contexts; foreach ($contexts as $key => $value) { if ($value === null) { unset($contexts[$key]); $with_null = true; continue; } } if ($contexts && $with_null) { $where[] = qsprintf( $conn_r, 'context IN (%Ls) OR context IS NULL', $contexts); } else if ($contexts) { $where[] = qsprintf( $conn_r, 'context IN (%Ls)', $contexts); } else if ($with_null) { $where[] = qsprintf( $conn_r, 'context IS NULL'); } } if ($this->indexes) { $where[] = qsprintf( $conn_r, 'atomIndex IN (%Ld)', $this->indexes); } if ($this->isDocumentable !== null) { $where[] = qsprintf( $conn_r, 'isDocumentable = %d', (int)$this->isDocumentable); } if ($this->isGhost !== null) { if ($this->isGhost) { $where[] = qsprintf($conn_r, 'graphHash IS NULL'); } else { $where[] = qsprintf($conn_r, 'graphHash IS NOT NULL'); } } if ($this->nodeHashes) { $where[] = qsprintf( $conn_r, 'nodeHash IN (%Ls)', $this->nodeHashes); } if ($this->nameContains) { // NOTE: This `CONVERT()` call makes queries case-insensitive, since // the column has binary collation. Eventually, this should move into // fulltext. $where[] = qsprintf( $conn_r, 'CONVERT(name USING utf8) LIKE %~', $this->nameContains); } if ($this->repositoryPHIDs) { $where[] = qsprintf( $conn_r, 'repositoryPHID IN (%Ls)', $this->repositoryPHIDs); } $where[] = $this->buildPagingClause($conn_r); return $this->formatWhereClause($where); } /** * Walk a list of atoms and collect all the node hashes of the atoms' * children. When recursing, also walk up the tree and collect children of * atoms they extend. * * @param list List of symbols to collect child hashes of. * @param bool True to collect children of extended atoms, * as well. * @return map Hashes of atoms' children. */ private function getAllChildHashes(array $symbols, $recurse_up) { assert_instances_of($symbols, 'DivinerLiveSymbol'); $hashes = array(); foreach ($symbols as $symbol) { $child_hashes = array(); if ($symbol->getAtom()) { $child_hashes = $symbol->getAtom()->getChildHashes(); } foreach ($child_hashes as $hash) { $hashes[$hash] = $hash; } if ($recurse_up) { $hashes += $this->getAllChildHashes($symbol->getExtends(), true); } } return $hashes; } /** * Attach child atoms to existing atoms. In recursive mode, also attach child * atoms to atoms that these atoms extend. * * @param list List of symbols to attach children to. * @param map Map of symbols, keyed by node hash. * @param bool True to attach children to extended atoms, as well. * @return void */ private function attachAllChildren( array $symbols, array $children, $recurse_up) { assert_instances_of($symbols, 'DivinerLiveSymbol'); assert_instances_of($children, 'DivinerLiveSymbol'); foreach ($symbols as $symbol) { $child_hashes = array(); $symbol_children = array(); if ($symbol->getAtom()) { $child_hashes = $symbol->getAtom()->getChildHashes(); } foreach ($child_hashes as $hash) { if (isset($children[$hash])) { $symbol_children[] = $children[$hash]; } } $symbol->attachChildren($symbol_children); if ($recurse_up) { $this->attachAllChildren($symbol->getExtends(), $children, true); } } } public function getQueryApplicationClass() { return 'PhabricatorDivinerApplication'; } } diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php index d60107ee6d..6a71c9aeac 100644 --- a/src/applications/files/storage/PhabricatorFile.php +++ b/src/applications/files/storage/PhabricatorFile.php @@ -1,1390 +1,1390 @@ setViewer(PhabricatorUser::getOmnipotentUser()) ->withClasses(array('PhabricatorFilesApplication')) ->executeOne(); $view_policy = $app->getPolicy( FilesDefaultViewCapability::CAPABILITY); return id(new PhabricatorFile()) ->setViewPolicy($view_policy) ->setIsPartial(0) ->attachOriginalFile(null) ->attachObjects(array()) ->attachObjectPHIDs(array()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'metadata' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'text255?', 'mimeType' => 'text255?', 'byteSize' => 'uint64', 'storageEngine' => 'text32', 'storageFormat' => 'text32', 'storageHandle' => 'text255', 'authorPHID' => 'phid?', 'secretKey' => 'bytes20?', 'contentHash' => 'bytes40?', 'ttl' => 'epoch?', 'isExplicitUpload' => 'bool?', 'mailKey' => 'bytes20', 'isPartial' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_phid' => null, 'phid' => array( 'columns' => array('phid'), 'unique' => true, ), 'authorPHID' => array( 'columns' => array('authorPHID'), ), 'contentHash' => array( 'columns' => array('contentHash'), ), 'key_ttl' => array( 'columns' => array('ttl'), ), 'key_dateCreated' => array( 'columns' => array('dateCreated'), ), 'key_partial' => array( 'columns' => array('authorPHID', 'isPartial'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorFileFilePHIDType::TYPECONST); } public function save() { if (!$this->getSecretKey()) { $this->setSecretKey($this->generateSecretKey()); } if (!$this->getMailKey()) { $this->setMailKey(Filesystem::readRandomCharacters(20)); } return parent::save(); } public function getMonogram() { return 'F'.$this->getID(); } public static function readUploadedFileData($spec) { if (!$spec) { throw new Exception(pht('No file was uploaded!')); } $err = idx($spec, 'error'); if ($err) { throw new PhabricatorFileUploadException($err); } $tmp_name = idx($spec, 'tmp_name'); $is_valid = @is_uploaded_file($tmp_name); if (!$is_valid) { throw new Exception(pht('File is not an uploaded file.')); } $file_data = Filesystem::readFile($tmp_name); $file_size = idx($spec, 'size'); if (strlen($file_data) != $file_size) { throw new Exception(pht('File size disagrees with uploaded size.')); } return $file_data; } public static function newFromPHPUpload($spec, array $params = array()) { $file_data = self::readUploadedFileData($spec); $file_name = nonempty( idx($params, 'name'), idx($spec, 'name')); $params = array( 'name' => $file_name, ) + $params; return self::newFromFileData($file_data, $params); } public static function newFromXHRUpload($data, array $params = array()) { return self::newFromFileData($data, $params); } /** * Given a block of data, try to load an existing file with the same content * if one exists. If it does not, build a new file. * * This method is generally used when we have some piece of semi-trusted data * like a diff or a file from a repository that we want to show to the user. * We can't just dump it out because it may be dangerous for any number of * reasons; instead, we need to serve it through the File abstraction so it * ends up on the CDN domain if one is configured and so on. However, if we * simply wrote a new file every time we'd potentially end up with a lot * of redundant data in file storage. * * To solve these problems, we use file storage as a cache and reuse the * same file again if we've previously written it. * * NOTE: This method unguards writes. * * @param string Raw file data. * @param dict Dictionary of file information. */ public static function buildFromFileDataOrHash( $data, array $params = array()) { $file = id(new PhabricatorFile())->loadOneWhere( 'name = %s AND contentHash = %s LIMIT 1', idx($params, 'name'), self::hashFileContent($data)); if (!$file) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $file = self::newFromFileData($data, $params); unset($unguarded); } return $file; } public static function newFileFromContentHash($hash, array $params) { // Check to see if a file with same contentHash exist $file = id(new PhabricatorFile())->loadOneWhere( 'contentHash = %s LIMIT 1', $hash); if ($file) { // copy storageEngine, storageHandle, storageFormat $copy_of_storage_engine = $file->getStorageEngine(); $copy_of_storage_handle = $file->getStorageHandle(); $copy_of_storage_format = $file->getStorageFormat(); $copy_of_byte_size = $file->getByteSize(); $copy_of_mime_type = $file->getMimeType(); $new_file = self::initializeNewFile(); $new_file->setByteSize($copy_of_byte_size); $new_file->setContentHash($hash); $new_file->setStorageEngine($copy_of_storage_engine); $new_file->setStorageHandle($copy_of_storage_handle); $new_file->setStorageFormat($copy_of_storage_format); $new_file->setMimeType($copy_of_mime_type); $new_file->copyDimensions($file); $new_file->readPropertiesFromParameters($params); $new_file->save(); return $new_file; } return $file; } public static function newChunkedFile( PhabricatorFileStorageEngine $engine, $length, array $params) { $file = self::initializeNewFile(); $file->setByteSize($length); // TODO: We might be able to test the first chunk in order to figure // this out more reliably, since MIME detection usually examines headers. // However, enormous files are probably always either actually raw data // or reasonable to treat like raw data. $file->setMimeType('application/octet-stream'); $chunked_hash = idx($params, 'chunkedHash'); if ($chunked_hash) { $file->setContentHash($chunked_hash); } else { // See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some // discussion of this. $seed = Filesystem::readRandomBytes(64); $hash = PhabricatorChunkedFileStorageEngine::getChunkedHashForInput( $seed); $file->setContentHash($hash); } $file->setStorageEngine($engine->getEngineIdentifier()); $file->setStorageHandle(PhabricatorFileChunk::newChunkHandle()); $file->setStorageFormat(self::STORAGE_FORMAT_RAW); $file->setIsPartial(1); $file->readPropertiesFromParameters($params); return $file; } private static function buildFromFileData($data, array $params = array()) { if (isset($params['storageEngines'])) { $engines = $params['storageEngines']; } else { $size = strlen($data); $engines = PhabricatorFileStorageEngine::loadStorageEngines($size); if (!$engines) { throw new Exception( pht( 'No configured storage engine can store this file. See '. '"Configuring File Storage" in the documentation for '. 'information on configuring storage engines.')); } } assert_instances_of($engines, 'PhabricatorFileStorageEngine'); if (!$engines) { throw new Exception(pht('No valid storage engines are available!')); } $file = self::initializeNewFile(); $data_handle = null; $engine_identifier = null; $exceptions = array(); foreach ($engines as $engine) { $engine_class = get_class($engine); try { list($engine_identifier, $data_handle) = $file->writeToEngine( $engine, $data, $params); // We stored the file somewhere so stop trying to write it to other // places. break; } catch (PhabricatorFileStorageConfigurationException $ex) { // If an engine is outright misconfigured (or misimplemented), raise // that immediately since it probably needs attention. throw $ex; } catch (Exception $ex) { phlog($ex); // If an engine doesn't work, keep trying all the other valid engines // in case something else works. $exceptions[$engine_class] = $ex; } } if (!$data_handle) { throw new PhutilAggregateException( pht('All storage engines failed to write file:'), $exceptions); } $file->setByteSize(strlen($data)); $file->setContentHash(self::hashFileContent($data)); $file->setStorageEngine($engine_identifier); $file->setStorageHandle($data_handle); // TODO: This is probably YAGNI, but allows for us to do encryption or // compression later if we want. $file->setStorageFormat(self::STORAGE_FORMAT_RAW); $file->readPropertiesFromParameters($params); if (!$file->getMimeType()) { $tmp = new TempFile(); Filesystem::writeFile($tmp, $data); $file->setMimeType(Filesystem::getMimeType($tmp)); } try { $file->updateDimensions(false); } catch (Exception $ex) { // Do nothing } $file->save(); return $file; } public static function newFromFileData($data, array $params = array()) { $hash = self::hashFileContent($data); $file = self::newFileFromContentHash($hash, $params); if ($file) { return $file; } return self::buildFromFileData($data, $params); } public function migrateToEngine(PhabricatorFileStorageEngine $engine) { if (!$this->getID() || !$this->getStorageHandle()) { throw new Exception( pht("You can not migrate a file which hasn't yet been saved.")); } $data = $this->loadFileData(); $params = array( 'name' => $this->getName(), ); list($new_identifier, $new_handle) = $this->writeToEngine( $engine, $data, $params); $old_engine = $this->instantiateStorageEngine(); $old_identifier = $this->getStorageEngine(); $old_handle = $this->getStorageHandle(); $this->setStorageEngine($new_identifier); $this->setStorageHandle($new_handle); $this->save(); $this->deleteFileDataIfUnused( $old_engine, $old_identifier, $old_handle); return $this; } private function writeToEngine( PhabricatorFileStorageEngine $engine, $data, array $params) { $engine_class = get_class($engine); $data_handle = $engine->writeFile($data, $params); if (!$data_handle || strlen($data_handle) > 255) { // This indicates an improperly implemented storage engine. throw new PhabricatorFileStorageConfigurationException( pht( "Storage engine '%s' executed %s but did not return a valid ". "handle ('%s') to the data: it must be nonempty and no longer ". "than 255 characters.", $engine_class, 'writeFile()', $data_handle)); } $engine_identifier = $engine->getEngineIdentifier(); if (!$engine_identifier || strlen($engine_identifier) > 32) { throw new PhabricatorFileStorageConfigurationException( pht( "Storage engine '%s' returned an improper engine identifier '{%s}': ". "it must be nonempty and no longer than 32 characters.", $engine_class, $engine_identifier)); } return array($engine_identifier, $data_handle); } /** * Download a remote resource over HTTP and save the response body as a file. * * This method respects `security.outbound-blacklist`, and protects against * HTTP redirection (by manually following "Location" headers and verifying * each destination). It does not protect against DNS rebinding. See * discussion in T6755. */ public static function newFromFileDownload($uri, array $params = array()) { $timeout = 5; $redirects = array(); $current = $uri; while (true) { try { if (count($redirects) > 10) { throw new Exception( pht('Too many redirects trying to fetch remote URI.')); } $resolved = PhabricatorEnv::requireValidRemoteURIForFetch( $current, array( 'http', 'https', )); list($resolved_uri, $resolved_domain) = $resolved; $current = new PhutilURI($current); if ($current->getProtocol() == 'http') { // For HTTP, we can use a pre-resolved URI to defuse DNS rebinding. $fetch_uri = $resolved_uri; $fetch_host = $resolved_domain; } else { // For HTTPS, we can't: cURL won't verify the SSL certificate if // the domain has been replaced with an IP. But internal services // presumably will not have valid certificates for rebindable // domain names on attacker-controlled domains, so the DNS rebinding // attack should generally not be possible anyway. $fetch_uri = $current; $fetch_host = null; } $future = id(new HTTPSFuture($fetch_uri)) ->setFollowLocation(false) ->setTimeout($timeout); if ($fetch_host !== null) { $future->addHeader('Host', $fetch_host); } list($status, $body, $headers) = $future->resolve(); if ($status->isRedirect()) { // This is an HTTP 3XX status, so look for a "Location" header. $location = null; foreach ($headers as $header) { list($name, $value) = $header; if (phutil_utf8_strtolower($name) == 'location') { $location = $value; break; } } // HTTP 3XX status with no "Location" header, just treat this like // a normal HTTP error. if ($location === null) { throw $status; } if (isset($redirects[$location])) { throw new Exception( pht('Encountered loop while following redirects.')); } $redirects[$location] = $location; $current = $location; // We'll fall off the bottom and go try this URI now. } else if ($status->isError()) { // This is something other than an HTTP 2XX or HTTP 3XX status, so // just bail out. throw $status; } else { - // This is HTTP 2XX, so use the the response body to save the + // This is HTTP 2XX, so use the response body to save the // file data. $params = $params + array( 'name' => basename($uri), ); return self::newFromFileData($body, $params); } } catch (Exception $ex) { if ($redirects) { throw new PhutilProxyException( pht( 'Failed to fetch remote URI "%s" after following %s redirect(s) '. '(%s): %s', $uri, new PhutilNumber(count($redirects)), implode(' > ', array_keys($redirects)), $ex->getMessage()), $ex); } else { throw $ex; } } } } public static function normalizeFileName($file_name) { $pattern = "@[\\x00-\\x19#%&+!~'\$\"\/=\\\\?<> ]+@"; $file_name = preg_replace($pattern, '_', $file_name); $file_name = preg_replace('@_+@', '_', $file_name); $file_name = trim($file_name, '_'); $disallowed_filenames = array( '.' => 'dot', '..' => 'dotdot', '' => 'file', ); $file_name = idx($disallowed_filenames, $file_name, $file_name); return $file_name; } public function delete() { // We want to delete all the rows which mark this file as the transformation // of some other file (since we're getting rid of it). We also delete all // the transformations of this file, so that a user who deletes an image // doesn't need to separately hunt down and delete a bunch of thumbnails and // resizes of it. $outbound_xforms = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $this->getPHID(), 'transform' => true, ), )) ->execute(); foreach ($outbound_xforms as $outbound_xform) { $outbound_xform->delete(); } $inbound_xforms = id(new PhabricatorTransformedFile())->loadAllWhere( 'transformedPHID = %s', $this->getPHID()); $this->openTransaction(); foreach ($inbound_xforms as $inbound_xform) { $inbound_xform->delete(); } $ret = parent::delete(); $this->saveTransaction(); $this->deleteFileDataIfUnused( $this->instantiateStorageEngine(), $this->getStorageEngine(), $this->getStorageHandle()); return $ret; } /** * Destroy stored file data if there are no remaining files which reference * it. */ public function deleteFileDataIfUnused( PhabricatorFileStorageEngine $engine, $engine_identifier, $handle) { // Check to see if any files are using storage. $usage = id(new PhabricatorFile())->loadAllWhere( 'storageEngine = %s AND storageHandle = %s LIMIT 1', $engine_identifier, $handle); // If there are no files using the storage, destroy the actual storage. if (!$usage) { try { $engine->deleteFile($handle); } catch (Exception $ex) { // In the worst case, we're leaving some data stranded in a storage // engine, which is not a big deal. phlog($ex); } } } public static function hashFileContent($data) { return sha1($data); } public function loadFileData() { $engine = $this->instantiateStorageEngine(); $data = $engine->readFile($this->getStorageHandle()); switch ($this->getStorageFormat()) { case self::STORAGE_FORMAT_RAW: $data = $data; break; default: throw new Exception(pht('Unknown storage format.')); } return $data; } /** * Return an iterable which emits file content bytes. * * @param int Offset for the start of data. * @param int Offset for the end of data. * @return Iterable Iterable object which emits requested data. */ public function getFileDataIterator($begin = null, $end = null) { $engine = $this->instantiateStorageEngine(); return $engine->getFileDataIterator($this, $begin, $end); } public function getViewURI() { if (!$this->getPHID()) { throw new Exception( pht('You must save a file before you can generate a view URI.')); } return $this->getCDNURI(null); } private function getCDNURI($token) { $name = self::normalizeFileName($this->getName()); $name = phutil_escape_uri($name); $parts = array(); $parts[] = 'file'; $parts[] = 'data'; // If this is an instanced install, add the instance identifier to the URI. // Instanced configurations behind a CDN may not be able to control the // request domain used by the CDN (as with AWS CloudFront). Embedding the // instance identity in the path allows us to distinguish between requests // originating from different instances but served through the same CDN. $instance = PhabricatorEnv::getEnvConfig('cluster.instance'); if (strlen($instance)) { $parts[] = '@'.$instance; } $parts[] = $this->getSecretKey(); $parts[] = $this->getPHID(); if ($token) { $parts[] = $token; } $parts[] = $name; $path = '/'.implode('/', $parts); // If this file is only partially uploaded, we're just going to return a // local URI to make sure that Ajax works, since the page is inevitably // going to give us an error back. if ($this->getIsPartial()) { return PhabricatorEnv::getURI($path); } else { return PhabricatorEnv::getCDNURI($path); } } /** * Get the CDN URI for this file, including a one-time-use security token. * */ public function getCDNURIWithToken() { if (!$this->getPHID()) { throw new Exception( pht('You must save a file before you can generate a CDN URI.')); } return $this->getCDNURI($this->generateOneTimeToken()); } public function getInfoURI() { return '/'.$this->getMonogram(); } public function getBestURI() { if ($this->isViewableInBrowser()) { return $this->getViewURI(); } else { return $this->getInfoURI(); } } public function getDownloadURI() { $uri = id(new PhutilURI($this->getViewURI())) ->setQueryParam('download', true); return (string)$uri; } public function getURIForTransform(PhabricatorFileTransform $transform) { return $this->getTransformedURI($transform->getTransformKey()); } private function getTransformedURI($transform) { $parts = array(); $parts[] = 'file'; $parts[] = 'xform'; $instance = PhabricatorEnv::getEnvConfig('cluster.instance'); if (strlen($instance)) { $parts[] = '@'.$instance; } $parts[] = $transform; $parts[] = $this->getPHID(); $parts[] = $this->getSecretKey(); $path = implode('/', $parts); $path = $path.'/'; return PhabricatorEnv::getCDNURI($path); } public function isViewableInBrowser() { return ($this->getViewableMimeType() !== null); } public function isViewableImage() { if (!$this->isViewableInBrowser()) { return false; } $mime_map = PhabricatorEnv::getEnvConfig('files.image-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type); } public function isAudio() { if (!$this->isViewableInBrowser()) { return false; } $mime_map = PhabricatorEnv::getEnvConfig('files.audio-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type); } public function isTransformableImage() { // NOTE: The way the 'gd' extension works in PHP is that you can install it // with support for only some file types, so it might be able to handle // PNG but not JPEG. Try to generate thumbnails for whatever we can. Setup // warns you if you don't have complete support. $matches = null; $ok = preg_match( '@^image/(gif|png|jpe?g)@', $this->getViewableMimeType(), $matches); if (!$ok) { return false; } switch ($matches[1]) { case 'jpg'; case 'jpeg': return function_exists('imagejpeg'); break; case 'png': return function_exists('imagepng'); break; case 'gif': return function_exists('imagegif'); break; default: throw new Exception(pht('Unknown type matched as image MIME type.')); } } public static function getTransformableImageFormats() { $supported = array(); if (function_exists('imagejpeg')) { $supported[] = 'jpg'; } if (function_exists('imagepng')) { $supported[] = 'png'; } if (function_exists('imagegif')) { $supported[] = 'gif'; } return $supported; } public function instantiateStorageEngine() { return self::buildEngine($this->getStorageEngine()); } public static function buildEngine($engine_identifier) { $engines = self::buildAllEngines(); foreach ($engines as $engine) { if ($engine->getEngineIdentifier() == $engine_identifier) { return $engine; } } throw new Exception( pht( "Storage engine '%s' could not be located!", $engine_identifier)); } public static function buildAllEngines() { $engines = id(new PhutilSymbolLoader()) ->setType('class') ->setConcreteOnly(true) ->setAncestorClass('PhabricatorFileStorageEngine') ->selectAndLoadSymbols(); $results = array(); foreach ($engines as $engine_class) { $results[] = newv($engine_class['name'], array()); } return $results; } public function getViewableMimeType() { $mime_map = PhabricatorEnv::getEnvConfig('files.viewable-mime-types'); $mime_type = $this->getMimeType(); $mime_parts = explode(';', $mime_type); $mime_type = trim(reset($mime_parts)); return idx($mime_map, $mime_type); } public function getDisplayIconForMimeType() { $mime_map = PhabricatorEnv::getEnvConfig('files.icon-mime-types'); $mime_type = $this->getMimeType(); return idx($mime_map, $mime_type, 'fa-file-o'); } public function validateSecretKey($key) { return ($key == $this->getSecretKey()); } public function generateSecretKey() { return Filesystem::readRandomCharacters(20); } public function updateDimensions($save = true) { if (!$this->isViewableImage()) { throw new Exception(pht('This file is not a viewable image.')); } if (!function_exists('imagecreatefromstring')) { throw new Exception(pht('Cannot retrieve image information.')); } $data = $this->loadFileData(); $img = imagecreatefromstring($data); if ($img === false) { throw new Exception(pht('Error when decoding image.')); } $this->metadata[self::METADATA_IMAGE_WIDTH] = imagesx($img); $this->metadata[self::METADATA_IMAGE_HEIGHT] = imagesy($img); if ($save) { $this->save(); } return $this; } public function copyDimensions(PhabricatorFile $file) { $metadata = $file->getMetadata(); $width = idx($metadata, self::METADATA_IMAGE_WIDTH); if ($width) { $this->metadata[self::METADATA_IMAGE_WIDTH] = $width; } $height = idx($metadata, self::METADATA_IMAGE_HEIGHT); if ($height) { $this->metadata[self::METADATA_IMAGE_HEIGHT] = $height; } return $this; } /** * Load (or build) the {@class:PhabricatorFile} objects for builtin file * resources. The builtin mechanism allows files shipped with Phabricator * to be treated like normal files so that APIs do not need to special case * things like default images or deleted files. * * Builtins are located in `resources/builtin/` and identified by their * name. * * @param PhabricatorUser Viewing user. * @param list List of builtin file names. * @return dict Dictionary of named builtins. */ public static function loadBuiltins(PhabricatorUser $user, array $names) { $specs = array(); foreach ($names as $name) { $specs[] = array( 'originalPHID' => PhabricatorPHIDConstants::PHID_VOID, 'transform' => 'builtin:'.$name, ); } // NOTE: Anyone is allowed to access builtin files. $files = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms($specs) ->execute(); $files = mpull($files, null, 'getName'); $root = dirname(phutil_get_library_root('phabricator')); $root = $root.'/resources/builtin/'; $build = array(); foreach ($names as $name) { if (isset($files[$name])) { continue; } // This is just a sanity check to prevent loading arbitrary files. if (basename($name) != $name) { throw new Exception(pht("Invalid builtin name '%s'!", $name)); } $path = $root.$name; if (!Filesystem::pathExists($path)) { throw new Exception(pht("Builtin '%s' does not exist!", $path)); } $data = Filesystem::readFile($path); $params = array( 'name' => $name, 'ttl' => time() + (60 * 60 * 24 * 7), 'canCDN' => true, 'builtin' => $name, ); $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $file = self::newFromFileData($data, $params); $xform = id(new PhabricatorTransformedFile()) ->setOriginalPHID(PhabricatorPHIDConstants::PHID_VOID) ->setTransform('builtin:'.$name) ->setTransformedPHID($file->getPHID()) ->save(); unset($unguarded); $file->attachObjectPHIDs(array()); $file->attachObjects(array()); $files[$name] = $file; } return $files; } /** * Convenience wrapper for @{method:loadBuiltins}. * * @param PhabricatorUser Viewing user. * @param string Single builtin name to load. * @return PhabricatorFile Corresponding builtin file. */ public static function loadBuiltin(PhabricatorUser $user, $name) { return idx(self::loadBuiltins($user, array($name)), $name); } public function getObjects() { return $this->assertAttached($this->objects); } public function attachObjects(array $objects) { $this->objects = $objects; return $this; } public function getObjectPHIDs() { return $this->assertAttached($this->objectPHIDs); } public function attachObjectPHIDs(array $object_phids) { $this->objectPHIDs = $object_phids; return $this; } public function getOriginalFile() { return $this->assertAttached($this->originalFile); } public function attachOriginalFile(PhabricatorFile $file = null) { $this->originalFile = $file; return $this; } public function getImageHeight() { if (!$this->isViewableImage()) { return null; } return idx($this->metadata, self::METADATA_IMAGE_HEIGHT); } public function getImageWidth() { if (!$this->isViewableImage()) { return null; } return idx($this->metadata, self::METADATA_IMAGE_WIDTH); } public function getCanCDN() { if (!$this->isViewableImage()) { return false; } return idx($this->metadata, self::METADATA_CAN_CDN); } public function setCanCDN($can_cdn) { $this->metadata[self::METADATA_CAN_CDN] = $can_cdn ? 1 : 0; return $this; } public function isBuiltin() { return ($this->getBuiltinName() !== null); } public function getBuiltinName() { return idx($this->metadata, self::METADATA_BUILTIN); } public function setBuiltinName($name) { $this->metadata[self::METADATA_BUILTIN] = $name; return $this; } public function getIsProfileImage() { return idx($this->metadata, self::METADATA_PROFILE); } public function setIsProfileImage($value) { $this->metadata[self::METADATA_PROFILE] = $value; return $this; } protected function generateOneTimeToken() { $key = Filesystem::readRandomCharacters(16); // Save the new secret. $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $token = id(new PhabricatorAuthTemporaryToken()) ->setObjectPHID($this->getPHID()) ->setTokenType(self::ONETIME_TEMPORARY_TOKEN_TYPE) ->setTokenExpires(time() + phutil_units('1 hour in seconds')) ->setTokenCode(PhabricatorHash::digest($key)) ->save(); unset($unguarded); return $key; } public function validateOneTimeToken($token_code) { $token = id(new PhabricatorAuthTemporaryTokenQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withObjectPHIDs(array($this->getPHID())) ->withTokenTypes(array(self::ONETIME_TEMPORARY_TOKEN_TYPE)) ->withExpired(false) ->withTokenCodes(array(PhabricatorHash::digest($token_code))) ->executeOne(); return $token; } /** * Write the policy edge between this file and some object. * * @param phid Object PHID to attach to. * @return this */ public function attachToObject($phid) { $edge_type = PhabricatorObjectHasFileEdgeType::EDGECONST; id(new PhabricatorEdgeEditor()) ->addEdge($phid, $edge_type, $this->getPHID()) ->save(); return $this; } /** * Remove the policy edge between this file and some object. * * @param phid Object PHID to detach from. * @return this */ public function detachFromObject($phid) { $edge_type = PhabricatorObjectHasFileEdgeType::EDGECONST; id(new PhabricatorEdgeEditor()) ->removeEdge($phid, $edge_type, $this->getPHID()) ->save(); return $this; } /** * Configure a newly created file object according to specified parameters. * * This method is called both when creating a file from fresh data, and * when creating a new file which reuses existing storage. * * @param map Bag of parameters, see @{class:PhabricatorFile} * for documentation. * @return this */ private function readPropertiesFromParameters(array $params) { $file_name = idx($params, 'name'); $this->setName($file_name); $author_phid = idx($params, 'authorPHID'); $this->setAuthorPHID($author_phid); $file_ttl = idx($params, 'ttl'); $this->setTtl($file_ttl); $view_policy = idx($params, 'viewPolicy'); if ($view_policy) { $this->setViewPolicy($params['viewPolicy']); } $is_explicit = (idx($params, 'isExplicitUpload') ? 1 : 0); $this->setIsExplicitUpload($is_explicit); $can_cdn = idx($params, 'canCDN'); if ($can_cdn) { $this->setCanCDN(true); } $builtin = idx($params, 'builtin'); if ($builtin) { $this->setBuiltinName($builtin); } $profile = idx($params, 'profile'); if ($profile) { $this->setIsProfileImage(true); } $mime_type = idx($params, 'mime-type'); if ($mime_type) { $this->setMimeType($mime_type); } return $this; } public function getRedirectResponse() { $uri = $this->getBestURI(); // TODO: This is a bit iffy. Sometimes, getBestURI() returns a CDN URI // (if the file is a viewable image) and sometimes a local URI (if not). // For now, just detect which one we got and configure the response // appropriately. In the long run, if this endpoint is served from a CDN // domain, we can't issue a local redirect to an info URI (which is not // present on the CDN domain). We probably never actually issue local // redirects here anyway, since we only ever transform viewable images // right now. $is_external = strlen(id(new PhutilURI($uri))->getDomain()); return id(new AphrontRedirectResponse()) ->setIsExternal($is_external) ->setURI($uri); } /* -( PhabricatorApplicationTransactionInterface )------------------------- */ public function getApplicationTransactionEditor() { return new PhabricatorFileEditor(); } public function getApplicationTransactionObject() { return $this; } public function getApplicationTransactionTemplate() { return new PhabricatorFileTransaction(); } public function willRenderTimeline( PhabricatorApplicationTransactionView $timeline, AphrontRequest $request) { return $timeline; } /* -( PhabricatorPolicyInterface Implementation )-------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: if ($this->isBuiltin()) { return PhabricatorPolicies::getMostOpenPolicy(); } if ($this->getIsProfileImage()) { return PhabricatorPolicies::getMostOpenPolicy(); } return $this->getViewPolicy(); case PhabricatorPolicyCapability::CAN_EDIT: return PhabricatorPolicies::POLICY_NOONE; } } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { $viewer_phid = $viewer->getPHID(); if ($viewer_phid) { if ($this->getAuthorPHID() == $viewer_phid) { return true; } } switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: // If you can see the file this file is a transform of, you can see // this file. if ($this->getOriginalFile()) { return true; } // If you can see any object this file is attached to, you can see // the file. return (count($this->getObjects()) > 0); } return false; } public function describeAutomaticCapability($capability) { $out = array(); $out[] = pht('The user who uploaded a file can always view and edit it.'); switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: $out[] = pht( 'Files attached to objects are visible to users who can view '. 'those objects.'); $out[] = pht( 'Thumbnails are visible only to users who can view the original '. 'file.'); break; } return $out; } /* -( PhabricatorSubscribableInterface Implementation )-------------------- */ public function isAutomaticallySubscribed($phid) { return ($this->authorPHID == $phid); } public function shouldShowSubscribersProperty() { return true; } public function shouldAllowSubscription($phid) { return true; } /* -( PhabricatorTokenReceiverInterface )---------------------------------- */ public function getUsersToNotifyOfTokenGiven() { return array( $this->getAuthorPHID(), ); } /* -( PhabricatorDestructibleInterface )----------------------------------- */ public function destroyObjectPermanently( PhabricatorDestructionEngine $engine) { $this->openTransaction(); $this->delete(); $this->saveTransaction(); } } diff --git a/src/applications/files/storage/__tests__/PhabricatorFileTestCase.php b/src/applications/files/storage/__tests__/PhabricatorFileTestCase.php index d4d5afcd71..e90b60f6f2 100644 --- a/src/applications/files/storage/__tests__/PhabricatorFileTestCase.php +++ b/src/applications/files/storage/__tests__/PhabricatorFileTestCase.php @@ -1,363 +1,363 @@ true, ); } public function testFileVisibility() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $author = $this->generateNewTestUser(); $viewer = $this->generateNewTestUser(); $users = array($author, $viewer); $params = array( 'name' => 'test.dat', 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, 'authorPHID' => $author->getPHID(), 'storageEngines' => array( $engine, ), ); $file = PhabricatorFile::newFromFileData($data, $params); $filter = new PhabricatorPolicyFilter(); // Test bare file policies. $this->assertEqual( array( true, false, ), $this->canViewFile($users, $file), pht('File Visibility')); // Create an object and test object policies. $object = ManiphestTask::initializeNewTask($author); $object->setViewPolicy(PhabricatorPolicies::getMostOpenPolicy()); $object->save(); $this->assertTrue( $filter->hasCapability( $author, $object, PhabricatorPolicyCapability::CAN_VIEW), pht('Object Visible to Author')); $this->assertTrue( $filter->hasCapability( $viewer, $object, PhabricatorPolicyCapability::CAN_VIEW), pht('Object Visible to Others')); // Attach the file to the object and test that the association opens a // policy exception for the non-author viewer. $file->attachToObject($object->getPHID()); // Test the attached file's visibility. $this->assertEqual( array( true, true, ), $this->canViewFile($users, $file), pht('Attached File Visibility')); // Create a "thumbnail" of the original file. $params = array( 'name' => 'test.thumb.dat', 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, 'storageEngines' => array( $engine, ), ); $xform = PhabricatorFile::newFromFileData($data, $params); id(new PhabricatorTransformedFile()) ->setOriginalPHID($file->getPHID()) ->setTransform('test-thumb') ->setTransformedPHID($xform->getPHID()) ->save(); // Test the thumbnail's visibility. $this->assertEqual( array( true, true, ), $this->canViewFile($users, $xform), pht('Attached Thumbnail Visibility')); // Detach the object and make sure it affects the thumbnail. $file->detachFromObject($object->getPHID()); // Test the detached thumbnail's visibility. $this->assertEqual( array( true, false, ), $this->canViewFile($users, $xform), pht('Detached Thumbnail Visibility')); } private function canViewFile(array $users, PhabricatorFile $file) { $results = array(); foreach ($users as $user) { $results[] = (bool)id(new PhabricatorFileQuery()) ->setViewer($user) ->withPHIDs(array($file->getPHID())) ->execute(); } return $results; } public function testFileStorageReadWrite() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $params = array( 'name' => 'test.dat', 'storageEngines' => array( $engine, ), ); $file = PhabricatorFile::newFromFileData($data, $params); // Test that the storage engine worked, and was the target of the write. We // don't actually care what the data is (future changes may compress or // encrypt it), just that it exists in the test storage engine. $engine->readFile($file->getStorageHandle()); // Now test that we get the same data back out. $this->assertEqual($data, $file->loadFileData()); } public function testFileStorageUploadDifferentFiles() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $other_data = Filesystem::readRandomCharacters(64); $params = array( 'name' => 'test.dat', 'storageEngines' => array( $engine, ), ); $first_file = PhabricatorFile::newFromFileData($data, $params); $second_file = PhabricatorFile::newFromFileData($other_data, $params); - // Test that the the second file uses different storage handle from + // Test that the second file uses different storage handle from // the first file. $first_handle = $first_file->getStorageHandle(); $second_handle = $second_file->getStorageHandle(); $this->assertTrue($first_handle != $second_handle); } public function testFileStorageUploadSameFile() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $params = array( 'name' => 'test.dat', 'storageEngines' => array( $engine, ), ); $first_file = PhabricatorFile::newFromFileData($data, $params); $second_file = PhabricatorFile::newFromFileData($data, $params); // Test that the the second file uses the same storage handle as // the first file. $handle = $first_file->getStorageHandle(); $second_handle = $second_file->getStorageHandle(); $this->assertEqual($handle, $second_handle); } public function testFileStorageDelete() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $params = array( 'name' => 'test.dat', 'storageEngines' => array( $engine, ), ); $file = PhabricatorFile::newFromFileData($data, $params); $handle = $file->getStorageHandle(); $file->delete(); $caught = null; try { $engine->readFile($handle); } catch (Exception $ex) { $caught = $ex; } $this->assertTrue($caught instanceof Exception); } public function testFileStorageDeleteSharedHandle() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $params = array( 'name' => 'test.dat', 'storageEngines' => array( $engine, ), ); $first_file = PhabricatorFile::newFromFileData($data, $params); $second_file = PhabricatorFile::newFromFileData($data, $params); $first_file->delete(); $this->assertEqual($data, $second_file->loadFileData()); } public function testReadWriteTtlFiles() { $engine = new PhabricatorTestStorageEngine(); $data = Filesystem::readRandomCharacters(64); $ttl = (time() + 60 * 60 * 24); $params = array( 'name' => 'test.dat', 'ttl' => ($ttl), 'storageEngines' => array( $engine, ), ); $file = PhabricatorFile::newFromFileData($data, $params); $this->assertEqual($ttl, $file->getTTL()); } public function testFileTransformDelete() { // We want to test that a file deletes all its inbound transformation // records and outbound transformed derivatives when it is deleted. // First, we create a chain of transforms, A -> B -> C. $engine = new PhabricatorTestStorageEngine(); $params = array( 'name' => 'test.txt', 'storageEngines' => array( $engine, ), ); $a = PhabricatorFile::newFromFileData('a', $params); $b = PhabricatorFile::newFromFileData('b', $params); $c = PhabricatorFile::newFromFileData('c', $params); id(new PhabricatorTransformedFile()) ->setOriginalPHID($a->getPHID()) ->setTransform('test:a->b') ->setTransformedPHID($b->getPHID()) ->save(); id(new PhabricatorTransformedFile()) ->setOriginalPHID($b->getPHID()) ->setTransform('test:b->c') ->setTransformedPHID($c->getPHID()) ->save(); // Now, verify that A -> B and B -> C exist. $xform_a = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $a->getPHID(), 'transform' => true, ), )) ->execute(); $this->assertEqual(1, count($xform_a)); $this->assertEqual($b->getPHID(), head($xform_a)->getPHID()); $xform_b = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $b->getPHID(), 'transform' => true, ), )) ->execute(); $this->assertEqual(1, count($xform_b)); $this->assertEqual($c->getPHID(), head($xform_b)->getPHID()); // Delete "B". $b->delete(); // Now, verify that the A -> B and B -> C links are gone. $xform_a = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $a->getPHID(), 'transform' => true, ), )) ->execute(); $this->assertEqual(0, count($xform_a)); $xform_b = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTransforms( array( array( 'originalPHID' => $b->getPHID(), 'transform' => true, ), )) ->execute(); $this->assertEqual(0, count($xform_b)); // Also verify that C has been deleted. $alternate_c = id(new PhabricatorFileQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($c->getPHID())) ->execute(); $this->assertEqual(array(), $alternate_c); } } diff --git a/src/docs/user/configuration/troubleshooting_https.diviner b/src/docs/user/configuration/troubleshooting_https.diviner index 7d9ab71a44..6b93a4f690 100644 --- a/src/docs/user/configuration/troubleshooting_https.diviner +++ b/src/docs/user/configuration/troubleshooting_https.diviner @@ -1,80 +1,80 @@ @title Troubleshooting HTTPS @group config Detailed instructions for troubleshooting HTTPS connection problems. = Overview = If you're having trouble connecting to an HTTPS install of Phabricator, and particularly if you're receiving a "There was an error negotiating the SSL connection." error, this document may be able to help you diagnose and resolve the problem. Connection negotiation can fail for several reasons. The major ones are: - - You have not added the the Certificate Authority as a trusted authority + - You have not added the Certificate Authority as a trusted authority (this is the most common problem, and usually the issue for self-signed certificates). - The SSL certificate is signed for the wrong domain. For example, a certificate signed for `www.example.com` will not work for `phabricator.example.com`. - The server rejects TLSv1 SNI connections for the domain (this is complicated, see below). = Certificate Authority Problems = SSL certificates need to be signed by a trusted authority (called a Certificate Authority or "CA") to be accepted. If the CA for a certificate is untrusted, the connection will fail (this defends the connection from an eavesdropping attack called "man in the middle"). Normally, you purchase a certificate from a known authority and clients have a list of trusted authorities. You can self-sign a certificate by creating your own CA, but clients will not trust it by default. They need to add the CA as a trusted authority. For instructions on adding CAs, see `libphutil/resources/ssl/README`. If you'd prefer that `arc` not verify the identity of the server whatsoever, you can use the `https.blindly-trust-domains` setting. This will make it dramatically easier for adversaries to perform certain types of attacks, and is **strongly discouraged**: $ arc set-config https.blindly-trust-domains '["example.com"]' = Domain Problems = Verify the domain the certificate was issued for. You can generally do this with: $ openssl x509 -text -in If the certificate was accidentally generated for, e.g. `www.example.com` but you installed Phabricator on `phabricator.example.com`, you need to generate a new certificate for the right domain. = SNI Problems = Server Name Identification ("SNI") is a feature of TLSv1 which works a bit like Apache VirtualHosts, and allows a server to present different certificates to clients who are connecting to it using different names. Servers that are not configured properly may reject TSLv1 SNI requests because they do not recognize the name the client is connecting with. This topic is complicated, but you can test for it by running: $ openssl s_client -connect example.com:443 -servername example.com Replace **both** instances of "example.com" with your domain. If you receive an error in `SSL23_GET_SERVER_HELLO` with `reason(1112)`, like this: CONNECTED(00000003) 87871:error:14077458:SSL routines:SSL23_GET_SERVER_HELLO:reason(1112): /SourceCache/OpenSSL098/OpenSSL098-44/src/ssl/s23_clnt.c:602: ...it indicates server is misconfigured. The most common cause of this problem is an Apache server that does not explicitly name the Phabricator domain as a valid VirtualHost. This error occurs only for some versions of the OpenSSL client library (from v0.9.8r or earlier until 1.0.0), so only some users may experience it. diff --git a/src/docs/user/field/repository_imports.diviner b/src/docs/user/field/repository_imports.diviner index a59bee051f..495758ea4a 100644 --- a/src/docs/user/field/repository_imports.diviner +++ b/src/docs/user/field/repository_imports.diviner @@ -1,247 +1,247 @@ @title Troubleshooting Repository Imports @group fieldmanual Guide to the troubleshooting repositories which import incompletely. Overview ======== When you first import an external source code repository (or push new commits to a hosted repository), Phabricator imports those commits in the background. While a repository is initially importing, some features won't work. While individual commits are importing, some of their metadata won't be available in the web UI. Sometimes, the import process may hang or fail to complete. This document can help you understand the import process and troubleshoot problems with it. Understanding the Import Pipeline ================================= Phabricator first performs commit discovery on repositories. This examines a repository and identifies all the commits in it at a very shallow level, then creates stub objects for them. These stub objects primarily serve to assign various internal IDs to each commit. Commit discovery occurs in the update phase, and you can learn more about updates in @{article:Diffusion User Guide: Repository Updates}. After commits are discovered, background tasks are queued to actually import commits. These tasks do things like look at commit messages, trigger mentions and autoclose rules, cache changes, trigger Herald, publish feed stories and email, and apply Owners rules. You can learn more about some of these steps in @{article:Diffusion User Guide: Autoclose}. Specifically, the import pipeline has four steps: - **Message**: Parses the commit message and author metadata. - **Change**: Caches the paths the commit affected. - **Owners**: Runs Owners rules. - **Herald**: Runs Herald rules and publishes notifications. These steps run in sequence for each commit, but all discovered commits import in parallel. Identifying Missing Steps ========================= There are a few major pieces of information you can look at to understand where the import process is stuck. First, to identify which commits have missing import steps, run this command: ``` phabricator/ $ ./bin/repository importing rXYZ ``` That will show what work remains to be done. Each line shows a commit which is discovered but not imported, and the import steps that are remaining for that commit. Generally, the commit is stuck on the first step in the list. Second, load the Daemon Console (at `/daemon/` in the web UI). This will show what work is currently being done and waiting to complete. The most important sections are "Queued Tasks" (work waiting in queue) and "Leased Tasks" (work currently being done). Third, run this command to look at the daemon logs: ``` phabricator/ $ ./bin/phd log ``` This can show you any errors the daemons have encountered recently. The next sections will walk through how to use this information to understand and resolve the issue. Handling Permanent Failures =========================== Some commits can not be imported, which will permanently stop a repository from fully importing. These are rare, but can be caused by unusual data in a repository, version peculiarities, or bugs in the importer. Permanent failures usually look like a small number of commits stuck on the "Message" or "Change" steps in the output of `repository importing`. If you have a larger number of commits, it is less likely that there are any permanent problems. In the Daemon console, permanent failures usually look like a small number of tasks in "Leased Tasks" with a large failure count. These tasks are retrying until they succeed, but a bug is permanently preventing them from succeeding, so they'll rack up a large number of retries over time. In the daemon log, these commits usually emit specific errors showing why they're failing to import. These failures are the easiest to identify and understand, and can often be resolved quickly. Choose some failing commit from the output of `bin/repository importing` and use this command to re-run any missing steps manually in the foreground: ``` phabricator/ $ ./bin/repository reparse --importing --trace rXYZabcdef012... ``` This command is always safe to run, no matter what the actual root cause of the problem is. If this fails with an error, you've likely identified a problem with Phabricator. Collect as much information as you can about what makes the commit special and file a bug in the upstream by following the instructions in @{article:Contributing Bug Reports}. If the commit imports cleanly, this is more likely to be caused by some other issue. Handling Temporary Failures =========================== Some commits may temporarily fail to import: perhaps the network or services may have briefly been down, or some configuration wasn't quite right, or the daemons were killed halfway through the work. These commits will retry eventually and usually succeed, but some of the retry time limits are very conserative (up to 24 hours) and you might not want to wait that long. In the Daemon console, temporarily failures usually look like tasks in the "Leased Tasks" column with a large "Expires" value but a low "Failures" count (usually 0 or 1). The "Expires" column is showing how long Phabricator is waiting to retry these tasks. In the daemon log, these temporary failures might have created log entries, but might also not have. For example, if the failure was rooted in a network issue that probably will create a log entry, but if the faiulre was rooted in the daemons being abruptly killed that may not create a log entry. You can follow the instructions from "Handling Permanent Failures" above to reparse steps individually to look for an error that represents a root cause, but sometimes this can happen because of some transient issue which won't be identifiable. The easiest way to fix this is to restart the daemons. When you restart daemons, all task leases are immediately expired, so any tasks waiting for a long time will run right away: ``` phabricator/ $ ./bin/phd restart ``` This command is always safe to run, no matter what the actual root cause of the problem is. After restarting the daemons, any pending tasks should be able to retry immediately. For more information on managing the daemons, see @{article:Managing Daemons with phd}. Forced Parsing ============== In rare cases, the actual tasks may be lost from the task queue. Usually, they have been stolen by gremlins or spritied away by ghosts, or someone may have been too ambitious with running manual SQL commands and deleted a bunch of extra things they shouldn't have. There is no normal set of conditions under which this should occur, but you can force Phabricator to re-queue the tasks to recover from it if it does occur. This will look like missing steps in `repository importing`, but nothing in the "Queued Tasks" or "Leased Tasks" sections of the daemon console. The daemon logs will also be empty, since the tasks have vanished. To re-queue parse tasks for a repository, run this command, which will queue up all of the missing work in `repository importing`: ``` phabricator/ $ ./bin/repository reparse --importing --all rXYZ ``` This command may cause duplicate work to occur if you have misdiagnosed the problem and the tasks aren't actually lost. For example, it could queue a second task to perform publishing, which could cause Phabricator to send a second copy of email about the commit. Other than that, it is safe to run even if this isn't the problem. After running this command, you should see tasks in "Queued Tasks" and "Leased Tasks" in the console which correspond to the commits in `repository importing`, and progress should resume. Forced Imports ============== In some cases, you might want to force a repository to be flagged as imported even though the import isn't complete. The most common and reasonable case where you might want to do this is if you've identified a permanent failure with a small number of commits (maybe just one) and reported it upstream, and are waiting for a fix. You might want to start using the repository immediately, even if a few things can't import yet. You should be cautious about doing this. The "importing" flag controls publishing of notifications and email, so if you flag a repository as imported but it still has a lot of work queued, it may send an enormous amount of email as that work completes. To mark a repository as imported even though it really isn't, run this command: ``` phabricator/ $ ./bin/repository mark-imported rXYZ ``` If you do this by mistake, you can reverse it later by using the `--mark-not-imported` flag. General Tips ============ Broadly, `bin/repository` contains several useful debugging commands which let you figure out where failures are occuring. You can add the `--trace` flag to any command to get more details about what it is doing. For any command, you can use `help` to learn more about what it does and which flag it takes: ``` phabricator/ $ bin/repository help ``` -In particular, you can use flags with the the `repository reparse` command to +In particular, you can use flags with the `repository reparse` command to manually run parse steps in the foreground, including re-running steps and running steps out of order. Next Steps ========== Continue by: - returning to the @{article:Diffusion User Guide}. diff --git a/src/docs/user/userguide/diffusion_updates.diviner b/src/docs/user/userguide/diffusion_updates.diviner index 3dae59b3a5..eb01b8c9bc 100644 --- a/src/docs/user/userguide/diffusion_updates.diviner +++ b/src/docs/user/userguide/diffusion_updates.diviner @@ -1,123 +1,123 @@ @title Diffusion User Guide: Repository Updates @group userguide Explains how Diffusion updates repositories to discover new changes. Overview ======== When Phabricator is configured to import repositories which are hosted elsewhere, it needs to poll those repositories for changes. If it polls too frequently, it can create too much load locally and on remote services. If it polls too rarely, it may take a long time for commits to show up in the web interface. This document describes the rules around polling and how to understand and adjust the behavior. In general: - Phabricator chooses a default poll interval based on repository activity. These intervals range from every 15 seconds (for active repositories) to every 6 hours (for repositories with no commits in two months). - If you use `arc` to push commits, or you host repositories on Phabricator, repositories automatically update after changes are pushed. - If you don't use `arc` and your repository is hosted elsewhere, this document describes ways you can make polling more responsive. Default Behavior ================ By default, Phabricator determines how frequently to poll repositories by examining how long it has been since the last commit. In most cases this is fairly accurate and produces good behavior. In particular, it automatically reduces the polling frequency for rarely-used repositories. This dramatically reduces load for installs with a large number of inactive repositories, which is common. For repositories with activity in the last 3 days, we wait 1 second for every 10 minutes without activity. The table below has some examples. | Time Since Commit | Poll Interval | |-------------------|------------------| | //Minimum// | 15 seconds | | 6h | about 30 seconds | | 12h | about 1 minute | | 1 day | about 2 minutes | | 2 days | about 5 minutes | | 3 days | about 7 minutes | This means that you may need to wait about 2 minutes for the first commit to be imported in the morning, and about 5 minutes after a long weekend, but other commits to active repositories should usually be recognized in 30 seconds or less. For repositories with no activity in the last 3 days, we wait longer between updates (1 second for every 4 minutes without activity). The table below has some examples. | Time Since Commit | Poll Interval | |-------------------|------------------| | 4 days | about 30 minutes | | 7 days | about 45 minutes | | 10 days | about 1 hour | | 20 days | about 2 hours | | 30 days | about 3 hours | | //Maximum// | 6 hours | You can find the exact default poll frequency of a repository in Diffusion > (Choose a Repository) > Edit Repository, under "Update Frequency". You can also see the time when the repository was last updated in this interface. Repositories that are currently importing are always updated at the minimum update frequency so the import finishes as quickly as possible. Triggering Repository Updates ============================= If you want Phabricator to update a repository more quickly than the default update frequency (for example, because you just pushed a commit to it), you can tell Phabricator that it should schedule an update as soon as possible. There are several ways to do this: - If you push changes with `arc land` or `arc commit`, this will be done for you automatically. These commits should normally be recognized within a few seconds. - If your repository is hosted on Phabricator, this will also be done for you automatically. - You can schedule an update from the web interface, in Diffusion > (Choose a Repository) > Edit Repository > Update Now. - You can make a call to the Conduit API method `diffusion.looksoon`. This hints to Phabricator that it should poll a repository as soon as it can. All of the other mechanisms do this under the hood. In particular, you may be able to add a commit hook to your external repository which calls `diffusion.looksoon`. This should make an external repository about as responsive as a hosted repository. If a repository has an update scheduled, the Diffusion > (Choose a -Repository) > Edit Repository interface will show that the the repository is +Repository) > Edit Repository interface will show that the repository is prioritized and will be updated soon. Troubleshooting Updates ======================= You can manually run a repository update from the command line to troubleshoot issues, using the `--trace` flag to get full details: phabricator/ $ ./bin/repository update --trace To catch potential issues with permissions, run this command as the same user that the daemons run as. Next Steps ========== Continue by: - troubleshooting in greater depth with @{article:Troubleshooting Repository Imports}.