diff --git a/scripts/fpm/warmup.php b/scripts/fpm/warmup.php deleted file mode 100644 index 956a474131..0000000000 --- a/scripts/fpm/warmup.php +++ /dev/null @@ -1,38 +0,0 @@ -selectAndLoadSymbols(); - - define('__WARMUP__', true); -} - -__warmup__(); diff --git a/scripts/install/update_phabricator.sh b/scripts/install/update_phabricator.sh index 5a3950088e..3831acd963 100755 --- a/scripts/install/update_phabricator.sh +++ b/scripts/install/update_phabricator.sh @@ -1,56 +1,53 @@ #!/bin/sh set -e set -x # This is an example script for updating Phabricator, similar to the one used to # update . It might not work perfectly on your # system, but hopefully it should be easy to adapt. This script is not intended # to work without modifications. # NOTE: This script assumes you are running it from a directory which contains -# arcanist/, libphutil/, and phabricator/. +# arcanist/ and phabricator/. ROOT=`pwd` # You can hard-code the path here instead. ### UPDATE WORKING COPIES ###################################################### -cd $ROOT/libphutil -git pull - cd $ROOT/arcanist git pull cd $ROOT/phabricator git pull ### CYCLE WEB SERVER AND DAEMONS ############################################### # Stop daemons. $ROOT/phabricator/bin/phd stop # If running the notification server, stop it. # $ROOT/phabricator/bin/aphlict stop # Stop the webserver (apache, nginx, lighttpd, etc). This command will differ # depending on which system and webserver you are running: replace it with an # appropriate command for your system. # NOTE: If you're running php-fpm, you should stop it here too. sudo /etc/init.d/httpd stop # Upgrade the database schema. You may want to add the "--force" flag to allow # this script to run noninteractively. $ROOT/phabricator/bin/storage upgrade # Restart the webserver. As above, this depends on your system and webserver. # NOTE: If you're running php-fpm, restart it here too. sudo /etc/init.d/httpd start # Restart daemons. $ROOT/phabricator/bin/phd start # If running the notification server, start it. # $ROOT/phabricator/bin/aphlict start diff --git a/src/applications/conduit/controller/PhabricatorConduitController.php b/src/applications/conduit/controller/PhabricatorConduitController.php index 0f7b2ef54b..e66c1f8a0b 100644 --- a/src/applications/conduit/controller/PhabricatorConduitController.php +++ b/src/applications/conduit/controller/PhabricatorConduitController.php @@ -1,304 +1,304 @@ getRequest()->getUser(); $nav = new AphrontSideNavFilterView(); $nav->setBaseURI(new PhutilURI($this->getApplicationURI())); id(new PhabricatorConduitSearchEngine()) ->setViewer($viewer) ->addNavigationItems($nav->getMenu()); $nav->addLabel('Logs'); $nav->addFilter('log', pht('Call Logs')); $nav->selectFilter(null); return $nav; } public function buildApplicationMenu() { return $this->buildSideNavView()->getMenu(); } protected function renderExampleBox(ConduitAPIMethod $method, $params) { $viewer = $this->getViewer(); $arc_example = id(new PHUIPropertyListView()) ->addRawContent($this->renderExample($method, 'arc', $params)); $curl_example = id(new PHUIPropertyListView()) ->addRawContent($this->renderExample($method, 'curl', $params)); $php_example = id(new PHUIPropertyListView()) ->addRawContent($this->renderExample($method, 'php', $params)); $panel_uri = id(new PhabricatorConduitTokensSettingsPanel()) ->setViewer($viewer) ->setUser($viewer) ->getPanelURI(); $panel_link = phutil_tag( 'a', array( 'href' => $panel_uri, ), pht('Conduit API Tokens')); $panel_link = phutil_tag('strong', array(), $panel_link); $messages = array( pht( 'Use the %s panel in Settings to generate or manage API tokens.', $panel_link), ); if ($params === null) { $messages[] = pht( 'If you submit parameters, these examples will update to show '. 'exactly how to encode the parameters you submit.'); } $info_view = id(new PHUIInfoView()) ->setErrors($messages) ->setSeverity(PHUIInfoView::SEVERITY_NOTICE); $tab_group = id(new PHUITabGroupView()) ->addTab( id(new PHUITabView()) ->setName(pht('arc call-conduit')) ->setKey('arc') ->appendChild($arc_example)) ->addTab( id(new PHUITabView()) ->setName(pht('cURL')) ->setKey('curl') ->appendChild($curl_example)) ->addTab( id(new PHUITabView()) ->setName(pht('PHP')) ->setKey('php') ->appendChild($php_example)); return id(new PHUIObjectBoxView()) ->setHeaderText(pht('Examples')) ->setInfoView($info_view) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->addTabGroup($tab_group); } private function renderExample( ConduitAPIMethod $method, $kind, $params) { switch ($kind) { case 'arc': $example = $this->buildArcanistExample($method, $params); break; case 'php': $example = $this->buildPHPExample($method, $params); break; case 'curl': $example = $this->buildCURLExample($method, $params); break; default: throw new Exception(pht('Conduit client "%s" is not known.', $kind)); } return $example; } private function buildArcanistExample( ConduitAPIMethod $method, $params) { $parts = array(); $parts[] = '$ echo '; if ($params === null) { $parts[] = phutil_tag('strong', array(), ''); } else { $params = $this->simplifyParams($params); $params = id(new PhutilJSON())->encodeFormatted($params); $params = trim($params); $params = csprintf('%s', $params); $parts[] = phutil_tag('strong', array('class' => 'real'), $params); } $parts[] = ' | '; $parts[] = 'arc call-conduit '; $parts[] = '--conduit-uri '; $parts[] = phutil_tag( 'strong', array('class' => 'real'), PhabricatorEnv::getURI('/')); $parts[] = ' '; $parts[] = '--conduit-token '; $parts[] = phutil_tag('strong', array(), ''); $parts[] = ' '; $parts[] = '--'; $parts[] = ' '; $parts[] = $method->getAPIMethodName(); return $this->renderExampleCode($parts); } private function buildPHPExample( ConduitAPIMethod $method, $params) { $parts = array(); - $libphutil_path = 'path/to/libphutil/src/__phutil_library_init__.php'; + $libphutil_path = 'path/to/arcanist/support/init/init-script.php'; $parts[] = '')); $parts[] = "\";\n"; $parts[] = '$api_parameters = '; if ($params === null) { $parts[] = 'array('; $parts[] = phutil_tag('strong', array(), pht('')); $parts[] = ');'; } else { $params = $this->simplifyParams($params); $params = phutil_var_export($params); $parts[] = phutil_tag('strong', array('class' => 'real'), $params); $parts[] = ';'; } $parts[] = "\n\n"; $parts[] = '$client = new ConduitClient('; $parts[] = phutil_tag( 'strong', array('class' => 'real'), phutil_var_export(PhabricatorEnv::getURI('/'))); $parts[] = ");\n"; $parts[] = '$client->setConduitToken($api_token);'; $parts[] = "\n\n"; $parts[] = '$result = $client->callMethodSynchronous('; $parts[] = phutil_tag( 'strong', array('class' => 'real'), phutil_var_export($method->getAPIMethodName())); $parts[] = ', '; $parts[] = '$api_parameters'; $parts[] = ");\n"; $parts[] = 'print_r($result);'; return $this->renderExampleCode($parts); } private function buildCURLExample( ConduitAPIMethod $method, $params) { $call_uri = '/api/'.$method->getAPIMethodName(); $parts = array(); $linebreak = array('\\', phutil_tag('br'), ' '); $parts[] = '$ curl '; $parts[] = phutil_tag( 'strong', array('class' => 'real'), csprintf('%R', PhabricatorEnv::getURI($call_uri))); $parts[] = ' '; $parts[] = $linebreak; $parts[] = '-d api.token='; $parts[] = phutil_tag('strong', array(), 'api-token'); $parts[] = ' '; $parts[] = $linebreak; if ($params === null) { $parts[] = '-d '; $parts[] = phutil_tag('strong', array(), 'param'); $parts[] = '='; $parts[] = phutil_tag('strong', array(), 'value'); $parts[] = ' '; $parts[] = $linebreak; $parts[] = phutil_tag('strong', array(), '...'); } else { $lines = array(); $params = $this->simplifyParams($params); foreach ($params as $key => $value) { $pieces = $this->getQueryStringParts(null, $key, $value); foreach ($pieces as $piece) { $lines[] = array( '-d ', phutil_tag('strong', array('class' => 'real'), $piece), ); } } $parts[] = phutil_implode_html(array(' ', $linebreak), $lines); } return $this->renderExampleCode($parts); } private function renderExampleCode($example) { require_celerity_resource('conduit-api-css'); return phutil_tag( 'div', array( 'class' => 'PhabricatorMonospaced conduit-api-example-code', ), $example); } private function simplifyParams(array $params) { foreach ($params as $key => $value) { if ($value === null) { unset($params[$key]); } } return $params; } private function getQueryStringParts($prefix, $key, $value) { if ($prefix === null) { $head = phutil_escape_uri($key); } else { $head = $prefix.'['.phutil_escape_uri($key).']'; } if (!is_array($value)) { return array( $head.'='.phutil_escape_uri($value), ); } $results = array(); foreach ($value as $subkey => $subvalue) { $subparts = $this->getQueryStringParts($head, $subkey, $subvalue); foreach ($subparts as $subpart) { $results[] = $subpart; } } return $results; } } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php index 5e169fda6a..508a886185 100644 --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1,2881 +1,2881 @@ setViewer($actor) ->withClasses(array('PhabricatorDiffusionApplication')) ->executeOne(); $view_policy = $app->getPolicy(DiffusionDefaultViewCapability::CAPABILITY); $edit_policy = $app->getPolicy(DiffusionDefaultEditCapability::CAPABILITY); $push_policy = $app->getPolicy(DiffusionDefaultPushCapability::CAPABILITY); $repository = id(new PhabricatorRepository()) ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setPushPolicy($push_policy) ->setSpacePHID($actor->getDefaultSpacePHID()); // Put the repository in "Importing" mode until we finish // parsing it. $repository->setDetail('importing', true); return $repository; } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'details' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'sort255', 'callsign' => 'sort32?', 'repositorySlug' => 'sort64?', 'versionControlSystem' => 'text32', 'uuid' => 'text64?', 'pushPolicy' => 'policy', 'credentialPHID' => 'phid?', 'almanacServicePHID' => 'phid?', 'localPath' => 'text128?', 'profileImagePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'callsign' => array( 'columns' => array('callsign'), 'unique' => true, ), 'key_name' => array( 'columns' => array('name(128)'), ), 'key_vcs' => array( 'columns' => array('versionControlSystem'), ), 'key_slug' => array( 'columns' => array('repositorySlug'), 'unique' => true, ), 'key_local' => array( 'columns' => array('localPath'), 'unique' => true, ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryRepositoryPHIDType::TYPECONST); } public static function getStatusMap() { return array( self::STATUS_ACTIVE => array( 'name' => pht('Active'), 'isTracked' => 1, ), self::STATUS_INACTIVE => array( 'name' => pht('Inactive'), 'isTracked' => 0, ), ); } public static function getStatusNameMap() { return ipull(self::getStatusMap(), 'name'); } public function getStatus() { if ($this->isTracked()) { return self::STATUS_ACTIVE; } else { return self::STATUS_INACTIVE; } } public function toDictionary() { return array( 'id' => $this->getID(), 'name' => $this->getName(), 'phid' => $this->getPHID(), 'callsign' => $this->getCallsign(), 'monogram' => $this->getMonogram(), 'vcs' => $this->getVersionControlSystem(), 'uri' => PhabricatorEnv::getProductionURI($this->getURI()), 'remoteURI' => (string)$this->getRemoteURI(), 'description' => $this->getDetail('description'), 'isActive' => $this->isTracked(), 'isHosted' => $this->isHosted(), 'isImporting' => $this->isImporting(), 'encoding' => $this->getDefaultTextEncoding(), 'staging' => array( 'supported' => $this->supportsStaging(), 'prefix' => 'phabricator', 'uri' => $this->getStagingURI(), ), ); } public function getDefaultTextEncoding() { return $this->getDetail('encoding', 'UTF-8'); } public function getMonogram() { $callsign = $this->getCallsign(); if (strlen($callsign)) { return "r{$callsign}"; } $id = $this->getID(); return "R{$id}"; } public function getDisplayName() { $slug = $this->getRepositorySlug(); if (strlen($slug)) { return $slug; } return $this->getMonogram(); } public function getAllMonograms() { $monograms = array(); $monograms[] = 'R'.$this->getID(); $callsign = $this->getCallsign(); if (strlen($callsign)) { $monograms[] = 'r'.$callsign; } return $monograms; } public function setLocalPath($path) { // Convert any extra slashes ("//") in the path to a single slash ("/"). $path = preg_replace('(//+)', '/', $path); return parent::setLocalPath($path); } public function getDetail($key, $default = null) { return idx($this->details, $key, $default); } public function setDetail($key, $value) { $this->details[$key] = $value; return $this; } public function attachCommitCount($count) { $this->commitCount = $count; return $this; } public function getCommitCount() { return $this->assertAttached($this->commitCount); } public function attachMostRecentCommit( PhabricatorRepositoryCommit $commit = null) { $this->mostRecentCommit = $commit; return $this; } public function getMostRecentCommit() { return $this->assertAttached($this->mostRecentCommit); } public function getDiffusionBrowseURIForPath( PhabricatorUser $user, $path, $line = null, $branch = null) { $drequest = DiffusionRequest::newFromDictionary( array( 'user' => $user, 'repository' => $this, 'path' => $path, 'branch' => $branch, )); return $drequest->generateURI( array( 'action' => 'browse', 'line' => $line, )); } public function getSubversionBaseURI($commit = null) { $subpath = $this->getDetail('svn-subpath'); if (!strlen($subpath)) { $subpath = null; } return $this->getSubversionPathURI($subpath, $commit); } public function getSubversionPathURI($path = null, $commit = null) { $vcs = $this->getVersionControlSystem(); if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) { throw new Exception(pht('Not a subversion repository!')); } if ($this->isHosted()) { $uri = 'file://'.$this->getLocalPath(); } else { $uri = $this->getDetail('remote-uri'); } $uri = rtrim($uri, '/'); if (strlen($path)) { $path = rawurlencode($path); $path = str_replace('%2F', '/', $path); $uri = $uri.'/'.ltrim($path, '/'); } if ($path !== null || $commit !== null) { $uri .= '@'; } if ($commit !== null) { $uri .= $commit; } return $uri; } public function attachProjectPHIDs(array $project_phids) { $this->projectPHIDs = $project_phids; return $this; } public function getProjectPHIDs() { return $this->assertAttached($this->projectPHIDs); } /** * Get the name of the directory this repository should clone or checkout * into. For example, if the repository name is "Example Repository", a * reasonable name might be "example-repository". This is used to help users * get reasonable results when cloning repositories, since they generally do * not want to clone into directories called "X/" or "Example Repository/". * * @return string */ public function getCloneName() { $name = $this->getRepositorySlug(); // Make some reasonable effort to produce reasonable default directory // names from repository names. if (!strlen($name)) { $name = $this->getName(); $name = phutil_utf8_strtolower($name); $name = preg_replace('@[ -/:->]+@', '-', $name); $name = trim($name, '-'); if (!strlen($name)) { $name = $this->getCallsign(); } } return $name; } public static function isValidRepositorySlug($slug) { try { self::assertValidRepositorySlug($slug); return true; } catch (Exception $ex) { return false; } } public static function assertValidRepositorySlug($slug) { if (!strlen($slug)) { throw new Exception( pht( 'The empty string is not a valid repository short name. '. 'Repository short names must be at least one character long.')); } if (strlen($slug) > 64) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must not be longer than 64 characters.', $slug)); } if (preg_match('/[^a-zA-Z0-9._-]/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may only contain letters, numbers, periods, hyphens '. 'and underscores.', $slug)); } if (!preg_match('/^[a-zA-Z0-9]/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must begin with a letter or number.', $slug)); } if (!preg_match('/[a-zA-Z0-9]\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must end with a letter or number.', $slug)); } if (preg_match('/__|--|\\.\\./', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must not contain multiple consecutive underscores, '. 'hyphens, or periods.', $slug)); } if (preg_match('/^[A-Z]+\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may not contain only uppercase letters.', $slug)); } if (preg_match('/^\d+\z/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names may not contain only numbers.', $slug)); } if (preg_match('/\\.git/', $slug)) { throw new Exception( pht( 'The name "%s" is not a valid repository short name. Repository '. 'short names must not end in ".git". This suffix will be added '. 'automatically in appropriate contexts.', $slug)); } } public static function assertValidCallsign($callsign) { if (!strlen($callsign)) { throw new Exception( pht( 'A repository callsign must be at least one character long.')); } if (strlen($callsign) > 32) { throw new Exception( pht( 'The callsign "%s" is not a valid repository callsign. Callsigns '. 'must be no more than 32 bytes long.', $callsign)); } if (!preg_match('/^[A-Z]+\z/', $callsign)) { throw new Exception( pht( 'The callsign "%s" is not a valid repository callsign. Callsigns '. 'may only contain UPPERCASE letters.', $callsign)); } } public function getProfileImageURI() { return $this->getProfileImageFile()->getBestURI(); } public function attachProfileImageFile(PhabricatorFile $file) { $this->profileImageFile = $file; return $this; } public function getProfileImageFile() { return $this->assertAttached($this->profileImageFile); } /* -( Remote Command Execution )------------------------------------------- */ public function execRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args)->resolve(); } public function execxRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args)->resolvex(); } public function getRemoteCommandFuture($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandFuture($args); } public function passthruRemoteCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newRemoteCommandPassthru($args)->execute(); } private function newRemoteCommandFuture(array $argv) { return $this->newRemoteCommandEngine($argv) ->newFuture(); } private function newRemoteCommandPassthru(array $argv) { return $this->newRemoteCommandEngine($argv) ->setPassthru(true) ->newFuture(); } private function newRemoteCommandEngine(array $argv) { return DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->setCredentialPHID($this->getCredentialPHID()) ->setURI($this->getRemoteURIObject()); } /* -( Local Command Execution )-------------------------------------------- */ public function execLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args)->resolve(); } public function execxLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args)->resolvex(); } public function getLocalCommandFuture($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandFuture($args); } public function passthruLocalCommand($pattern /* , $arg, ... */) { $args = func_get_args(); return $this->newLocalCommandPassthru($args)->execute(); } private function newLocalCommandFuture(array $argv) { $this->assertLocalExists(); $future = DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); } return $future; } private function newLocalCommandPassthru(array $argv) { $this->assertLocalExists(); $future = DiffusionCommandEngine::newCommandEngine($this) ->setArgv($argv) ->setPassthru(true) ->newFuture(); if ($this->usesLocalWorkingCopy()) { $future->setCWD($this->getLocalPath()); } return $future; } public function getURI() { $short_name = $this->getRepositorySlug(); if (strlen($short_name)) { return "/source/{$short_name}/"; } $callsign = $this->getCallsign(); if (strlen($callsign)) { return "/diffusion/{$callsign}/"; } $id = $this->getID(); return "/diffusion/{$id}/"; } public function getPathURI($path) { return $this->getURI().ltrim($path, '/'); } public function getCommitURI($identifier) { $callsign = $this->getCallsign(); if (strlen($callsign)) { return "/r{$callsign}{$identifier}"; } $id = $this->getID(); return "/R{$id}:{$identifier}"; } public static function parseRepositoryServicePath($request_path, $vcs) { $is_git = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT); $patterns = array( '(^'. '(?P/?(?:diffusion|source)/(?P[^/]+))'. '(?P.*)'. '\z)', ); $identifier = null; foreach ($patterns as $pattern) { $matches = null; if (!preg_match($pattern, $request_path, $matches)) { continue; } $identifier = $matches['identifier']; if ($is_git) { $identifier = preg_replace('/\\.git\z/', '', $identifier); } $base = $matches['base']; $path = $matches['path']; break; } if ($identifier === null) { return null; } return array( 'identifier' => $identifier, 'base' => $base, 'path' => $path, ); } public function getCanonicalPath($request_path) { $standard_pattern = '(^'. '(?P/(?:diffusion|source)/)'. '(?P[^/]+)'. '(?P(?:/.*)?)'. '\z)'; $matches = null; if (preg_match($standard_pattern, $request_path, $matches)) { $suffix = $matches['suffix']; return $this->getPathURI($suffix); } $commit_pattern = '(^'. '(?P/)'. '(?P'. '(?:'. 'r(?P[A-Z]+)'. '|'. 'R(?P[1-9]\d*):'. ')'. '(?P[a-f0-9]+)'. ')'. '\z)'; $matches = null; if (preg_match($commit_pattern, $request_path, $matches)) { $commit = $matches['commit']; return $this->getCommitURI($commit); } return null; } public function generateURI(array $params) { $req_branch = false; $req_commit = false; $action = idx($params, 'action'); switch ($action) { case 'history': case 'clone': case 'blame': case 'browse': case 'document': case 'change': case 'lastmodified': case 'tags': case 'branches': case 'lint': case 'pathtree': case 'refs': case 'compare': break; case 'branch': // NOTE: This does not actually require a branch, and won't have one // in Subversion. Possibly this should be more clear. break; case 'commit': case 'rendering-ref': $req_commit = true; break; default: throw new Exception( pht( 'Action "%s" is not a valid repository URI action.', $action)); } $path = idx($params, 'path'); $branch = idx($params, 'branch'); $commit = idx($params, 'commit'); $line = idx($params, 'line'); $head = idx($params, 'head'); $against = idx($params, 'against'); if ($req_commit && !strlen($commit)) { throw new Exception( pht( 'Diffusion URI action "%s" requires commit!', $action)); } if ($req_branch && !strlen($branch)) { throw new Exception( pht( 'Diffusion URI action "%s" requires branch!', $action)); } if ($action === 'commit') { return $this->getCommitURI($commit); } if (strlen($path)) { $path = ltrim($path, '/'); $path = str_replace(array(';', '$'), array(';;', '$$'), $path); $path = phutil_escape_uri($path); } $raw_branch = $branch; if (strlen($branch)) { $branch = phutil_escape_uri_path_component($branch); $path = "{$branch}/{$path}"; } $raw_commit = $commit; if (strlen($commit)) { $commit = str_replace('$', '$$', $commit); $commit = ';'.phutil_escape_uri($commit); } if (strlen($line)) { $line = '$'.phutil_escape_uri($line); } $query = array(); switch ($action) { case 'change': case 'history': case 'blame': case 'browse': case 'document': case 'lastmodified': case 'tags': case 'branches': case 'lint': case 'pathtree': case 'refs': $uri = $this->getPathURI("/{$action}/{$path}{$commit}{$line}"); break; case 'compare': $uri = $this->getPathURI("/{$action}/"); if (strlen($head)) { $query['head'] = $head; } else if (strlen($raw_commit)) { $query['commit'] = $raw_commit; } else if (strlen($raw_branch)) { $query['head'] = $raw_branch; } if (strlen($against)) { $query['against'] = $against; } break; case 'branch': if (strlen($path)) { $uri = $this->getPathURI("/repository/{$path}"); } else { $uri = $this->getPathURI('/'); } break; case 'external': $commit = ltrim($commit, ';'); $uri = "/diffusion/external/{$commit}/"; break; case 'rendering-ref': // This isn't a real URI per se, it's passed as a query parameter to // the ajax changeset stuff but then we parse it back out as though // it came from a URI. $uri = rawurldecode("{$path}{$commit}"); break; case 'clone': $uri = $this->getPathURI("/{$action}/"); break; } if ($action == 'rendering-ref') { return $uri; } if (isset($params['lint'])) { $params['params'] = idx($params, 'params', array()) + array( 'lint' => $params['lint'], ); } $query = idx($params, 'params', array()) + $query; return new PhutilURI($uri, $query); } public function updateURIIndex() { $indexes = array(); $uris = $this->getURIs(); foreach ($uris as $uri) { if ($uri->getIsDisabled()) { continue; } $indexes[] = $uri->getNormalizedURI(); } PhabricatorRepositoryURIIndex::updateRepositoryURIs( $this->getPHID(), $indexes); return $this; } public function isTracked() { $status = $this->getDetail('tracking-enabled'); $map = self::getStatusMap(); $spec = idx($map, $status); if (!$spec) { if ($status) { $status = self::STATUS_ACTIVE; } else { $status = self::STATUS_INACTIVE; } $spec = idx($map, $status); } return (bool)idx($spec, 'isTracked', false); } public function getDefaultBranch() { $default = $this->getDetail('default-branch'); if (strlen($default)) { return $default; } $default_branches = array( PhabricatorRepositoryType::REPOSITORY_TYPE_GIT => 'master', PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL => 'default', ); return idx($default_branches, $this->getVersionControlSystem()); } public function getDefaultArcanistBranch() { return coalesce($this->getDefaultBranch(), 'svn'); } private function isBranchInFilter($branch, $filter_key) { $vcs = $this->getVersionControlSystem(); $is_git = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT); $use_filter = ($is_git); if (!$use_filter) { // If this VCS doesn't use filters, pass everything through. return true; } $filter = $this->getDetail($filter_key, array()); // If there's no filter set, let everything through. if (!$filter) { return true; } // If this branch isn't literally named `regexp(...)`, and it's in the // filter list, let it through. if (isset($filter[$branch])) { if (self::extractBranchRegexp($branch) === null) { return true; } } // If the branch matches a regexp, let it through. foreach ($filter as $pattern => $ignored) { $regexp = self::extractBranchRegexp($pattern); if ($regexp !== null) { if (preg_match($regexp, $branch)) { return true; } } } // Nothing matched, so filter this branch out. return false; } public static function extractBranchRegexp($pattern) { $matches = null; if (preg_match('/^regexp\\((.*)\\)\z/', $pattern, $matches)) { return $matches[1]; } return null; } public function shouldTrackRef(DiffusionRepositoryRef $ref) { // At least for now, don't track the staging area tags. if ($ref->isTag()) { if (preg_match('(^phabricator/)', $ref->getShortName())) { return false; } } if (!$ref->isBranch()) { return true; } return $this->shouldTrackBranch($ref->getShortName()); } public function shouldTrackBranch($branch) { return $this->isBranchInFilter($branch, 'branch-filter'); } public function isBranchPermanentRef($branch) { return $this->isBranchInFilter($branch, 'close-commits-filter'); } public function formatCommitName($commit_identifier, $local = false) { $vcs = $this->getVersionControlSystem(); $type_git = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; $type_hg = PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL; $is_git = ($vcs == $type_git); $is_hg = ($vcs == $type_hg); if ($is_git || $is_hg) { $name = substr($commit_identifier, 0, 12); $need_scope = false; } else { $name = $commit_identifier; $need_scope = true; } if (!$local) { $need_scope = true; } if ($need_scope) { $callsign = $this->getCallsign(); if ($callsign) { $scope = "r{$callsign}"; } else { $id = $this->getID(); $scope = "R{$id}:"; } $name = $scope.$name; } return $name; } public function isImporting() { return (bool)$this->getDetail('importing', false); } public function isNewlyInitialized() { return (bool)$this->getDetail('newly-initialized', false); } public function loadImportProgress() { $progress = queryfx_all( $this->establishConnection('r'), 'SELECT importStatus, count(*) N FROM %T WHERE repositoryID = %d GROUP BY importStatus', id(new PhabricatorRepositoryCommit())->getTableName(), $this->getID()); $done = 0; $total = 0; foreach ($progress as $row) { $total += $row['N'] * 3; $status = $row['importStatus']; if ($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE) { $done += $row['N']; } if ($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE) { $done += $row['N']; } if ($status & PhabricatorRepositoryCommit::IMPORTED_PUBLISH) { $done += $row['N']; } } if ($total) { $ratio = ($done / $total); } else { $ratio = 0; } // Cap this at "99.99%", because it's confusing to users when the actual // fraction is "99.996%" and it rounds up to "100.00%". if ($ratio > 0.9999) { $ratio = 0.9999; } return $ratio; } /* -( Publishing )--------------------------------------------------------- */ public function newPublisher() { return id(new PhabricatorRepositoryPublisher()) ->setRepository($this); } public function isPublishingDisabled() { return $this->getDetail('herald-disabled'); } public function getPermanentRefRules() { return array_keys($this->getDetail('close-commits-filter', array())); } public function setPermanentRefRules(array $rules) { $rules = array_fill_keys($rules, true); $this->setDetail('close-commits-filter', $rules); return $this; } public function getTrackOnlyRules() { return array_keys($this->getDetail('branch-filter', array())); } public function setTrackOnlyRules(array $rules) { $rules = array_fill_keys($rules, true); $this->setDetail('branch-filter', $rules); return $this; } public function supportsFetchRules() { if ($this->isGit()) { return true; } return false; } public function getFetchRules() { return $this->getDetail('fetch-rules', array()); } public function setFetchRules(array $rules) { return $this->setDetail('fetch-rules', $rules); } /* -( Repository URI Management )------------------------------------------ */ /** * Get the remote URI for this repository. * * @return string * @task uri */ public function getRemoteURI() { return (string)$this->getRemoteURIObject(); } /** * Get the remote URI for this repository, including credentials if they're * used by this repository. * * @return PhutilOpaqueEnvelope URI, possibly including credentials. * @task uri */ public function getRemoteURIEnvelope() { $uri = $this->getRemoteURIObject(); $remote_protocol = $this->getRemoteProtocol(); if ($remote_protocol == 'http' || $remote_protocol == 'https') { // For SVN, we use `--username` and `--password` flags separately, so // don't add any credentials here. if (!$this->isSVN()) { $credential_phid = $this->getCredentialPHID(); if ($credential_phid) { $key = PassphrasePasswordKey::loadFromPHID( $credential_phid, PhabricatorUser::getOmnipotentUser()); $uri->setUser($key->getUsernameEnvelope()->openEnvelope()); $uri->setPass($key->getPasswordEnvelope()->openEnvelope()); } } } return new PhutilOpaqueEnvelope((string)$uri); } /** * Get the clone (or checkout) URI for this repository, without authentication * information. * * @return string Repository URI. * @task uri */ public function getPublicCloneURI() { return (string)$this->getCloneURIObject(); } /** * Get the protocol for the repository's remote. * * @return string Protocol, like "ssh" or "git". * @task uri */ public function getRemoteProtocol() { $uri = $this->getRemoteURIObject(); return $uri->getProtocol(); } /** * Get a parsed object representation of the repository's remote URI.. * - * @return wild A @{class@libphutil:PhutilURI}. + * @return wild A @{class@arcanist:PhutilURI}. * @task uri */ public function getRemoteURIObject() { $raw_uri = $this->getDetail('remote-uri'); if (!strlen($raw_uri)) { return new PhutilURI(''); } if (!strncmp($raw_uri, '/', 1)) { return new PhutilURI('file://'.$raw_uri); } return new PhutilURI($raw_uri); } /** * Get the "best" clone/checkout URI for this repository, on any protocol. */ public function getCloneURIObject() { if (!$this->isHosted()) { if ($this->isSVN()) { // Make sure we pick up the "Import Only" path for Subversion, so // the user clones the repository starting at the correct path, not // from the root. $base_uri = $this->getSubversionBaseURI(); $base_uri = new PhutilURI($base_uri); $path = $base_uri->getPath(); if (!$path) { $path = '/'; } // If the trailing "@" is not required to escape the URI, strip it for // readability. if (!preg_match('/@.*@/', $path)) { $path = rtrim($path, '@'); } $base_uri->setPath($path); return $base_uri; } else { return $this->getRemoteURIObject(); } } // TODO: This should be cleaned up to deal with all the new URI handling. $another_copy = id(new PhabricatorRepositoryQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($this->getPHID())) ->needURIs(true) ->executeOne(); $clone_uris = $another_copy->getCloneURIs(); if (!$clone_uris) { return null; } return head($clone_uris)->getEffectiveURI(); } private function getRawHTTPCloneURIObject() { $uri = PhabricatorEnv::getProductionURI($this->getURI()); $uri = new PhutilURI($uri); if ($this->isGit()) { $uri->setPath($uri->getPath().$this->getCloneName().'.git'); } else if ($this->isHg()) { $uri->setPath($uri->getPath().$this->getCloneName().'/'); } return $uri; } /** * Determine if we should connect to the remote using SSH flags and * credentials. * * @return bool True to use the SSH protocol. * @task uri */ private function shouldUseSSH() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); if ($this->isSSHProtocol($protocol)) { return true; } return false; } /** * Determine if we should connect to the remote using HTTP flags and * credentials. * * @return bool True to use the HTTP protocol. * @task uri */ private function shouldUseHTTP() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); return ($protocol == 'http' || $protocol == 'https'); } /** * Determine if we should connect to the remote using SVN flags and * credentials. * * @return bool True to use the SVN protocol. * @task uri */ private function shouldUseSVNProtocol() { if ($this->isHosted()) { return false; } $protocol = $this->getRemoteProtocol(); return ($protocol == 'svn'); } /** * Determine if a protocol is SSH or SSH-like. * * @param string A protocol string, like "http" or "ssh". * @return bool True if the protocol is SSH-like. * @task uri */ private function isSSHProtocol($protocol) { return ($protocol == 'ssh' || $protocol == 'svn+ssh'); } public function delete() { $this->openTransaction(); $paths = id(new PhabricatorOwnersPath()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($paths as $path) { $path->delete(); } queryfx( $this->establishConnection('w'), 'DELETE FROM %T WHERE repositoryPHID = %s', id(new PhabricatorRepositorySymbol())->getTableName(), $this->getPHID()); $commits = id(new PhabricatorRepositoryCommit()) ->loadAllWhere('repositoryID = %d', $this->getID()); foreach ($commits as $commit) { // note PhabricatorRepositoryAuditRequests and // PhabricatorRepositoryCommitData are deleted here too. $commit->delete(); } $uris = id(new PhabricatorRepositoryURI()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($uris as $uri) { $uri->delete(); } $ref_cursors = id(new PhabricatorRepositoryRefCursor()) ->loadAllWhere('repositoryPHID = %s', $this->getPHID()); foreach ($ref_cursors as $cursor) { $cursor->delete(); } $conn_w = $this->establishConnection('w'); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_FILESYSTEM, $this->getID()); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_PATHCHANGE, $this->getID()); queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d', self::TABLE_SUMMARY, $this->getID()); $result = parent::delete(); $this->saveTransaction(); return $result; } public function isGit() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT); } public function isSVN() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN); } public function isHg() { $vcs = $this->getVersionControlSystem(); return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL); } public function isHosted() { return (bool)$this->getDetail('hosting-enabled', false); } public function setHosted($enabled) { return $this->setDetail('hosting-enabled', $enabled); } public function canServeProtocol( $protocol, $write, $is_intracluster = false) { // See T13192. If a repository is inactive, don't serve it to users. We // still synchronize it within the cluster and serve it to other repository // nodes. if (!$is_intracluster) { if (!$this->isTracked()) { return false; } } $clone_uris = $this->getCloneURIs(); foreach ($clone_uris as $uri) { if ($uri->getBuiltinProtocol() !== $protocol) { continue; } $io_type = $uri->getEffectiveIoType(); if ($io_type == PhabricatorRepositoryURI::IO_READWRITE) { return true; } if (!$write) { if ($io_type == PhabricatorRepositoryURI::IO_READ) { return true; } } } if ($write) { if ($this->isReadOnly()) { return false; } } return false; } public function hasLocalWorkingCopy() { try { self::assertLocalExists(); return true; } catch (Exception $ex) { return false; } } /** * Raise more useful errors when there are basic filesystem problems. */ private function assertLocalExists() { if (!$this->usesLocalWorkingCopy()) { return; } $local = $this->getLocalPath(); Filesystem::assertExists($local); Filesystem::assertIsDirectory($local); Filesystem::assertReadable($local); } /** * Determine if the working copy is bare or not. In Git, this corresponds * to `--bare`. In Mercurial, `--noupdate`. */ public function isWorkingCopyBare() { switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return false; case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $local = $this->getLocalPath(); if (Filesystem::pathExists($local.'/.git')) { return false; } else { return true; } } } public function usesLocalWorkingCopy() { switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->isHosted(); case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return true; } } public function getHookDirectories() { $directories = array(); if (!$this->isHosted()) { return $directories; } $root = $this->getLocalPath(); switch ($this->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: if ($this->isWorkingCopyBare()) { $directories[] = $root.'/hooks/pre-receive-phabricator.d/'; } else { $directories[] = $root.'/.git/hooks/pre-receive-phabricator.d/'; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $directories[] = $root.'/hooks/pre-commit-phabricator.d/'; break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // NOTE: We don't support custom Mercurial hooks for now because they're // messy and we can't easily just drop a `hooks.d/` directory next to // the hooks. break; } return $directories; } public function canDestroyWorkingCopy() { if ($this->isHosted()) { // Never destroy hosted working copies. return false; } $default_path = PhabricatorEnv::getEnvConfig( 'repository.default-local-path'); return Filesystem::isDescendant($this->getLocalPath(), $default_path); } public function canUsePathTree() { return !$this->isSVN(); } public function canUseGitLFS() { if (!$this->isGit()) { return false; } if (!$this->isHosted()) { return false; } if (!PhabricatorEnv::getEnvConfig('diffusion.allow-git-lfs')) { return false; } return true; } public function getGitLFSURI($path = null) { if (!$this->canUseGitLFS()) { throw new Exception( pht( 'This repository does not support Git LFS, so Git LFS URIs can '. 'not be generated for it.')); } $uri = $this->getRawHTTPCloneURIObject(); $uri = (string)$uri; $uri = $uri.'/'.$path; return $uri; } public function canMirror() { if ($this->isGit() || $this->isHg()) { return true; } return false; } public function canAllowDangerousChanges() { if (!$this->isHosted()) { return false; } // In Git and Mercurial, ref deletions and rewrites are dangerous. // In Subversion, editing revprops is dangerous. return true; } public function shouldAllowDangerousChanges() { return (bool)$this->getDetail('allow-dangerous-changes'); } public function canAllowEnormousChanges() { if (!$this->isHosted()) { return false; } return true; } public function shouldAllowEnormousChanges() { return (bool)$this->getDetail('allow-enormous-changes'); } public function writeStatusMessage( $status_type, $status_code, array $parameters = array()) { $table = new PhabricatorRepositoryStatusMessage(); $conn_w = $table->establishConnection('w'); $table_name = $table->getTableName(); if ($status_code === null) { queryfx( $conn_w, 'DELETE FROM %T WHERE repositoryID = %d AND statusType = %s', $table_name, $this->getID(), $status_type); } else { // If the existing message has the same code (e.g., we just hit an // error and also previously hit an error) we increment the message // count. This allows us to determine how many times in a row we've // run into an error. // NOTE: The assignments in "ON DUPLICATE KEY UPDATE" are evaluated // in order, so the "messageCount" assignment must occur before the // "statusCode" assignment. See T11705. queryfx( $conn_w, 'INSERT INTO %T (repositoryID, statusType, statusCode, parameters, epoch, messageCount) VALUES (%d, %s, %s, %s, %d, %d) ON DUPLICATE KEY UPDATE messageCount = IF( statusCode = VALUES(statusCode), messageCount + VALUES(messageCount), VALUES(messageCount)), statusCode = VALUES(statusCode), parameters = VALUES(parameters), epoch = VALUES(epoch)', $table_name, $this->getID(), $status_type, $status_code, json_encode($parameters), time(), 1); } return $this; } public static function assertValidRemoteURI($uri) { if (trim($uri) != $uri) { throw new Exception( pht('The remote URI has leading or trailing whitespace.')); } $uri_object = new PhutilURI($uri); $protocol = $uri_object->getProtocol(); // Catch confusion between Git/SCP-style URIs and normal URIs. See T3619 // for discussion. This is usually a user adding "ssh://" to an implicit // SSH Git URI. if ($protocol == 'ssh') { if (preg_match('(^[^:@]+://[^/:]+:[^\d])', $uri)) { throw new Exception( pht( "The remote URI is not formatted correctly. Remote URIs ". "with an explicit protocol should be in the form ". "'%s', not '%s'. The '%s' syntax is only valid in SCP-style URIs.", 'proto://domain/path', 'proto://domain:/path', ':/path')); } } switch ($protocol) { case 'ssh': case 'http': case 'https': case 'git': case 'svn': case 'svn+ssh': break; default: // NOTE: We're explicitly rejecting 'file://' because it can be // used to clone from the working copy of another repository on disk // that you don't normally have permission to access. throw new Exception( pht( 'The URI protocol is unrecognized. It should begin with '. '"%s", "%s", "%s", "%s", "%s", "%s", or be in the form "%s".', 'ssh://', 'http://', 'https://', 'git://', 'svn://', 'svn+ssh://', 'git@domain.com:path')); } return true; } /** * Load the pull frequency for this repository, based on the time since the * last activity. * * We pull rarely used repositories less frequently. This finds the most * recent commit which is older than the current time (which prevents us from * spinning on repositories with a silly commit post-dated to some time in * 2037). We adjust the pull frequency based on when the most recent commit * occurred. * * @param int The minimum update interval to use, in seconds. * @return int Repository update interval, in seconds. */ public function loadUpdateInterval($minimum = 15) { // First, check if we've hit errors recently. If we have, wait one period // for each consecutive error. Normally, this corresponds to a backoff of // 15s, 30s, 45s, etc. $message_table = new PhabricatorRepositoryStatusMessage(); $conn = $message_table->establishConnection('r'); $error_count = queryfx_one( $conn, 'SELECT MAX(messageCount) error_count FROM %T WHERE repositoryID = %d AND statusType IN (%Ls) AND statusCode IN (%Ls)', $message_table->getTableName(), $this->getID(), array( PhabricatorRepositoryStatusMessage::TYPE_INIT, PhabricatorRepositoryStatusMessage::TYPE_FETCH, ), array( PhabricatorRepositoryStatusMessage::CODE_ERROR, )); $error_count = (int)$error_count['error_count']; if ($error_count > 0) { return (int)($minimum * $error_count); } // If a repository is still importing, always pull it as frequently as // possible. This prevents us from hanging for a long time at 99.9% when // importing an inactive repository. if ($this->isImporting()) { return $minimum; } $window_start = (PhabricatorTime::getNow() + $minimum); $table = id(new PhabricatorRepositoryCommit()); $last_commit = queryfx_one( $table->establishConnection('r'), 'SELECT epoch FROM %T WHERE repositoryID = %d AND epoch <= %d ORDER BY epoch DESC LIMIT 1', $table->getTableName(), $this->getID(), $window_start); if ($last_commit) { $time_since_commit = ($window_start - $last_commit['epoch']); } else { // If the repository has no commits, treat the creation date as // though it were the date of the last commit. This makes empty // repositories update quickly at first but slow down over time // if they don't see any activity. $time_since_commit = ($window_start - $this->getDateCreated()); } $last_few_days = phutil_units('3 days in seconds'); if ($time_since_commit <= $last_few_days) { // For repositories with activity in the recent past, we wait one // extra second for every 10 minutes since the last commit. This // shorter backoff is intended to handle weekends and other short // breaks from development. $smart_wait = ($time_since_commit / 600); } else { // For repositories without recent activity, we wait one extra second // for every 4 minutes since the last commit. This longer backoff // handles rarely used repositories, up to the maximum. $smart_wait = ($time_since_commit / 240); } // We'll never wait more than 6 hours to pull a repository. $longest_wait = phutil_units('6 hours in seconds'); $smart_wait = min($smart_wait, $longest_wait); $smart_wait = max($minimum, $smart_wait); return (int)$smart_wait; } /** * Time limit for cloning or copying this repository. * * This limit is used to timeout operations like `git clone` or `git fetch` * when doing intracluster synchronization, building working copies, etc. * * @return int Maximum number of seconds to spend copying this repository. */ public function getCopyTimeLimit() { return $this->getDetail('limit.copy'); } public function setCopyTimeLimit($limit) { return $this->setDetail('limit.copy', $limit); } public function getDefaultCopyTimeLimit() { return phutil_units('15 minutes in seconds'); } public function getEffectiveCopyTimeLimit() { $limit = $this->getCopyTimeLimit(); if ($limit) { return $limit; } return $this->getDefaultCopyTimeLimit(); } public function getFilesizeLimit() { return $this->getDetail('limit.filesize'); } public function setFilesizeLimit($limit) { return $this->setDetail('limit.filesize', $limit); } public function getTouchLimit() { return $this->getDetail('limit.touch'); } public function setTouchLimit($limit) { return $this->setDetail('limit.touch', $limit); } /** * Retrieve the service URI for the device hosting this repository. * * See @{method:newConduitClient} for a general discussion of interacting * with repository services. This method provides lower-level resolution of * services, returning raw URIs. * * @param PhabricatorUser Viewing user. * @param map Constraints on selectable services. * @return string|null URI, or `null` for local repositories. */ public function getAlmanacServiceURI( PhabricatorUser $viewer, array $options) { $refs = $this->getAlmanacServiceRefs($viewer, $options); if (!$refs) { return null; } $ref = head($refs); return $ref->getURI(); } public function getAlmanacServiceRefs( PhabricatorUser $viewer, array $options) { PhutilTypeSpec::checkMap( $options, array( 'neverProxy' => 'bool', 'protocols' => 'list', 'writable' => 'optional bool', )); $never_proxy = $options['neverProxy']; $protocols = $options['protocols']; $writable = idx($options, 'writable', false); $cache_key = $this->getAlmanacServiceCacheKey(); if (!$cache_key) { return array(); } $cache = PhabricatorCaches::getMutableStructureCache(); $uris = $cache->getKey($cache_key, false); // If we haven't built the cache yet, build it now. if ($uris === false) { $uris = $this->buildAlmanacServiceURIs(); $cache->setKey($cache_key, $uris); } if ($uris === null) { return array(); } $local_device = AlmanacKeys::getDeviceID(); if ($never_proxy && !$local_device) { throw new Exception( pht( 'Unable to handle proxied service request. This device is not '. 'registered, so it can not identify local services. Register '. 'this device before sending requests here.')); } $protocol_map = array_fuse($protocols); $results = array(); foreach ($uris as $uri) { // If we're never proxying this and it's locally satisfiable, return // `null` to tell the caller to handle it locally. If we're allowed to // proxy, we skip this check and may proxy the request to ourselves. // (That proxied request will end up here with proxying forbidden, // return `null`, and then the request will actually run.) if ($local_device && $never_proxy) { if ($uri['device'] == $local_device) { return array(); } } if (isset($protocol_map[$uri['protocol']])) { $results[] = $uri; } } if (!$results) { throw new Exception( pht( 'The Almanac service for this repository is not bound to any '. 'interfaces which support the required protocols (%s).', implode(', ', $protocols))); } if ($never_proxy) { // See PHI1030. This error can arise from various device name/address // mismatches which are hard to detect, so try to provide as much // information as we can. if ($writable) { $request_type = pht('(This is a write request.)'); } else { $request_type = pht('(This is a read request.)'); } throw new Exception( pht( 'This repository request (for repository "%s") has been '. 'incorrectly routed to a cluster host (with device name "%s", '. 'and hostname "%s") which can not serve the request.'. "\n\n". 'The Almanac device address for the correct device may improperly '. 'point at this host, or the "device.id" configuration file on '. 'this host may be incorrect.'. "\n\n". 'Requests routed within the cluster by Phabricator are always '. 'expected to be sent to a node which can serve the request. To '. 'prevent loops, this request will not be proxied again.'. "\n\n". "%s", $this->getDisplayName(), $local_device, php_uname('n'), $request_type)); } if (count($results) > 1) { if (!$this->supportsSynchronization()) { throw new Exception( pht( 'Repository "%s" is bound to multiple active repository hosts, '. 'but this repository does not support cluster synchronization. '. 'Declusterize this repository or move it to a service with only '. 'one host.', $this->getDisplayName())); } } $refs = array(); foreach ($results as $result) { $refs[] = DiffusionServiceRef::newFromDictionary($result); } // If we require a writable device, remove URIs which aren't writable. if ($writable) { foreach ($refs as $key => $ref) { if (!$ref->isWritable()) { unset($refs[$key]); } } if (!$refs) { throw new Exception( pht( 'This repository ("%s") is not writable with the given '. 'protocols (%s). The Almanac service for this repository has no '. 'writable bindings that support these protocols.', $this->getDisplayName(), implode(', ', $protocols))); } } if ($writable) { $refs = $this->sortWritableAlmanacServiceRefs($refs); } else { $refs = $this->sortReadableAlmanacServiceRefs($refs); } return array_values($refs); } private function sortReadableAlmanacServiceRefs(array $refs) { assert_instances_of($refs, 'DiffusionServiceRef'); shuffle($refs); return $refs; } private function sortWritableAlmanacServiceRefs(array $refs) { assert_instances_of($refs, 'DiffusionServiceRef'); // See T13109 for discussion of how this method routes requests. // In the absence of other rules, we'll send traffic to devices randomly. // We also want to select randomly among nodes which are equally good // candidates to receive the write, and accomplish that by shuffling the // list up front. shuffle($refs); $order = array(); // If some device is currently holding the write lock, send all requests // to that device. We're trying to queue writes on a single device so they // do not need to wait for read synchronization after earlier writes // complete. $writer = PhabricatorRepositoryWorkingCopyVersion::loadWriter( $this->getPHID()); if ($writer) { $device_phid = $writer->getWriteProperty('devicePHID'); foreach ($refs as $key => $ref) { if ($ref->getDevicePHID() === $device_phid) { $order[] = $key; } } } // If no device is currently holding the write lock, try to send requests // to a device which is already up to date and will not need to synchronize // before it can accept the write. $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $this->getPHID()); if ($versions) { $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); $max_devices = array(); foreach ($versions as $version) { if ($version->getRepositoryVersion() == $max_version) { $max_devices[] = $version->getDevicePHID(); } } $max_devices = array_fuse($max_devices); foreach ($refs as $key => $ref) { if (isset($max_devices[$ref->getDevicePHID()])) { $order[] = $key; } } } // Reorder the results, putting any we've selected as preferred targets for // the write at the head of the list. $refs = array_select_keys($refs, $order) + $refs; return $refs; } public function supportsSynchronization() { // TODO: For now, this is only supported for Git. if (!$this->isGit()) { return false; } return true; } public function supportsRefs() { if ($this->isSVN()) { return false; } return true; } public function getAlmanacServiceCacheKey() { $service_phid = $this->getAlmanacServicePHID(); if (!$service_phid) { return null; } $repository_phid = $this->getPHID(); $parts = array( "repo({$repository_phid})", "serv({$service_phid})", 'v4', ); return implode('.', $parts); } private function buildAlmanacServiceURIs() { $service = $this->loadAlmanacService(); if (!$service) { return null; } $bindings = $service->getActiveBindings(); if (!$bindings) { throw new Exception( pht( 'The Almanac service for this repository is not bound to any '. 'interfaces.')); } $uris = array(); foreach ($bindings as $binding) { $iface = $binding->getInterface(); $uri = $this->getClusterRepositoryURIFromBinding($binding); $protocol = $uri->getProtocol(); $device_name = $iface->getDevice()->getName(); $device_phid = $iface->getDevice()->getPHID(); $uris[] = array( 'protocol' => $protocol, 'uri' => (string)$uri, 'device' => $device_name, 'writable' => (bool)$binding->getAlmanacPropertyValue('writable'), 'devicePHID' => $device_phid, ); } return $uris; } /** * Build a new Conduit client in order to make a service call to this * repository. * * If the repository is hosted locally, this method may return `null`. The * caller should use `ConduitCall` or other local logic to complete the * request. * * By default, we will return a @{class:ConduitClient} for any repository with * a service, even if that service is on the current device. * * We do this because this configuration does not make very much sense in a * production context, but is very common in a test/development context * (where the developer's machine is both the web host and the repository * service). By proxying in development, we get more consistent behavior * between development and production, and don't have a major untested * codepath. * * The `$never_proxy` parameter can be used to prevent this local proxying. * If the flag is passed: * * - The method will return `null` (implying a local service call) * if the repository service is hosted on the current device. * - The method will throw if it would need to return a client. * * This is used to prevent loops in Conduit: the first request will proxy, * even in development, but the second request will be identified as a * cluster request and forced not to proxy. * * For lower-level service resolution, see @{method:getAlmanacServiceURI}. * * @param PhabricatorUser Viewing user. * @param bool `true` to throw if a client would be returned. * @return ConduitClient|null Client, or `null` for local repositories. */ public function newConduitClient( PhabricatorUser $viewer, $never_proxy = false) { $uri = $this->getAlmanacServiceURI( $viewer, array( 'neverProxy' => $never_proxy, 'protocols' => array( 'http', 'https', ), // At least today, no Conduit call can ever write to a repository, // so it's fine to send anything to a read-only node. 'writable' => false, )); if ($uri === null) { return null; } $domain = id(new PhutilURI(PhabricatorEnv::getURI('/')))->getDomain(); $client = id(new ConduitClient($uri)) ->setHost($domain); if ($viewer->isOmnipotent()) { // If the caller is the omnipotent user (normally, a daemon), we will // sign the request with this host's asymmetric keypair. $public_path = AlmanacKeys::getKeyPath('device.pub'); try { $public_key = Filesystem::readFile($public_path); } catch (Exception $ex) { throw new PhutilAggregateException( pht( 'Unable to read device public key while attempting to make '. 'authenticated method call within the Phabricator cluster. '. 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex)); } $private_path = AlmanacKeys::getKeyPath('device.key'); try { $private_key = Filesystem::readFile($private_path); $private_key = new PhutilOpaqueEnvelope($private_key); } catch (Exception $ex) { throw new PhutilAggregateException( pht( 'Unable to read device private key while attempting to make '. 'authenticated method call within the Phabricator cluster. '. 'Use `%s` to register keys for this device. Exception: %s', 'bin/almanac register', $ex->getMessage()), array($ex)); } $client->setSigningKeys($public_key, $private_key); } else { // If the caller is a normal user, we generate or retrieve a cluster // API token. $token = PhabricatorConduitToken::loadClusterTokenForUser($viewer); if ($token) { $client->setConduitToken($token->getToken()); } } return $client; } public function newConduitClientForRequest(ConduitAPIRequest $request) { // Figure out whether we're going to handle this request on this device, // or proxy it to another node in the cluster. // If this is a cluster request and we need to proxy, we'll explode here // to prevent infinite recursion. $viewer = $request->getViewer(); $is_cluster_request = $request->getIsClusterRequest(); $client = $this->newConduitClient( $viewer, $is_cluster_request); return $client; } public function newConduitFuture( PhabricatorUser $viewer, $method, array $params, $never_proxy = false) { $client = $this->newConduitClient( $viewer, $never_proxy); if (!$client) { $result = id(new ConduitCall($method, $params)) ->setUser($viewer) ->execute(); $future = new ImmediateFuture($result); } else { $future = $client->callMethod($method, $params); } return $future; } public function getPassthroughEnvironmentalVariables() { $env = $_ENV; if ($this->isGit()) { // $_ENV does not populate in CLI contexts if "E" is missing from // "variables_order" in PHP config. Currently, we do not require this // to be configured. Since it may not be, explicitly bring expected Git // environmental variables into scope. This list is not exhaustive, but // only lists variables with a known impact on commit hook behavior. // This can be removed if we later require "E" in "variables_order". $git_env = array( 'GIT_OBJECT_DIRECTORY', 'GIT_ALTERNATE_OBJECT_DIRECTORIES', 'GIT_QUARANTINE_PATH', ); foreach ($git_env as $key) { $value = getenv($key); if (strlen($value)) { $env[$key] = $value; } } $key = 'GIT_PUSH_OPTION_COUNT'; $git_count = getenv($key); if (strlen($git_count)) { $git_count = (int)$git_count; $env[$key] = $git_count; for ($ii = 0; $ii < $git_count; $ii++) { $key = 'GIT_PUSH_OPTION_'.$ii; $env[$key] = getenv($key); } } } $result = array(); foreach ($env as $key => $value) { // In Git, pass anything matching "GIT_*" though. Some of these variables // need to be preserved to allow `git` operations to work properly when // running from commit hooks. if ($this->isGit()) { if (preg_match('/^GIT_/', $key)) { $result[$key] = $value; } } } return $result; } public function supportsBranchComparison() { return $this->isGit(); } public function isReadOnly() { return (bool)$this->getDetail('read-only'); } public function setReadOnly($read_only) { return $this->setDetail('read-only', $read_only); } public function getReadOnlyMessage() { return $this->getDetail('read-only-message'); } public function setReadOnlyMessage($message) { return $this->setDetail('read-only-message', $message); } public function getReadOnlyMessageForDisplay() { $parts = array(); $parts[] = pht( 'This repository is currently in read-only maintenance mode.'); $message = $this->getReadOnlyMessage(); if ($message !== null) { $parts[] = $message; } return implode("\n\n", $parts); } /* -( Repository URIs )---------------------------------------------------- */ public function attachURIs(array $uris) { $custom_map = array(); foreach ($uris as $key => $uri) { $builtin_key = $uri->getRepositoryURIBuiltinKey(); if ($builtin_key !== null) { $custom_map[$builtin_key] = $key; } } $builtin_uris = $this->newBuiltinURIs(); $seen_builtins = array(); foreach ($builtin_uris as $builtin_uri) { $builtin_key = $builtin_uri->getRepositoryURIBuiltinKey(); $seen_builtins[$builtin_key] = true; // If this builtin URI is disabled, don't attach it and remove the // persisted version if it exists. if ($builtin_uri->getIsDisabled()) { if (isset($custom_map[$builtin_key])) { unset($uris[$custom_map[$builtin_key]]); } continue; } // If the URI exists, make sure it's marked as not being disabled. if (isset($custom_map[$builtin_key])) { $uris[$custom_map[$builtin_key]]->setIsDisabled(false); } } // Remove any builtins which no longer exist. foreach ($custom_map as $builtin_key => $key) { if (empty($seen_builtins[$builtin_key])) { unset($uris[$key]); } } $this->uris = $uris; return $this; } public function getURIs() { return $this->assertAttached($this->uris); } public function getCloneURIs() { $uris = $this->getURIs(); $clone = array(); foreach ($uris as $uri) { if (!$uri->isBuiltin()) { continue; } if ($uri->getIsDisabled()) { continue; } $io_type = $uri->getEffectiveIoType(); $is_clone = ($io_type == PhabricatorRepositoryURI::IO_READ) || ($io_type == PhabricatorRepositoryURI::IO_READWRITE); if (!$is_clone) { continue; } $clone[] = $uri; } $clone = msort($clone, 'getURIScore'); $clone = array_reverse($clone); return $clone; } public function newBuiltinURIs() { $has_callsign = ($this->getCallsign() !== null); $has_shortname = ($this->getRepositorySlug() !== null); $identifier_map = array( PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_CALLSIGN => $has_callsign, PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_SHORTNAME => $has_shortname, PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_ID => true, ); // If the view policy of the repository is public, support anonymous HTTP // even if authenticated HTTP is not supported. if ($this->getViewPolicy() === PhabricatorPolicies::POLICY_PUBLIC) { $allow_http = true; } else { $allow_http = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth'); } $base_uri = PhabricatorEnv::getURI('/'); $base_uri = new PhutilURI($base_uri); $has_https = ($base_uri->getProtocol() == 'https'); $has_https = ($has_https && $allow_http); $has_http = !PhabricatorEnv::getEnvConfig('security.require-https'); $has_http = ($has_http && $allow_http); // HTTP is not supported for Subversion. if ($this->isSVN()) { $has_http = false; $has_https = false; } $has_ssh = (bool)strlen(PhabricatorEnv::getEnvConfig('phd.user')); $protocol_map = array( PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH => $has_ssh, PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTPS => $has_https, PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTP => $has_http, ); $uris = array(); foreach ($protocol_map as $protocol => $proto_supported) { foreach ($identifier_map as $identifier => $id_supported) { // This is just a dummy value because it can't be empty; we'll force // it to a proper value when using it in the UI. $builtin_uri = "{$protocol}://{$identifier}"; $uris[] = PhabricatorRepositoryURI::initializeNewURI() ->setRepositoryPHID($this->getPHID()) ->attachRepository($this) ->setBuiltinProtocol($protocol) ->setBuiltinIdentifier($identifier) ->setURI($builtin_uri) ->setIsDisabled((int)(!$proto_supported || !$id_supported)); } } return $uris; } public function getClusterRepositoryURIFromBinding( AlmanacBinding $binding) { $protocol = $binding->getAlmanacPropertyValue('protocol'); if ($protocol === null) { $protocol = 'https'; } $iface = $binding->getInterface(); $address = $iface->renderDisplayAddress(); $path = $this->getURI(); return id(new PhutilURI("{$protocol}://{$address}")) ->setPath($path); } public function loadAlmanacService() { $service_phid = $this->getAlmanacServicePHID(); if (!$service_phid) { // No service, so this is a local repository. return null; } $service = id(new AlmanacServiceQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($service_phid)) ->needBindings(true) ->needProperties(true) ->executeOne(); if (!$service) { throw new Exception( pht( 'The Almanac service for this repository is invalid or could not '. 'be loaded.')); } $service_type = $service->getServiceImplementation(); if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) { throw new Exception( pht( 'The Almanac service for this repository does not have the correct '. 'service type.')); } return $service; } public function markImporting() { $this->openTransaction(); $this->beginReadLocking(); $repository = $this->reload(); $repository->setDetail('importing', true); $repository->save(); $this->endReadLocking(); $this->saveTransaction(); return $repository; } /* -( Symbols )-------------------------------------------------------------*/ public function getSymbolSources() { return $this->getDetail('symbol-sources', array()); } public function getSymbolLanguages() { return $this->getDetail('symbol-languages', array()); } /* -( Staging )------------------------------------------------------------ */ public function supportsStaging() { return $this->isGit(); } public function getStagingURI() { if (!$this->supportsStaging()) { return null; } return $this->getDetail('staging-uri', null); } /* -( Automation )--------------------------------------------------------- */ public function supportsAutomation() { return $this->isGit(); } public function canPerformAutomation() { if (!$this->supportsAutomation()) { return false; } if (!$this->getAutomationBlueprintPHIDs()) { return false; } return true; } public function getAutomationBlueprintPHIDs() { if (!$this->supportsAutomation()) { return array(); } return $this->getDetail('automation.blueprintPHIDs', array()); } /* -( PhabricatorApplicationTransactionInterface )------------------------- */ public function getApplicationTransactionEditor() { return new PhabricatorRepositoryEditor(); } public function getApplicationTransactionTemplate() { return new PhabricatorRepositoryTransaction(); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, DiffusionPushCapability::CAPABILITY, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: return $this->getViewPolicy(); case PhabricatorPolicyCapability::CAN_EDIT: return $this->getEditPolicy(); case DiffusionPushCapability::CAPABILITY: return $this->getPushPolicy(); } } public function hasAutomaticCapability($capability, PhabricatorUser $user) { return false; } /* -( PhabricatorMarkupInterface )----------------------------------------- */ public function getMarkupFieldKey($field) { $hash = PhabricatorHash::digestForIndex($this->getMarkupText($field)); return "repo:{$hash}"; } public function newMarkupEngine($field) { return PhabricatorMarkupEngine::newMarkupEngine(array()); } public function getMarkupText($field) { return $this->getDetail('description'); } public function didMarkupText( $field, $output, PhutilMarkupEngine $engine) { require_celerity_resource('phabricator-remarkup-css'); return phutil_tag( 'div', array( 'class' => 'phabricator-remarkup', ), $output); } public function shouldUseMarkupCache($field) { return true; } /* -( PhabricatorDestructibleInterface )----------------------------------- */ public function destroyObjectPermanently( PhabricatorDestructionEngine $engine) { $phid = $this->getPHID(); $this->openTransaction(); $this->delete(); PhabricatorRepositoryURIIndex::updateRepositoryURIs($phid, array()); $books = id(new DivinerBookQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($books as $book) { $engine->destroyObject($book); } $atoms = id(new DivinerAtomQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($atoms as $atom) { $engine->destroyObject($atom); } $lfs_refs = id(new PhabricatorRepositoryGitLFSRefQuery()) ->setViewer($engine->getViewer()) ->withRepositoryPHIDs(array($phid)) ->execute(); foreach ($lfs_refs as $ref) { $engine->destroyObject($ref); } $this->saveTransaction(); } /* -( PhabricatorDestructibleCodexInterface )------------------------------ */ public function newDestructibleCodex() { return new PhabricatorRepositoryDestructibleCodex(); } /* -( PhabricatorSpacesInterface )----------------------------------------- */ public function getSpacePHID() { return $this->spacePHID; } /* -( PhabricatorConduitResultInterface )---------------------------------- */ public function getFieldSpecificationsForConduit() { return array( id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('name') ->setType('string') ->setDescription(pht('The repository name.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('vcs') ->setType('string') ->setDescription( pht('The VCS this repository uses ("git", "hg" or "svn").')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('callsign') ->setType('string') ->setDescription(pht('The repository callsign, if it has one.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('shortName') ->setType('string') ->setDescription(pht('Unique short name, if the repository has one.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('status') ->setType('string') ->setDescription(pht('Active or inactive status.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('isImporting') ->setType('bool') ->setDescription( pht( 'True if the repository is importing initial commits.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('almanacServicePHID') ->setType('phid?') ->setDescription( pht( 'The Almanac Service that hosts this repository, if the '. 'repository is clustered.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('refRules') ->setType('map>') ->setDescription( pht( 'The "Fetch" and "Permanent Ref" rules for this repository.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('defaultBranch') ->setType('string?') ->setDescription(pht('Default branch name.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('description') ->setType('remarkup') ->setDescription(pht('Repository description.')), ); } public function getFieldValuesForConduit() { $fetch_rules = $this->getFetchRules(); $track_rules = $this->getTrackOnlyRules(); $permanent_rules = $this->getPermanentRefRules(); $fetch_rules = $this->getStringListForConduit($fetch_rules); $track_rules = $this->getStringListForConduit($track_rules); $permanent_rules = $this->getStringListForConduit($permanent_rules); $default_branch = $this->getDefaultBranch(); if (!strlen($default_branch)) { $default_branch = null; } return array( 'name' => $this->getName(), 'vcs' => $this->getVersionControlSystem(), 'callsign' => $this->getCallsign(), 'shortName' => $this->getRepositorySlug(), 'status' => $this->getStatus(), 'isImporting' => (bool)$this->isImporting(), 'almanacServicePHID' => $this->getAlmanacServicePHID(), 'refRules' => array( 'fetchRules' => $fetch_rules, 'trackRules' => $track_rules, 'permanentRefRules' => $permanent_rules, ), 'defaultBranch' => $default_branch, 'description' => array( 'raw' => (string)$this->getDetail('description'), ), ); } private function getStringListForConduit($list) { if (!is_array($list)) { $list = array(); } foreach ($list as $key => $value) { $value = (string)$value; if (!strlen($value)) { unset($list[$key]); } } return array_values($list); } public function getConduitSearchAttachments() { return array( id(new DiffusionRepositoryURIsSearchEngineAttachment()) ->setAttachmentKey('uris'), id(new DiffusionRepositoryMetricsSearchEngineAttachment()) ->setAttachmentKey('metrics'), ); } /* -( PhabricatorFulltextInterface )--------------------------------------- */ public function newFulltextEngine() { return new PhabricatorRepositoryFulltextEngine(); } /* -( PhabricatorFerretInterface )----------------------------------------- */ public function newFerretEngine() { return new PhabricatorRepositoryFerretEngine(); } } diff --git a/src/docs/contributor/adding_new_classes.diviner b/src/docs/contributor/adding_new_classes.diviner index beb4567210..ea932eba5c 100644 --- a/src/docs/contributor/adding_new_classes.diviner +++ b/src/docs/contributor/adding_new_classes.diviner @@ -1,256 +1,256 @@ @title Adding New Classes @group developer Guide to adding new classes to extend Phabricator. Overview ======== Phabricator is highly modular, and many parts of it can be extended by adding new classes. This document explains how to write new classes to change or expand the behavior of Phabricator. IMPORTANT: The upstream does not offer support with extension development. Fundamentals ============ Phabricator primarily discovers functionality by looking at concrete subclasses of some base class. For example, Phabricator determines which applications are available by looking at all of the subclasses of @{class@phabricator:PhabricatorApplication}. It discovers available workflows in `arc` by looking at all of the subclasses of @{class@arcanist:ArcanistWorkflow}. It discovers available locales -by looking at all of the subclasses of @{class@libphutil:PhutilLocale}. +by looking at all of the subclasses of @{class@arcanist:PhutilLocale}. This pattern holds in many cases, so you can often add functionality by adding new classes with no other work. Phabricator will automatically discover and integrate the new capabilities or features at runtime. There are two main ways to add classes: - **Extensions Directory**: This is a simple way to add new code. It is less powerful, but takes a lot less work. This is good for quick changes, testing and development, or getting started on a larger project. - **Creating Libraries**: This is a more advanced and powerful way to organize extension code. This is better for larger or longer-lived projects, or any code which you plan to distribute. The next sections walk through these approaches in greater detail. Extensions Directory ==================== The easiest way to extend Phabricator by adding new classes is to drop them into the extensions directory, at `phabricator/src/extensions/`. This is intended as a quick way to add small pieces of functionality, test new features, or get started on a larger project. Extending Phabricator like this imposes a small performance penalty compared to using a library. -This directory exists in all libphutil libraries, so you can find similar -directories in `arcanist/src/extensions/` and `libphutil/src/extensions/`. +This directory exists in all libphutil libraries, so you can find a similar +directory in `arcanist/src/extensions/`. For example, to add a new application, create a file like this one and add it to `phabricator/src/extensions/`. ```name=phabricator/src/extensions/ExampleApplication.php, lang=php array( 'libcustom' => 'libcustom/src/', ), ... ``` Now, Phabricator will be able to load classes from your custom library. Writing Classes =============== To actually write classes, create a new module and put code in it: libcustom/ $ mkdir src/example/ libcustom/ $ nano src/example/ExampleClass.php # Edit some code. Now, run `arc liberate` to regenerate the static resource map: libcustom/ $ arc liberate src/ This will automatically regenerate the static map of the library. What You Can Extend And Invoke ============================== -libphutil, Arcanist and Phabricator are strict about extensibility of classes -and visibility of methods and properties. Most classes are marked `final`, and +Arcanist and Phabricator are strict about extensibility of classes and +visibility of methods and properties. Most classes are marked `final`, and methods have the minimum required visibility (protected or private). The goal of this strictness is to make it clear what you can safely extend, access, and invoke, so your code will keep working as the upstream changes. IMPORTANT: We'll still break APIs frequently. The upstream does not support extension development, and none of these APIs are stable. -When developing libraries to work with libphutil, Arcanist and Phabricator, you -should respect method and property visibility. +When developing libraries to work with Arcanist and Phabricator, you should +respect method and property visibility. If you want to add features but can't figure out how to do it without changing Phabricator code, here are some approaches you may be able to take: - {icon check, color=green} **Use Composition**: If possible, use composition rather than extension to build your feature. - {icon check, color=green} **Find Another Approach**: Check the documentation for a better way to accomplish what you're trying to do. - {icon check, color=green} **File a Feature Request**: Let us know what your use case is so we can make the class tree more flexible or configurable, or point you at the right way to do whatever you're trying to do, or explain why we don't let you do it. Note that we **do not support** extension development so you may have mixed luck with this one. These approaches are **discouraged**, but also possible: - {icon times, color=red} **Fork**: Create an ad-hoc local fork and remove `final` in your copy of the code. This will make it more difficult for you to upgrade in the future, although it may be the only real way forward depending on what you're trying to do. - {icon times, color=red} **Use Reflection**: You can use [[ http://php.net/manual/en/book.reflection.php | Reflection ]] to remove modifiers at runtime. This is fragile and discouraged, but technically possible. - {icon times, color=red} **Remove Modifiers**: Send us a patch removing `final` (or turning `protected` or `private` into `public`). We will almost never accept these patches unless there's a very good reason that the current behavior is wrong. Next Steps ========== Continue by: - visiting the [[ https://secure.phabricator.com/w/community_resources/ | Community Resources ]] page to find or share extensions and libraries. diff --git a/src/docs/contributor/bug_reports.diviner b/src/docs/contributor/bug_reports.diviner index fcf3eac5c5..59cb5fe715 100644 --- a/src/docs/contributor/bug_reports.diviner +++ b/src/docs/contributor/bug_reports.diviner @@ -1,173 +1,172 @@ @title Contributing Bug Reports @group detail Describes how to file an effective Phabricator bug report. Level Requirements ================== We accept bug reports through two channels: paid support and community support. If you are a paying customer, use the [[ https://admin.phacility.com/u/support | Support Channel ]] for your account to report bugs. This document may help you file reports which we can resolve more quickly, but you do not need to read it or follow the guidelines. Other users can follow the guidelines in this document to file bug reports on the community forum. Overview ======== This article describes how to file an effective Phabricator bug report. The most important things to do are: - check the list of common fixes below; - make sure Phabricator is up to date; - make sure we support your setup; - gather debugging information; and - explain how to reproduce the issue. The rest of this article walks through these points in detail. For general information on contributing to Phabricator, see @{article:Contributor Introduction}. Common Fixes ============ Before you file a report, here are some common solutions to problems: - **Update Phabricator**: We receive a lot of bug reports about issues we have already fixed in HEAD. Updating often resolves issues. It is common for issues to be fixed in less than 24 hours, so even if you've updated recently you should update again. If you aren't sure how to update, see the next section. - - **Update Libraries**: Make sure `libphutil/`, `arcanist/` and - `phabricator/` are all up to date. Users often update `phabricator/` but - forget to update `arcanist/` or `libphutil/`. When you update, make sure you - update all three libraries. + - **Update Libraries**: Make sure `arcanist/` and `phabricator/` are all up + to date. Users often update `phabricator/` but forget to update `arcanist/`. + When you update, make sure you update all three libraries. - **Restart Apache or PHP-FPM**: Phabricator uses caches which don't get reset until you restart Apache or PHP-FPM. After updating, make sure you restart. Update Phabricator ================== Before filing a bug, make sure you are up to date. We receive many bug reports for issues we have already fixed, and even if we haven't fixed an issue we'll be able to resolve it more easily if you file a report based on HEAD. (For example, an old stack trace may not have the right line numbers, which will make it more difficult for us to figure out what's going wrong.) To update Phabricator, use a script like the one described in @{article:Upgrading Phabricator}. **If you can not update** for some reason, please include the version of Phabricator you are running when you file a report. For help, see @{article:Providing Version Information}. Supported Issues ================ Before filing a bug, make sure you're filing an issue against something we support. **We can NOT help you with issues we can not reproduce.** It is critical that you explain how to reproduce the issue when filing a report. For help, see @{article:Providing Reproduction Steps}. **We do NOT support prototype applications.** If you're running into an issue with a prototype application, you're on your own. For more information about prototype applications, see @{article:User Guide: Prototype Applications}. **We do NOT support third-party packages or instructions.** If you installed Phabricator (or configured some aspect of it) using a third-party package or by following a third-party guide (like a blog post), we can not help you. Phabricator changes quickly and third-party information is unreliable and often falls out of date. Contact the maintainer of the package or guide you used, or reinstall following the upstream instructions. **We do NOT support custom code development or third-party libraries.** If you're writing an extension, you're on your own. We provide some documentation, but can not help you with extension or library development. If you downloaded a library from somewhere, contact the library maintainer. **We do NOT support bizarre environments.** If your issue is specific to an unusual installation environment, we generally will not help you find a workaround. Install Phabricator in a normal environment instead. Examples of unusual environments are shared hosts, nontraditional hosts (gaming consoles, storage appliances), and hosts with unusually tight resource constraints. The vast majority of users run Phabricator in normal environments (modern computers with root access) and these are the only environments we support. Otherwise, if you're having an issue with a supported first-party application and followed the upstream install instructions on a normal computer, we're happy to try to help. Getting More Information ======================== For some issues, there are places you can check for more information. This may help you resolve the issue yourself. Even if it doesn't, this information can help us figure out and resolve an issue. - For issues with `arc` or any other command-line script, you can get more details about what the script is doing by adding the `--trace` flag. - For issues with Phabricator, check your webserver error logs. - For Apache, this is often `/var/log/httpd/error.log`, or `/var/log/apache2/error.log` or similar. - For nginx, check both the nginx and php-fpm logs. - For issues with the UI, check the Javascript error console in your web browser. - Some other things, like daemons, have their own debug flags or troubleshooting steps. Check the documentation for information on troubleshooting. Adjusting settings or enabling debugging modes may give you more information about the issue. Reproducibility =============== The most important part of your report content is instructions on how to reproduce the issue. What did you do? If you do it again, does it still break? Does it depend on a specific browser? Can you reproduce the issue on a test instance on `admin.phabricator.com`? It is nearly impossible for us to resolve many issues if we can not reproduce them. We will not accept reports which do not contain the information required to reproduce problems. For help, see @{article:Providing Reproduction Steps}. File a Bug Report ================= If you're up to date, have collected information about the problem, and have the best reproduction instructions you can come up with, you're ready to file a report. It is **particularly critical** that you include reproduction steps. You can file a report on the community forum, here: (NOTE) https://discourse.phabricator-community.org/c/bug Next Steps ========== Continue by: - reading general support information in @{article:Support Resources}; or - returning to the @{article:Contributor Introduction}. diff --git a/src/docs/contributor/contrib_intro.diviner b/src/docs/contributor/contrib_intro.diviner index e298edc080..59ad9b44df 100644 --- a/src/docs/contributor/contrib_intro.diviner +++ b/src/docs/contributor/contrib_intro.diviner @@ -1,54 +1,54 @@ @title Contributor Introduction @group contrib -Introduction to contributing to Phabricator, Arcanist and libphutil. +Introduction to contributing to Phabricator and Arcanist. Overview ======== If you'd like to contribute to Phabricator, this document can guide you though ways you can help improve the project. Writing code is valuable, but often isn't the best or easiest way to contribute. In most cases we are pretty good at fixing easy stuff quickly, so we don't have a big pile of easy stuff sitting around waiting for new contributors. This can make it difficult to contribute code if you only have a little bit of time to spend since most of the work that needs to be done usually requires some heavy lifting. Without writing any code, learning the whole codebase, making a big time commitment, or having to touch PHP, here are some ways you can materially contribute to Phabricator: - Drop by the [[ https://phurl.io/u/discourse | community forum ]] just to say "thanks". A big part of the reason we build this software is to help people solve problems, and knowing that our efforts are appreciated is really rewarding. - Recommend Phabricator to people who you think might find it useful. Our most powerful growth channel is word of mouth, and mentioning or tweeting about Phabricator helps the project grow. If writing a tweet sounds like too much work, you can use one of these form tweets written by our PR department to quickly and easily shill on our behalf. Hail corporate! > Phabricator seems like it's pretty okay > I am not being paid to mention Phabricator in this extemporaneous, completely organic tweet > Phabricator is objectively the best thing. Source: I am a certified, internationally recognized expert. - Submit high-quality bug reports by carefully following the guide in @{article:Contributing Bug Reports}. If all of this sounds nice but you really just want to write some code, be aware that this project often presents a high barrier to entry for new contributors. To continue, see @{article:Contributing Code}. Next Steps ========== Continue by: - learning about bug reports in @{article:Contributing Bug Reports}; - learning about code contributions in @{article:Contributing Code}. diff --git a/src/docs/contributor/contributing_code.diviner b/src/docs/contributor/contributing_code.diviner index faaf5b16da..accb55c8d1 100644 --- a/src/docs/contributor/contributing_code.diviner +++ b/src/docs/contributor/contributing_code.diviner @@ -1,238 +1,238 @@ @title Contributing Code @group detail Describes how to contribute code to Phabricator. Level Requirements ================== To contribute to the Phabricator upstream, you must first pass a series of ancient trials and be invited to register an account in the ancestral homeland of Phabricator, here on `secure.phabricator.com`. The nature and location of these trials is a closely guarded secret. If you have passed these trials, this document can guide you through contributing code. If you have not yet passed these trials, writing code is normally not the best way to contribute to Phabricator. See @{article:Contributor Introduction} for more information. Overview ======== If you're planning to send a patch to Phabricator, this guide can help you through the process. The most important parts of contributing code to Phabricator are: - File a task with a bug report or feature request //before// you write code. - We rarely accept patches which we haven't discussed first. - We do not accept patches against prototype applications. - You must sign the CLA. - We do not accept GitHub pull requests. - Some alternative approaches are available if your change isn't something we want to bring upstream. The rest of this article describes these points in more detail, and then provides guidance on writing and submitting patches. If you just want to contribute some code but don't have a specific bug or feature in mind, see the bottom of this document for tips on finding ways to get started. For general information on contributing to Phabricator, see @{article:Contributor Introduction}. Coordinate First ================ Before sending code, you should file a task describing what you'd like to write. When you file a task, mention that you'd like to write the code to fix it. We can help contextualize your request or bug and guide you through writing an upstreamable patch, provided it's something that's upstreamable. If it isn't upstreamable, we can let you know what the issues are and help find another plan of attack. You don't have to file first (for example, if you spot a misspelling it's normally fine to just send a diff), but for anything even moderately complex you're strongly encouraged to file first and coordinate with the upstream. Rejecting Patches ================= If you send us a patch without coordinating it with us first, it will probably be immediately rejected, or sit in limbo for a long time and eventually be rejected. The reasons we do this vary from patch to patch, but some of the most common reasons are: **Unjustifiable Costs**: We support code in the upstream forever. Support is enormously expensive and takes up a huge amount of our time. The cost to support a change over its lifetime is often 10x or 100x or 1000x greater than the cost to write the first version of it. Many uncoordinated patches we receive are "white elephants", which would cost much more to maintain than the value they provide. As an author, it may look like you're giving us free work and we're rejecting it as too expensive, but this viewpoint doesn't align with the reality of a large project which is actively supported by a small, experienced team. Writing code is cheap; maintaining it is expensive. By coordinating with us first, you can make sure the patch is something we consider valuable enough to put long-term support resources behind, and that you're building it in a way that we're comfortable taking over. **Not a Good Fit**: Many patches aren't good fits for the upstream: they implement features we simply don't want. Coordinating with us first helps make sure we're on the same page and interested in a feature. The most common type of patch along these lines is a patch which adds new configuration options. We consider additional configuration options to have an exceptionally high lifetime support cost and are very unlikely to accept them. Coordinate with us first. **Not a Priority**: If you send us a patch against something which isn't a priority, we probably won't have time to look at it. We don't give special treatment to low-priority issues just because there's code written: we'd still be spending time on something lower-priority when we could be spending it on something higher-priority instead. If you coordinate with us first, you can make sure your patch is in an area of the codebase that we can prioritize. **Overly Ambitious Patches**: Sometimes we'll get huge patches from new contributors. These can have a lot of fundamental problems and require a huge amount of our time to review and correct. If you're interested in contributing, you'll have more success if you start small and learn as you go. We can help you break a large change into smaller pieces and learn how the codebase works as you proceed through the implementation, but only if you coordinate with us first. **Generality**: We often receive several feature requests which ask for similar features, and can come up with a general approach which covers all of the use cases. If you send us a patch for //your use case only//, the approach may be too specific. When a cleaner and more general approach is available, we usually prefer to pursue it. By coordinating with us first, we can make you aware of similar use cases and opportunities to generalize an approach. These changes are often small, but can have a big impact on how useful a piece of code is. **Infrastructure and Sequencing**: Sometimes patches are written against a piece of infrastructure with major planned changes. We don't want to accept these because they'll make the infrastructure changes more difficult to implement. Coordinate with us first to make sure a change doesn't need to wait on other pieces of infrastructure. We can help you identify technical blockers and possibly guide you through resolving them if you're interested. No Prototype Changes ==================== With rare exceptions, we do not accept patches for prototype applications for the same reasons that we don't accept feature requests or bug reports. To learn more about prototype applications, see @{article:User Guide: Prototype Applications}. You Must Sign the CLA ===================== Before we can accept source code contributions, you need to submit a [[ https://secure.phabricator.com/L28 | Contributor License Agreement ]]. Your changes can not be accepted until you sign the agreement. If you haven't signed it by the time you send changes for review, you'll be reminded to sign it at that time. If you're submitting work on behalf of a company (like your employer), the company can sign the [[ https://secure.phabricator.com/L30 | Corporate Contributor License Agreement ]] instead. Both agreements are substantially similar to the Apache Foundation's CLAs. They protect Phacility and users of Phabricator by making sure we have permission to distribute your changes under an open source license. No Pull Requests ================ We do not accept pull requests on GitHub: - We can not monitor who has signed CLAs on GitHub. You must sign the CLA to contribute, and we can't tell if you've signed it or not when you send us a pull request. - Pull requests do not get lint and unit tests run, so issues which are normally caught statically can slip by. - Phabricator is code review software, and developed using its own workflows. Pull requests bypass some of these workflows (for example, they will not trigger Herald rules to notify interested parties). - GitHub is not the authoritative master repository and we maintain a linear history, so merging pull requests is cumbersome on our end. - If you're comfortable enough with Phabricator to contribute to it, you should also be comfortable using it to submit changes. Instead of sending a pull request, use `arc diff` to create a revision on the upstream install. Your change will go through the normal Phabricator review process. (GitHub does not allow repositories to disable pull requests, which is why it's technically possible to submit them.) Alternatives ============ If you've written code but we're not accepting it into the upstream, some alternative approaches include: **Maintain a local fork.** This will require some ongoing effort to port your changes forward when you update, but is often very reasonable for simple changes. **Develop as an application.** Many parts of Phabricator's infrastructure are modular, and modularity is increasing over time. A lot of changes can be built as external modules or applications without forking Phabricator itself. There isn't much documentation or support for this right now, but you can look at how other applications are implemented, and at other third-party code that extends Phabricator. **Rise to prominence.** We're more willing to accept borderline changes from community members who are active, make multiple contributions, or have a history with the project. This is not carte blanche, but distinguishing yourself can make us feel more comfortable about supporting a change which is slightly outside of our comfort zone. Writing and Submitting Patches ================== -To actually submit a patch, run `arc diff` in `phabricator/`, `arcanist/`, or -`libphutil/`. When executed in these directories, `arc` should automatically -talk to the upstream install. You can add `epriestley` as a reviewer. +To actually submit a patch, run `arc diff` in `phabricator/` or `arcanist/`. +When executed in these directories, `arc` should automatically talk to the +upstream install. You can add `epriestley` as a reviewer. You should read the relevant coding convention documents before you submit a change. If you're a new contributor, you don't need to worry about this too much. Just try to make your code look similar to the code around it, and we can help you through the details during review. - @{article:General Coding Standards} (for all languages) - @{article:PHP Coding Standards} (for PHP) - @{article:Javascript Coding Standards} (for Javascript) In general, if you're coordinating with us first, we can usually provide guidance on how to implement things. The other articles in this section also provide information on how to work in the Phabricator codebase. Next Steps ========== Continue by: - returning to the @{article:Contributor Introduction}. diff --git a/src/docs/contributor/general_coding_standards.diviner b/src/docs/contributor/general_coding_standards.diviner index 45081231bb..9b151312fd 100644 --- a/src/docs/contributor/general_coding_standards.diviner +++ b/src/docs/contributor/general_coding_standards.diviner @@ -1,148 +1,148 @@ @title General Coding Standards @group standards This document is a general coding standard for contributing to Phabricator, -Arcanist, libphutil and Diviner. +Arcanist, and Diviner. = Overview = This document contains practices and guidelines which apply across languages. Contributors should follow these guidelines. These guidelines are not hard-and-fast but should be followed unless there is a compelling reason to deviate from them. = Code Complexity = - Prefer to write simple code which is easy to understand. The simplest code is not necessarily the smallest, and some changes which make code larger (such as decomposing complex expressions and choosing more descriptive names) may also make it simpler. Be willing to make size tradeoffs in favor of simplicity. - Prefer simple methods and functions which take a small number of parameters. Avoid methods and functions which are long and complex, or take an innumerable host of parameters. When possible, decompose monolithic, complex methods into several focused, simpler ones. - Avoid putting many ideas on a single line of code. For example, avoid this kind of code: COUNTEREXAMPLE $category_map = array_combine( $dates, array_map(create_function('$z', 'return date("F Y", $z);'), $dates)); Expressing this complex transformation more simply produces more readable code: $category_map = array(); foreach ($dates as $date) { $category_map[$date] = date('F Y', $date); } And, obviously, don't do this sort of thing: COUNTEREXAMPLE if ($val = $some->complicatedConstruct() && !!~blarg_blarg_blarg() & $flags ? HOPE_YOU_MEMORIZED == $all_the_lexical_binding_powers : <<<'Q' ${hahaha} Q ); = Performance = - Prefer to write efficient code. - Strongly prefer to drive optimization decisions with hard data. Avoid optimizing based on intuition or rumor if you can not support it with concrete measurements. - Prefer to optimize code which is slow and runs often. Optimizing code which is fast and runs rarely is usually a waste of time, and can even be harmful if it makes that code more difficult to understand or maintain. You can determine if code is fast or slow by measuring it. - Reject performance discussions that aren't rooted in concrete data. In Phabricator, you can usually use the builtin XHProf profiling to quickly gather concrete performance data. = Naming Things = - Follow language-specific conventions. - Name things unambiguously. - Choose descriptive names. - Avoid nonstandard abbreviations (common abbreviations like ID, URI and HTTP are fine). - Spell words correctly. - Use correct grammar. For example, avoid these sorts of naming choices: COUNTEREXAMPLE $PIE->GET_FLAVOR(); // Unconventional. $thing->doStuff(); // Ambiguous. $list->empty(); // Ambiguous -- is it isEmpty() or makeEmpty()? $e = 3; // Not descriptive. $this->updtHndlr(); // Nonstandard abbreviation. $this->chackSpulls(); // Misspelling, ungrammatical. Prefer these: $pie->getFlavor(); // Conventional. $pie->bake(); // Unambiguous. $list->isEmpty(); // Unambiguous. $list->makeEmpty(); // Unambiguous. $edge_count = 3; // Descriptive. $this->updateHandler(); // No nonstandard abbreviations. $this->getID(); // Standard abbreviation. $this->checkSpelling(); // Correct spelling and grammar. = Error Handling = - Strongly prefer to detect errors. - Strongly prefer to fail fast and loudly. The maximum cost of script termination is known, bounded, and fairly small. The maximum cost of continuing script execution when errors have occurred is unknown and unbounded. This also makes APIs much easier to use and problems far easier to debug. When you ignore errors, defer error handling, or degrade the severity of errors by treating them as warnings and then dismissing them, you risk dangerous behavior which may be difficult to troubleshoot: COUNTEREXAMPLE exec('echo '.$data.' > file.bak'); // Bad! do_something_dangerous(); exec('echo '.$data.' > file.bak', $out, $err); // Also bad! if ($err) { debug_rlog("Unable to copy file!"); } do_something_dangerous(); Instead, fail loudly: exec('echo '.$data.' > file.bak', $out, $err); // Better if ($err) { throw new Exception("Unable to copy file!"); } do_something_dangerous(); But the best approach is to use or write an API which simplifies condition handling and makes it easier to get right than wrong: execx('echo %s > file.bak', $data); // Good do_something_dangerous(); Filesystem::writeFile('file.bak', $data); // Best do_something_dangerous(); -See @{article@libphutil:Command Execution} for details on the APIs used in this +See @{article@arcanist:Command Execution} for details on the APIs used in this example. = Documentation, Comments and Formatting = - Prefer to remove code by deleting it over removing it by commenting it out. It shall live forever in source control, and can be retrieved therefrom if it is ever again called upon. - In source code, use only ASCII printable characters plus space and linefeed. Do not use UTF-8 or other multibyte encodings. diff --git a/src/docs/contributor/internationalization.diviner b/src/docs/contributor/internationalization.diviner index 9c78eb60fb..99c35e675e 100644 --- a/src/docs/contributor/internationalization.diviner +++ b/src/docs/contributor/internationalization.diviner @@ -1,381 +1,381 @@ @title Internationalization @group developer Describes Phabricator translation and localization. Overview ======== Phabricator partially supports internationalization, but many of the tools are missing or in a prototype state. This document describes what tools exist today, how to add new translations, and how to use the translation tools to make a codebase translatable. Adding a New Locale =================== To add a new locale, subclass @{class:PhutilLocale}. This allows you to introduce a new locale, like "German" or "Klingon". Once you've created a locale, applications can add translations for that locale. For instructions on adding new classes, see @{article@phabcontrib:Adding New Classes}. Adding Translations to Locale ============================= To translate strings, subclass @{class:PhutilTranslation}. Translations need to belong to a locale: the locale defines an available language, and each translation subclass provides strings for it. Translations are separated from locales so that third-party applications can provide translations into different locales without needing to define those locales themselves. For instructions on adding new classes, see @{article@phabcontrib:Adding New Classes}. Writing Translatable Code ========================= -Strings are marked for translation with @{function@libphutil:pht}. +Strings are marked for translation with @{function@arcanist:pht}. The `pht()` function takes a string (and possibly some parameters) and returns the translated version of that string in the current viewer's locale, if a translation is available. If text strings will ultimately be read by humans, they should essentially always be wrapped in `pht()`. For example: ```lang=php $dialog->appendParagraph(pht('This is an example.')); ``` This allows the code to return the correct Spanish or German or Russian version of the text, if the viewer is using Phabricator in one of those languages and a translation is available. Using `pht()` properly so that strings are translatable can be tricky. Briefly, the major rules are: - Only pass static strings as the first parameter to `pht()`. - Use parameters to create strings containing user names, object names, etc. - Translate full sentences, not sentence fragments. - Let the translation framework handle plural rules. - - Use @{class@libphutil:PhutilNumber} for numbers. + - Use @{class@arcanist:PhutilNumber} for numbers. - Let the translation framework handle subject gender rules. - Translate all human-readable text, even exceptions and error messages. See the next few sections for details on these rules. Use Static Strings ================== The first parameter to `pht()` must always be a static string. Broadly, this means it should not contain variables or function or method calls (it's OK to split it across multiple lines and concatenate the parts together). These are good: ```lang=php pht('The night is dark.'); pht( 'Two roads diverged in a yellow wood, '. 'and sorry I could not travel both '. 'and be one traveler, long I stood.'); ``` These won't work (they might appear to work, but are wrong): ```lang=php, counterexample pht(some_function()); pht('The duck says, '.$quack); pht($string); ``` The first argument must be a static string so it can be extracted by static analysis tools and dumped in a big file for translators. If it contains functions or variables, it can't be extracted, so translators won't be able to translate it. Lint will warn you about problems with use of static strings in calls to `pht()`. Parameters ========== You can provide parameters to a translation string by using `sprintf()`-style patterns in the input string. For example: ```lang=php pht('%s earned an award.', $actor); pht('%s closed %s.', $actor, $task); ``` This is primarily appropriate for usernames, object names, counts, and untranslatable strings like URIs or instructions to run commands from the CLI. Parameters normally should not be used to combine two pieces of translated text: see the next section for guidance. Sentence Fragments ================== You should almost always pass the largest block of text to `pht()` that you can. Particularly, it's important to pass complete sentences, not try to build a translation by stringing together sentence fragments. There are several reasons for this: - It gives translators more context, so they can be more confident they are producing a satisfying, natural-sounding translation which will make sense and sound good to native speakers. - In some languages, one fragment may need to translate differently depending on what the other fragment says. - In some languages, the most natural-sounding translation may change the order of words in the sentence. For example, suppose we want to translate these sentence to give the user some instructions about how to use an interface: > Turn the switch to the right. > Turn the switch to the left. > Turn the dial to the right. > Turn the dial to the left. Maybe we have a function like this: ``` function get_string($is_switch, $is_right) { // ... } ``` One way to write the function body would be like this: ```lang=php, counterexample $what = $is_switch ? pht('switch') : pht('dial'); $dir = $is_right ? pht('right') : pht('left'); return pht('Turn the ').$what.pht(' to the ').$dir.pht('.'); ``` This will work fine in English, but won't work well in other languages. One problem with doing this is handling gendered nouns. Languages like Spanish have gendered nouns, where some nouns are "masculine" and others are "feminine". The gender of a noun affects which article (in English, the word "the" is an article) should be used with it. In English, we say "**the** knob" and "**the** switch", but a Spanish speaker would say "**la** perilla" and "**el** interruptor", because the noun for "knob" in Spanish is feminine (so it is used with the article "la") while the noun for "switch" is masculine (so it is used with the article "el"). A Spanish speaker can not translate the string "Turn the" correctly without knowing which gender the noun has. Spanish has //two// translations for this string ("Gira el", "Gira la"), and the form depends on which noun is being used. Another problem is that this reduces flexibility. Translating fragments like this locks translators into a specific word order, when rearranging the words might make the sentence sound much more natural to a native speaker. For example, if the string read "The knob, to the right, turn it.", it would technically be English and most English readers would understand the meaning, but no native English speaker would speak or write like this. However, some languages have different subject-verb order rules or colloquialisms, and a word order which transliterates like this may sound more natural to a native speaker. By translating fragments instead of complete sentences, you lock translators into English word order. Finally, the last fragment is just a period. If a translator is presented with this string in an interface without much context, they have no hope of guessing how it is used in the software (it could be an end-of-sentence marker, or a decimal point, or a date separator, or a currency separator, all of which have very different translations in many locales). It will also conflict with all other translations of the same string in the codebase, so even if they are given context they can't translate it without technical problems. To avoid these issues, provide complete sentences for translation. This almost always takes the form of writing out alternatives in full. This is a good way to implement the example function: ```lang=php if ($is_switch) { if ($is_right) { return pht('Turn the switch to the right.'); } else { return pht('Turn the switch to the left.'); } } else { if ($is_right) { return pht('Turn the dial to the right.'); } else { return pht('Turn the dial to the left.'); } } ``` Although this is more verbose, translators can now get genders correct, rearrange word order, and have far more context when translating. This enables better, natural-sounding translations which are more satisfying to native speakers. Singular and Plural =================== Different languages have various rules for plural nouns. In English there are usually two plural noun forms: for one thing, and any other number of things. For example, we say that one chair is a "chair" and any other number of chairs are "chairs": "0 chairs", "1 chair", "2 chairs", etc. In other languages, there are different (and, in some cases, more) plural forms. For example, in Czech, there are separate forms for "one", "several", and "many". Because plural noun rules depend on the language, you should not write code which hard-codes English rules. For example, this won't translate well: ```lang=php, counterexample if ($count == 1) { return pht('This will take an hour.'); } else { return pht('This will take hours.'); } ``` This code is hard-coding the English rule for plural nouns. In languages like Czech, the correct word for "hours" may be different if the count is 2 or 15, but a translator won't be able to provide the correct translation if the string is written like this. Instead, pass a generic string to the translation engine which //includes// the number of objects, and let it handle plural nouns. This is the correct way to write the translation: ```lang=php return pht('This will take %s hour(s).', new PhutilNumber($count)); ``` If you now load the web UI, you'll see "hour(s)" literally in the UI. To fix this so the translation sounds better in English, provide translations for this string in the @{class@phabricator:PhabricatorUSEnglishTranslation} file: ```lang=php 'This will take %s hour(s).' => array( 'This will take an hour.', 'This will take hours.', ), ``` The string will then sound natural in English, but non-English translators will also be able to produce a natural translation. Note that the translations don't actually include the number in this case. The number is being passed from the code, but that just lets the translation engine get the rules right: the number does not need to appear in the final translations shown to the user. Using PhutilNumber ================== When translating numbers, you should almost always use `%s` and wrap the count or number in `new PhutilNumber($count)`. For example: ```lang=php pht('You have %s experience point(s).', new PhutilNumber($xp)); ``` This will let the translation engine handle plural noun rules correctly, and also format large numbers correctly in a locale-aware way with proper unit and decimal separators (for example, `1000000` may be printed as "1,000,000", with commas for readability). The exception to this rule is IDs which should not be written with unit separators. For example, this is correct for an object ID: ```lang=php pht('This diff has ID %d.', $diff->getID()); ``` Male and Female =============== Different languages also use different words for talking about subjects who are male, female or have an unknown gender. In English this is mostly just pronouns (like "he" and "she") but there are more complex rules in other languages, and languages like Czech also require verb agreement. When a parameter refers to a gendered person, pass an object which implements -@{interface@libphutil:PhutilPerson} to `pht()` so translators can provide +@{interface@arcanist:PhutilPerson} to `pht()` so translators can provide gendered translation variants. ```lang=php pht('%s wrote', $actor); ``` Translators will create these translations: ```lang=php // English translation '%s wrote'; // Czech translation array('%s napsal', '%s napsala'); ``` (You usually don't need to worry very much about this rule, it is difficult to get wrong in standard code.) Exceptions and Errors ===================== You should translate all human-readable text, even exceptions and error messages. This is primarily a rule of convenience which is straightforward and easy to follow, not a technical rule. Some exceptions and error messages don't //technically// need to be translated, as they will never be shown to a user, but many exceptions and error messages are (or will become) user-facing on some way. When writing a message, there is often no clear and objective way to determine which type of message you are writing. Rather than try to distinguish which are which, we simply translate all human-readable text. This rule is unambiguous and easy to follow. In cases where similar error or exception text is often repeated, it is probably appropriate to define an exception for that category of error rather than write the text out repeatedly, anyway. Two examples are -@{class@libphutil:PhutilInvalidStateException} and -@{class@libphutil:PhutilMethodNotImplementedException}, which mostly exist to +@{class@arcanist:PhutilInvalidStateException} and +@{class@arcanist:PhutilMethodNotImplementedException}, which mostly exist to produce a consistent message about a common error state in a convenient way. There are a handful of error strings in the codebase which may be used before the translation framework is loaded, or may be used during handling other errors, possibly raised from within the translation framework. This handful of special cases are left untranslated to prevent fatals and cycles in the error handler. Next Steps ========== Continue by: - adding a new locale or translation file with @{article@phabcontrib:Adding New Classes}. diff --git a/src/docs/contributor/php_coding_standards.diviner b/src/docs/contributor/php_coding_standards.diviner index ff62ebe98e..a14acf17f2 100644 --- a/src/docs/contributor/php_coding_standards.diviner +++ b/src/docs/contributor/php_coding_standards.diviner @@ -1,178 +1,178 @@ @title PHP Coding Standards @group standards This document describes PHP coding standards for Phabricator and related -projects (like Arcanist and libphutil). +projects (like Arcanist). = Overview = This document outlines technical and style guidelines which are followed in -libphutil. Contributors should also follow these guidelines. Many of these -guidelines are automatically enforced by lint. +Phabricator and Arcanist. Contributors should also follow these guidelines. +Many of these guidelines are automatically enforced by lint. These guidelines are essentially identical to the Facebook guidelines, since I basically copy-pasted them. If you are already familiar with the Facebook guidelines, you probably don't need to read this super thoroughly. = Spaces, Linebreaks and Indentation = - Use two spaces for indentation. Don't use tab literal characters. - Use Unix linebreaks ("\n"), not MSDOS ("\r\n") or OS9 ("\r"). - Put a space after control keywords like `if` and `for`. - Put a space after commas in argument lists. - Put a space around operators like `=`, `<`, etc. - Don't put spaces after function names. - Parentheses should hug their contents. - Generally, prefer to wrap code at 80 columns. = Case and Capitalization = - Name variables and functions using `lowercase_with_underscores`. - Name classes using `UpperCamelCase`. - Name methods and properties using `lowerCamelCase`. - Use uppercase for common acronyms like ID and HTML. - Name constants using `UPPERCASE`. - Write `true`, `false` and `null` in lowercase. = Comments = - Do not use "#" (shell-style) comments. - Prefer "//" comments inside function and method bodies. = PHP Language Style = - Use "" tag. - Prefer casts like `(string)` to casting functions like `strval()`. - Prefer type checks like `$v === null` to type functions like `is_null()`. - Avoid all crazy alternate forms of language constructs like "endwhile" and "<>". - Always put braces around conditional and loop blocks. = PHP Language Features = - Use PHP as a programming language, not a templating language. - Avoid globals. - Avoid extract(). - Avoid eval(). - Avoid variable variables. - Prefer classes over functions. - Prefer class constants over defines. - Avoid naked class properties; instead, define accessors. - Use exceptions for error conditions. - Use type hints, use `assert_instances_of()` for arrays holding objects. = Examples = **if/else:** lang=php if ($some_variable > 3) { // ... } else if ($some_variable === null) { // ... } else { // ... } You should always put braces around the body of an if clause, even if it is only one line long. Note spaces around operators and after control statements. Do not use the "endif" construct, and write "else if" as two words. **for:** lang=php for ($ii = 0; $ii < 10; $ii++) { // ... } Prefer $ii, $jj, $kk, etc., as iterators, since they're easier to pick out visually and react better to "Find Next..." in editors. **foreach:** lang=php foreach ($map as $key => $value) { // ... } **switch:** lang=php switch ($value) { case 1: // ... break; case 2: if ($flag) { // ... break; } break; default: // ... break; } `break` statements should be indented to block level. **array literals:** lang=php $junk = array( 'nuts', 'bolts', 'refuse', ); Use a trailing comma and put the closing parenthesis on a separate line so that diffs which add elements to the array affect only one line. **operators:** lang=php $a + $b; // Put spaces around operators. $omg.$lol; // Exception: no spaces around string concatenation. $arr[] = $element; // Couple [] with the array when appending. $obj = new Thing(); // Always use parens. **function/method calls:** lang=php // One line eject($cargo); // Multiline AbstractFireFactoryFactoryEngine::promulgateConflagrationInstance( $fuel, $ignition_source); **function/method definitions:** lang=php function example_function($base_value, $additional_value) { return $base_value + $additional_value; } class C { public static function promulgateConflagrationInstance( IFuel $fuel, IgnitionSource $source) { // ... } } **class:** lang=php class Dog extends Animal { const CIRCLES_REQUIRED_TO_LIE_DOWN = 3; private $favoriteFood = 'dirt'; public function getFavoriteFood() { return $this->favoriteFood; } } diff --git a/src/docs/contributor/rendering_html.diviner b/src/docs/contributor/rendering_html.diviner index 70401d8bcf..a8fe5a899d 100644 --- a/src/docs/contributor/rendering_html.diviner +++ b/src/docs/contributor/rendering_html.diviner @@ -1,182 +1,182 @@ @title Rendering HTML @group developer Rendering HTML in the Phabricator environment. = Overview = Phabricator attempts to prevent XSS by treating strings as default-unsafe when rendering. This means that if you try to build HTML through string concatenation, it won't work: the string will be escaped by the rendering pipeline, and the browser will treat it as plain text, not HTML. This document describes the right way to build HTML components so they are safe from XSS and render correctly. Broadly: - - Use @{function@libphutil:phutil_tag} (and @{function:javelin_tag}) to build + - Use @{function@arcanist:phutil_tag} (and @{function:javelin_tag}) to build tags. - - Use @{function@libphutil:hsprintf} where @{function@libphutil:phutil_tag} + - Use @{function@arcanist:hsprintf} where @{function@arcanist:phutil_tag} is awkward. - Combine elements with arrays, not string concatenation. - @{class:AphrontView} subclasses should return a - @{class@libphutil:PhutilSafeHTML} object from their `render()` method. + @{class@arcanist:PhutilSafeHTML} object from their `render()` method. - @{class:AphrontView} subclasses act like tags when rendering. - @{function:pht} has some special rules. - There are some other things that you should be aware of. See below for discussion. = Building Tags: phutil_tag() = -Build HTML tags with @{function@libphutil:phutil_tag}. For example: +Build HTML tags with @{function@arcanist:phutil_tag}. For example: phutil_tag( 'div', array( 'class' => 'some-class', ), $content); -@{function@libphutil:phutil_tag} will properly escape the content and all the -attributes, and return a @{class@libphutil:PhutilSafeHTML} object. The rendering +@{function@arcanist:phutil_tag} will properly escape the content and all the +attributes, and return a @{class@arcanist:PhutilSafeHTML} object. The rendering pipeline knows that this object represents a properly escaped HTML tag. This -allows @{function@libphutil:phutil_tag} to render tags with other tags as +allows @{function@arcanist:phutil_tag} to render tags with other tags as content correctly (without double-escaping): phutil_tag( 'div', array(), phutil_tag( 'strong', array(), $content)); In Phabricator, the @{function:javelin_tag} function is similar to -@{function@libphutil:phutil_tag}, but provides special handling for the +@{function@arcanist:phutil_tag}, but provides special handling for the `sigil` and `meta` attributes. = Building Blocks: hsprintf() = -Sometimes, @{function@libphutil:phutil_tag} can be particularly awkward to -use. You can use @{function@libphutil:hsprintf} to build larger and more -complex blocks of HTML, when @{function@libphutil:phutil_tag} is a poor fit. +Sometimes, @{function@arcanist:phutil_tag} can be particularly awkward to +use. You can use @{function@arcanist:hsprintf} to build larger and more +complex blocks of HTML, when @{function@arcanist:phutil_tag} is a poor fit. @{function:hsprintf} has `sprintf()` semantics, but `%s` escapes HTML: // Safely build fragments or unwieldy blocks. hsprintf( '
', $div_id); @{function:hsprintf} can be especially useful when: - You need to build a block with a lot of tags, like a table with rows and cells. - You need to build part of a tag (usually you should avoid this, but if you - do need to, @{function@libphutil:phutil_tag} can not do it). + do need to, @{function@arcanist:phutil_tag} can not do it). Note that it is unsafe to provide any user-controlled data to the first -parameter of @{function@libphutil:hsprintf} (the `sprintf()`-style pattern). +parameter of @{function@arcanist:hsprintf} (the `sprintf()`-style pattern). -Like @{function@libphutil:phutil_tag}, this function returns a -@{class@libphutil:PhutilSafeHTML} object. +Like @{function@arcanist:phutil_tag}, this function returns a +@{class@arcanist:PhutilSafeHTML} object. = Composing Tags = When you are building a view which combines smaller components, like a section with a header and a body: $header = phutil_tag('h1', ...); $body = phutil_tag('p', ...); ...you should NOT use string concatenation: COUNTEREXAMPLE // Not dangerous, but does the wrong thing. phutil_tag('div', array(), $header.$body); Instead, use an array: // Render a tag containing other tags safely. phutil_tag('div', array(), array($header, $body)); -If you concatenate @{class@libphutil:PhutilSafeHTML} objects, they revert to +If you concatenate @{class@arcanist:PhutilSafeHTML} objects, they revert to normal strings and are no longer marked as properly escaped tags. (In the future, these objects may stop converting to strings, but for now they must to maintain backward compatibility.) If you need to build a list of items with some element in between each of them (like a middot, comma, or vertical bar) you can use @{function:phutil_implode_html}: // Render links with commas between them. phutil_tag( 'div', array(), phutil_implode_html(', ', $list_of_links)); = AphrontView Classes = Subclasses of @{class:AphrontView} in Phabricator should return a -@{class@libphutil:PhutilSafeHTML} object. The easiest way to do this is to +@{class@arcanist:PhutilSafeHTML} object. The easiest way to do this is to return `phutil_tag()` or `javelin_tag()`: return phutil_tag('div', ...); You can use an @{class:AphrontView} subclass like you would a tag: phutil_tag('div', array(), $view); = Internationalization: pht() = The @{function:pht} function has some special rules. If any input to -@{function:pht} is a @{class@libphutil:PhutilSafeHTML} object, @{function:pht} -returns a @{class@libphutil:PhutilSafeHTML} object itself. Otherwise, it returns +@{function:pht} is a @{class@arcanist:PhutilSafeHTML} object, @{function:pht} +returns a @{class@arcanist:PhutilSafeHTML} object itself. Otherwise, it returns normal text. This is generally safe because translations are not permitted to have more tags than the original text did (so if the original text had no tags, translations can not add any). Normally, this just means that @{function:pht} does the right thing and behaves like you would expect, but it is worth being aware of. = Special Cases = NOTE: This section describes dangerous methods which can bypass XSS protections. If possible, do not use them. -You can build @{class@libphutil:PhutilSafeHTML} out of a string explicitly by +You can build @{class@arcanist:PhutilSafeHTML} out of a string explicitly by calling @{function:phutil_safe_html} on it. This is **dangerous**, because if you are wrong and the string is not actually safe, you have introduced an XSS vulnerability. Consequently, you should avoid calling this if possible. -You can use @{function@libphutil:phutil_escape_html_newlines} to escape HTML +You can use @{function@arcanist:phutil_escape_html_newlines} to escape HTML while converting newlines to `
`. You should not need to explicitly use -@{function@libphutil:phutil_escape_html} anywhere. +@{function@arcanist:phutil_escape_html} anywhere. If you need to apply a string function (such as `trim()`) to safe HTML, use -@{method@libphutil:PhutilSafeHTML::applyFunction}. +@{method@arcanist:PhutilSafeHTML::applyFunction}. -If you need to extract the content of a @{class@libphutil:PhutilSafeHTML} +If you need to extract the content of a @{class@arcanist:PhutilSafeHTML} object, you should call `getHTMLContent()`, not cast it to a string. Eventually, we would like to remove the string cast entirely. -Functions @{function@libphutil:phutil_tag} and @{function@libphutil:hsprintf} +Functions @{function@arcanist:phutil_tag} and @{function@arcanist:hsprintf} are not safe if you pass the user input for the tag or attribute name. All the following examples are dangerous: counterexample phutil_tag($evil); phutil_tag('span', array($evil => $evil2)); phutil_tag('span', array('onmouseover' => $evil)); // Use PhutilURI to check if $evil is valid HTTP link. hsprintf('', $evil); hsprintf('<%s>%s', $evil, $evil2, $evil); // We have a lint rule disallowing this. hsprintf($evil); diff --git a/src/docs/contributor/unit_tests.diviner b/src/docs/contributor/unit_tests.diviner index 3ac14b3e00..7977a4a876 100644 --- a/src/docs/contributor/unit_tests.diviner +++ b/src/docs/contributor/unit_tests.diviner @@ -1,86 +1,86 @@ @title Writing Unit Tests @group developer -Simple guide to libphutil, Arcanist and Phabricator unit tests. +Simple guide to Arcanist and Phabricator unit tests. = Overview = -libphutil, Arcanist and Phabricator provide and use a simple unit test -framework. This document is aimed at project contributors and describes how to -use it to add and run tests in these projects or other libphutil libraries. +Arcanist and Phabricator provide and use a simple unit test framework. This +document is aimed at project contributors and describes how to use it to add +and run tests in these projects or other libphutil libraries. In the general case, you can integrate `arc` with a custom unit test engine (like PHPUnit or any other unit testing library) to run tests in other projects. See @{article:Arcanist User Guide: Customizing Lint, Unit Tests and Workflows} for information on customizing engines. = Adding Tests = -To add new tests to a libphutil, Arcanist or Phabricator module: +To add new tests to a Arcanist or Phabricator module: - Create a `__tests__/` directory in the module if it doesn't exist yet. - Add classes to the `__tests__/` directory which extend from @{class:PhabricatorTestCase} (in Phabricator) or @{class@arcanist:PhutilTestCase} (elsewhere). - Run `arc liberate` on the library root so your classes are loadable. = Running Tests = Once you've added test classes, you can run them with: - `arc unit path/to/module/`, to explicitly run module tests. - `arc unit`, to run tests for all modules affected by changes in the working copy. - `arc diff` will also run `arc unit` for you. = Example Test Case = Here's a simple example test: lang=php class PhabricatorTrivialTestCase extends PhabricatorTestCase { private $two; public function willRunOneTest($test_name) { // You can execute setup steps which will run before each test in this // method. $this->two = 2; } public function testAllIsRightWithTheWorld() { $this->assertEqual(4, $this->two + $this->two, '2 + 2 = 4'); } } You can see this class at @{class:PhabricatorTrivialTestCase} and run it with: phabricator/ $ arc unit src/infrastructure/testing/testcase/ PASS <1ms* testAllIsRightWithTheWorld For more information on writing tests, see @{class@arcanist:PhutilTestCase} and @{class:PhabricatorTestCase}. = Database Isolation = By default, Phabricator isolates unit tests from the database. It makes a crude effort to simulate some side effects (principally, ID assignment on insert), but any queries which read data will fail to select any rows and throw an exception about isolation. In general, isolation is good, but this can make certain types of tests difficult to write. When you encounter issues, you can deal with them in a number of ways. From best to worst: - Encounter no issues; your tests are fast and isolated. - Add more simulated side effects if you encounter minor issues and simulation is reasonable. - Build a real database simulation layer (fairly complex). - Disable isolation for a single test by using `LiskDAO::endIsolateAllLiskEffectsToCurrentProcess();` before your test and `LiskDAO::beginIsolateAllLiskEffectsToCurrentProcess();` after your test. This will disable isolation for one test. NOT RECOMMENDED. - Disable isolation for your entire test case by overriding `getPhabricatorTestCaseConfiguration()` and providing `self::PHABRICATOR_TESTCONFIG_ISOLATE_LISK => false` in the configuration dictionary you return. This will disable isolation entirely. STRONGLY NOT RECOMMENDED. diff --git a/src/docs/flavor/php_pitfalls.diviner b/src/docs/flavor/php_pitfalls.diviner index 0ffcaa42da..3f4be45dd7 100644 --- a/src/docs/flavor/php_pitfalls.diviner +++ b/src/docs/flavor/php_pitfalls.diviner @@ -1,329 +1,329 @@ @title PHP Pitfalls @group php This document discusses difficult traps and pitfalls in PHP, and how to avoid, work around, or at least understand them. = `array_merge()` in Incredibly Slow When Merging A List of Arrays = If you merge a list of arrays like this: COUNTEREXAMPLE, lang=php $result = array(); foreach ($list_of_lists as $one_list) { $result = array_merge($result, $one_list); } ...your program now has a huge runtime because it generates a large number of intermediate arrays and copies every element it has previously seen each time you iterate. -In a libphutil environment, you can use @{function@libphutil:array_mergev} +In a libphutil environment, you can use @{function@arcanist:array_mergev} instead. = `var_export()` Hates Baby Animals = If you try to `var_export()` an object that contains recursive references, your program will terminate. You have no chance to intercept or react to this or otherwise stop it from happening. Avoid `var_export()` unless you are certain you have only simple data. You can use `print_r()` or `var_dump()` to display complex variables safely. = `isset()`, `empty()` and Truthiness = A value is "truthy" if it evaluates to true in an `if` clause: lang=php $value = something(); if ($value) { // Value is truthy. } If a value is not truthy, it is "falsey". These values are falsey in PHP: null // null 0 // integer 0.0 // float "0" // string "" // empty string false // boolean array() // empty array Disregarding some bizarre edge cases, all other values are truthy. Note that because "0" is falsey, this sort of thing (intended to prevent users from making empty comments) is wrong in PHP: COUNTEREXAMPLE if ($comment_text) { make_comment($comment_text); } This is wrong because it prevents users from making the comment "0". //THIS COMMENT IS TOTALLY AWESOME AND I MAKE IT ALL THE TIME SO YOU HAD BETTER NOT BREAK IT!!!// A better test is probably `strlen()`. In addition to truth tests with `if`, PHP has two special truthiness operators which look like functions but aren't: `empty()` and `isset()`. These operators help deal with undeclared variables. In PHP, there are two major cases where you get undeclared variables -- either you directly use a variable without declaring it: COUNTEREXAMPLE, lang=php function f() { if ($not_declared) { // ... } } ...or you index into an array with an index which may not exist: COUNTEREXAMPLE function f(array $mystery) { if ($mystery['stuff']) { // ... } } When you do either of these, PHP issues a warning. Avoid these warnings by using `empty()` and `isset()` to do tests that are safe to apply to undeclared variables. `empty()` evaluates truthiness exactly opposite of `if()`. `isset()` returns `true` for everything except `null`. This is the truth table: | Value | `if()` | `empty()` | `isset()` | |-------|--------|-----------|-----------| | `null` | `false` | `true` | `false` | | `0` | `false` | `true` | `true` | | `0.0` | `false` | `true` | `true` | | `"0"` | `false` | `true` | `true` | | `""` | `false` | `true` | `true` | | `false` | `false` | `true` | `true` | | `array()` | `false` | `true` | `true` | | Everything else | `true` | `false` | `true` | The value of these operators is that they accept undeclared variables and do not issue a warning. Specifically, if you try to do this you get a warning: ```lang=php, COUNTEREXAMPLE if ($not_previously_declared) { // PHP Notice: Undefined variable! // ... } ``` But these are fine: ```lang=php if (empty($not_previously_declared)) { // No notice, returns true. // ... } if (isset($not_previously_declared)) { // No notice, returns false. // ... } ``` So, `isset()` really means `is_declared_and_is_set_to_something_other_than_null()`. `empty()` really means `is_falsey_or_is_not_declared()`. Thus: - If a variable is known to exist, test falsiness with `if (!$v)`, not `empty()`. In particular, test for empty arrays with `if (!$array)`. There is no reason to ever use `empty()` on a declared variable. - When you use `isset()` on an array key, like `isset($array['key'])`, it will evaluate to "false" if the key exists but has the value `null`! Test for index existence with `array_key_exists()`. Put another way, use `isset()` if you want to type `if ($value !== null)` but are testing something that may not be declared. Use `empty()` if you want to type `if (!$value)` but you are testing something that may not be declared. = usort(), uksort(), and uasort() are Slow = This family of functions is often extremely slow for large datasets. You should avoid them if at all possible. Instead, build an array which contains surrogate keys that are naturally sortable with a function that uses native comparison (e.g., `sort()`, `asort()`, `ksort()`, or `natcasesort()`). Sort this array instead, and use it to reorder the original array. In a libphutil environment, you can often do this easily with -@{function@libphutil:isort} or @{function@libphutil:msort}. +@{function@arcanist:isort} or @{function@arcanist:msort}. = `array_intersect()` and `array_diff()` are Also Slow = These functions are much slower for even moderately large inputs than `array_intersect_key()` and `array_diff_key()`, because they can not make the assumption that their inputs are unique scalars as the `key` varieties can. Strongly prefer the `key` varieties. = `array_uintersect()` and `array_udiff()` are Definitely Slow Too = These functions have the problems of both the `usort()` family and the `array_diff()` family. Avoid them. = `foreach()` Does Not Create Scope = Variables survive outside of the scope of `foreach()`. More problematically, references survive outside of the scope of `foreach()`. This code mutates `$array` because the reference leaks from the first loop to the second: ```lang=php, COUNTEREXAMPLE $array = range(1, 3); echo implode(',', $array); // Outputs '1,2,3' foreach ($array as &$value) {} echo implode(',', $array); // Outputs '1,2,3' foreach ($array as $value) {} echo implode(',', $array); // Outputs '1,2,2' ``` The easiest way to avoid this is to avoid using foreach-by-reference. If you do use it, unset the reference after the loop: ```lang=php foreach ($array as &$value) { // ... } unset($value); ``` = `unserialize()` is Incredibly Slow on Large Datasets = The performance of `unserialize()` is nonlinear in the number of zvals you unserialize, roughly `O(N^2)`. | zvals | Approximate time | |-------|------------------| | 10000 |5ms | | 100000 | 85ms | | 1000000 | 8,000ms | | 10000000 | 72 billion years | = `call_user_func()` Breaks References = If you use `call_use_func()` to invoke a function which takes parameters by reference, the variables you pass in will have their references broken and will emerge unmodified. That is, if you have a function that takes references: ```lang=php function add_one(&$v) { $v++; } ``` ...and you call it with `call_user_func()`: ```lang=php, COUNTEREXAMPLE $x = 41; call_user_func('add_one', $x); ``` ...`$x` will not be modified. The solution is to use `call_user_func_array()` and wrap the reference in an array: ```lang=php $x = 41; call_user_func_array( 'add_one', array(&$x)); // Note '&$x'! ``` This will work as expected. = You Can't Throw From `__toString()` = If you throw from `__toString()`, your program will terminate uselessly and you won't get the exception. = An Object Can Have Any Scalar as a Property = Object properties are not limited to legal variable names: ```lang=php $property = '!@#$%^&*()'; $obj->$property = 'zebra'; echo $obj->$property; // Outputs 'zebra'. ``` So, don't make assumptions about property names. = There is an `(object)` Cast = You can cast a dictionary into an object. ```lang=php $obj = (object)array('flavor' => 'coconut'); echo $obj->flavor; // Outputs 'coconut'. echo get_class($obj); // Outputs 'stdClass'. ``` This is occasionally useful, mostly to force an object to become a Javascript dictionary (vs a list) when passed to `json_encode()`. = Invoking `new` With an Argument Vector is Really Hard = If you have some `$class_name` and some `$argv` of constructor arguments and you want to do this: ```lang=php new $class_name($argv[0], $argv[1], ...); ``` ...you'll probably invent a very interesting, very novel solution that is very wrong. In a libphutil environment, solve this problem with -@{function@libphutil:newv}. Elsewhere, copy `newv()`'s implementation. +@{function@arcanist:newv}. Elsewhere, copy `newv()`'s implementation. = Equality is not Transitive = This isn't terribly surprising since equality isn't transitive in a lot of languages, but the `==` operator is not transitive: ```lang=php $a = ''; $b = 0; $c = '0a'; $a == $b; // true $b == $c; // true $c == $a; // false! ``` When either operand is an integer, the other operand is cast to an integer before comparison. Avoid this and similar pitfalls by using the `===` operator, which is transitive. = All 676 Letters in the Alphabet = This doesn't do what you'd expect it to do in C: ```lang=php for ($c = 'a'; $c <= 'z'; $c++) { // ... } ``` This is because the successor to `z` is `aa`, which is "less than" `z`. The loop will run for ~700 iterations until it reaches `zz` and terminates. That is, `$c` will take on these values: ``` a b ... y z aa // loop continues because 'aa' <= 'z' ab ... mf mg ... zw zx zy zz // loop now terminates because 'zz' > 'z' ``` Instead, use this loop: ```lang=php foreach (range('a', 'z') as $c) { // ... } ``` diff --git a/src/docs/user/configuration/managing_daemons.diviner b/src/docs/user/configuration/managing_daemons.diviner index 0a732d5836..cf2ba85ea2 100644 --- a/src/docs/user/configuration/managing_daemons.diviner +++ b/src/docs/user/configuration/managing_daemons.diviner @@ -1,131 +1,131 @@ @title Managing Daemons with phd @group config Explains Phabricator daemons and the daemon control program `phd`. = Overview = Phabricator uses daemons (background processing scripts) to handle a number of tasks: - tracking repositories, discovering new commits, and importing and parsing commits; - sending email; and - collecting garbage, like old logs and caches. Daemons are started and stopped with **phd** (the **Ph**abricator **D**aemon launcher). Daemons can be monitored via a web console. You do not need to run daemons for most parts of Phabricator to work, but some features (principally, repository tracking with Diffusion) require them and several features will benefit in performance or stability if you configure daemons. = phd = **phd** is a command-line script (located at `phabricator/bin/phd`). To get a list of commands, run `phd help`: phabricator/ $ ./bin/phd help NAME phd - phabricator daemon launcher ... Generally, you will use: - **phd start** to launch all daemons; - **phd restart** to restart all daemons; - **phd status** to get a list of running daemons; and - **phd stop** to stop all daemons. If you want finer-grained control, you can use: - **phd launch** to launch individual daemons; and - **phd debug** to debug problems with daemons. NOTE: When you upgrade Phabricator or change configuration, you should restart the daemons by running `phd restart`. = Daemon Console = You can view status and debugging information for daemons in the Daemon Console via the web interface. Go to `/daemon/` in your install or click **Daemon Console** from "More Stuff". The Daemon Console shows a list of all the daemons that have ever launched, and allows you to view log information for them. If you have issues with daemons, you may be able to find error information that will help you resolve the problem in the console. NOTE: The easiest way to figure out what's wrong with a daemon is usually to use **phd debug** to launch it instead of **phd start**. This will run it without daemonizing it, so you can see output in your console. = Available Daemons = You can get a list of launchable daemons with **phd list**: - - **libphutil test daemons** are not generally useful unless you are + - **test daemons** are not generally useful unless you are developing daemon infrastructure or debugging a daemon problem; - **PhabricatorTaskmasterDaemon** performs work from a task queue; - **PhabricatorRepositoryPullLocalDaemon** daemons track repositories, for more information see @{article:Diffusion User Guide}; and - **PhabricatorTriggerDaemon** schedules event triggers and cleans up old logs and caches. = Debugging and Tuning = In most cases, **phd start** handles launching all the daemons you need. However, you may want to use more granular daemon controls to debug daemons, launch custom daemons, or launch special daemons like the IRC bot. To debug a daemon, use `phd debug`: phabricator/bin/ $ ./phd debug You can pass arguments like this (normal arguments are passed to the daemon control mechanism, not to the daemon itself): phabricator/bin/ $ ./phd debug -- --flavor apple In debug mode, daemons do not daemonize, and they print additional debugging output to the console. This should make it easier to debug problems. You can terminate the daemon with `^C`. To launch a nonstandard daemon, use `phd launch`: phabricator/bin/ $ ./phd launch This daemon will daemonize and run normally. == General Tips == - You can set the maximum number of taskmasters that will run at once by adjusting `phd.taskmasters`. If you have a task backlog, try increasing it. - When you `phd launch` or `phd debug` a daemon, you can type any unique substring of its name, so `phd launch pull` will work correctly. - `phd stop` and `phd restart` stop **all** of the daemons on the machine, not just those started with `phd start`. If you're writing a restart script, have it launch any custom daemons explicitly after `phd restart`. - You can write your own daemons and manage them with `phd` by extending @{class:PhabricatorDaemon}. See @{article@phabcontrib:Adding New Classes}. - See @{article:Diffusion User Guide} for details about tuning the repository daemon. Multiple Hosts ============== For information about running daemons on multiple hosts, see @{article:Cluster: Daemons}. Next Steps ========== Continue by: - learning about the repository daemon with @{article:Diffusion User Guide}; or - writing your own daemons with @{article@phabcontrib:Adding New Classes}. diff --git a/src/docs/user/configuration/troubleshooting_https.diviner b/src/docs/user/configuration/troubleshooting_https.diviner index 6b93a4f690..bdc3439d7d 100644 --- a/src/docs/user/configuration/troubleshooting_https.diviner +++ b/src/docs/user/configuration/troubleshooting_https.diviner @@ -1,80 +1,80 @@ @title Troubleshooting HTTPS @group config Detailed instructions for troubleshooting HTTPS connection problems. = Overview = If you're having trouble connecting to an HTTPS install of Phabricator, and particularly if you're receiving a "There was an error negotiating the SSL connection." error, this document may be able to help you diagnose and resolve the problem. Connection negotiation can fail for several reasons. The major ones are: - You have not added the Certificate Authority as a trusted authority (this is the most common problem, and usually the issue for self-signed certificates). - The SSL certificate is signed for the wrong domain. For example, a certificate signed for `www.example.com` will not work for `phabricator.example.com`. - The server rejects TLSv1 SNI connections for the domain (this is complicated, see below). = Certificate Authority Problems = SSL certificates need to be signed by a trusted authority (called a Certificate Authority or "CA") to be accepted. If the CA for a certificate is untrusted, the connection will fail (this defends the connection from an eavesdropping attack called "man in the middle"). Normally, you purchase a certificate from a known authority and clients have a list of trusted authorities. You can self-sign a certificate by creating your own CA, but clients will not trust it by default. They need to add the CA as a trusted authority. -For instructions on adding CAs, see `libphutil/resources/ssl/README`. +For instructions on adding CAs, see `arcanist/resources/ssl/README`. If you'd prefer that `arc` not verify the identity of the server whatsoever, you can use the `https.blindly-trust-domains` setting. This will make it dramatically easier for adversaries to perform certain types of attacks, and is **strongly discouraged**: $ arc set-config https.blindly-trust-domains '["example.com"]' = Domain Problems = Verify the domain the certificate was issued for. You can generally do this with: $ openssl x509 -text -in If the certificate was accidentally generated for, e.g. `www.example.com` but you installed Phabricator on `phabricator.example.com`, you need to generate a new certificate for the right domain. = SNI Problems = Server Name Identification ("SNI") is a feature of TLSv1 which works a bit like Apache VirtualHosts, and allows a server to present different certificates to clients who are connecting to it using different names. Servers that are not configured properly may reject TSLv1 SNI requests because they do not recognize the name the client is connecting with. This topic is complicated, but you can test for it by running: $ openssl s_client -connect example.com:443 -servername example.com Replace **both** instances of "example.com" with your domain. If you receive an error in `SSL23_GET_SERVER_HELLO` with `reason(1112)`, like this: CONNECTED(00000003) 87871:error:14077458:SSL routines:SSL23_GET_SERVER_HELLO:reason(1112): /SourceCache/OpenSSL098/OpenSSL098-44/src/ssl/s23_clnt.c:602: ...it indicates server is misconfigured. The most common cause of this problem is an Apache server that does not explicitly name the Phabricator domain as a valid VirtualHost. This error occurs only for some versions of the OpenSSL client library (from v0.9.8r or earlier until 1.0.0), so only some users may experience it. diff --git a/src/docs/user/field/darkconsole.diviner b/src/docs/user/field/darkconsole.diviner index cbdfb9bda5..065be2d8f1 100644 --- a/src/docs/user/field/darkconsole.diviner +++ b/src/docs/user/field/darkconsole.diviner @@ -1,181 +1,181 @@ @title Using DarkConsole @group fieldmanual Enabling and using the built-in debugging and performance console. Overview ======== DarkConsole is a debugging console built into Phabricator which exposes configuration, performance and error information. It can help you detect, understand and resolve bugs and performance problems in Phabricator applications. Security Warning ================ WARNING: Because DarkConsole exposes some configuration and debugging information, it is disabled by default and you should be cautious about enabling it in production. Particularly, DarkConsole may expose some information about your session details or other private material. It has some crude safeguards against this, but does not completely sanitize output. This is mostly a risk if you take screenshots or copy/paste output and share it with others. Enabling DarkConsole ==================== You enable DarkConsole in your configuration, by setting `darkconsole.enabled` to `true`, and then turning it on in {nav Settings > Developer Settings}. Once DarkConsole is enabled, you can show or hide it by pressing ##`## on your keyboard. Since the setting is not available to logged-out users, you can also set `darkconsole.always-on` if you need to access DarkConsole on logged-out pages. DarkConsole has a number of tabs, each of which is powered by a "plugin". You can use them to access different debugging and performance features. Plugin: Error Log ================= The "Error Log" plugin shows errors that occurred while generating the page, similar to the httpd `error.log`. You can send information to the error log -explicitly with the @{function@libphutil:phlog} function. +explicitly with the @{function@arcanist:phlog} function. If errors occurred, a red dot will appear on the plugin tab. Plugin: Request =============== The "Request" plugin shows information about the HTTP request the server received, and the server itself. Plugin: Services ================ The "Services" plugin lists calls a page made to external services, like MySQL and subprocesses. The Services tab can help you understand and debug issues related to page behavior: for example, you can use it to see exactly what queries or commands a page is running. In some cases, you can re-run those queries or commands yourself to examine their output and look for problems. This tab can also be particularly useful in understanding page performance, because many performance problems are caused by inefficient queries (queries with bad query plans or which take too long) or repeated queries (queries which could be better structured or benefit from caching). When analyzing performance problems, the major things to look for are: **Summary**: In the summary table at the top of the tab, are any categories of events dominating the performance cost? For normal pages, the costs should be roughly along these lines: | Event Type | Approximate Cost | |---|---| | Connect | 1%-10% | | Query | 10%-40% | | Cache | 1% | | Event | 1% | | Conduit | 0%-80% | | Exec | 0%-80% | | All Services | 10%-75% | | Entire Page | 100ms - 1000ms | These ranges are rough, but should usually be what you expect from a page summary. If any of these numbers are way off (for example, "Event" is taking 50% of runtime), that points toward a possible problem in that section of the code, and can guide you to examining the related service calls more carefully. **Duration**: In the Duration column, look for service calls that take a long time. Sometimes these calls are just what the page is doing, but sometimes they may indicate a problem. Some questions that may help understanding this column are: are there a small number of calls which account for a majority of the total page generation time? Do these calls seem fundamental to the behavior of the page, or is it not clear why they need to be made? Do some of them seem like they could be cached? If there are queries which look slow, using the "Analyze Query Plans" button may help reveal poor query plans. Generally, this column can help pinpoint these kinds of problems: - Queries or other service calls which are huge and inefficient. - Work the page is doing which it could cache instead. - Problems with network services. - Missing keys or poor query plans. **Repeated Calls**: In the "Details" column, look for service calls that are being made over and over again. Sometimes this is normal, but usually it indicates a call that can be batched or cached. Some things to look for are: are similar calls being made over and over again? Do calls mostly make sense given what the page is doing? Could any calls be cached? Could multiple small calls be collected into one larger call? Are any of the service calls clearly goofy nonsense that shouldn't be happening? Generally, this column can help pinpoint these kinds of problems: - Unbatched queries which should be batched (see @{article:Performance: N+1 Query Problem}). - Opportunities to improve performance with caching. - General goofiness in how service calls are working. If the services tab looks fine, and particularly if a page is slow but the "All Services" cost is small, that may indicate a problem in PHP. The best tool to understand problems in PHP is XHProf. Plugin: Startup =============== The "Startup" plugin shows information about startup phases. This information can provide insight about performance problems which occur before the profiler can start. Normally, the profiler is the best tool for understanding runtime performance, but some work is performed before the profiler starts (for example, loading libraries and configuration). If there is a substantial difference between the wall time reported by the profiler and the "Entire Page" cost reported by the Services tab, the Startup tab can help account for that time. It is normal for starting the profiler to increase the cost of the page somewhat: the profiler itself adds overhead while it is running, and the page must do some work after the profiler is stopped to save the profile and complete other shutdown operations. Plugin: XHProf ============== The "XHProf" plugin gives you access to the XHProf profiler. To use it, you need to install the corresponding PHP plugin. Once it is installed, you can use XHProf to profile the runtime performance of a page. This will show you a detailed breakdown of where PHP spent time. This can help find slow or inefficient application code, and is the most powerful general-purpose performance tool available. For instructions on installing and using XHProf, see @{article:Using XHProf}. Next Steps ========== Continue by: - installing XHProf with @{article:Using XHProf}; or - understanding and reporting performance issues with @{article:Troubleshooting Performance Problems}. diff --git a/src/docs/user/userguide/arcanist.diviner b/src/docs/user/userguide/arcanist.diviner index e8d6bcd5ed..0de18a9358 100644 --- a/src/docs/user/userguide/arcanist.diviner +++ b/src/docs/user/userguide/arcanist.diviner @@ -1,180 +1,172 @@ @title Arcanist User Guide @group userguide Guide to Arcanist, a command-line interface to Phabricator. Arcanist provides command-line access to many Phabricator tools (like Differential, Files, and Paste), integrates with static analysis ("lint") and unit tests, and manages common workflows like getting changes into Differential for review. A detailed command reference is available by running `arc help`. This document provides an overview of common workflows and installation. Arcanist has technical, contributor-focused documentation here: = Quick Start = A quick start guide is available at @{article:Arcanist Quick Start}. It provides a much more compact summary of how to get `arc` set up and running for a new project. You may want to start there, and return here if you need more information. = Overview = Arcanist is a wrapper script that sits on top of other tools (e.g., Differential, linters, unit test frameworks, git, Mercurial, and SVN) and provides a simple command-line API to manage code review and some related revision control operations. For a detailed list of all available commands, run: $ arc help For detailed information about a specific command, run: $ arc help Arcanist allows you to do things like: - get detailed help about available commands with `arc help` - send your code to Differential for review with `arc diff` (for detailed instructions, see @{article:Arcanist User Guide: arc diff}) - show pending revision information with `arc list` - find likely reviewers for a change with `arc cover` - apply changes in a revision to the working copy with `arc patch` - download a patch from Differential with `arc export` - update Git commit messages after review with `arc amend` - commit SVN changes with `arc commit` - push Git and Mercurial changes with `arc land` - view enhanced information about Git branches with `arc branch` Once you've configured lint and unit test integration, you can also: - check your code for syntax and style errors with `arc lint` (see @{article:Arcanist User Guide: Lint}) - run unit tests that cover your changes with `arc unit` Arcanist integrates with other tools: - upload and download files with `arc upload` and `arc download` - create and view pastes with `arc paste` Arcanist has some advanced features as well, you can: - execute Conduit method calls with `arc call-conduit` - create or update libphutil libraries with `arc liberate` - activate tab completion with `arc shell-complete` - ...or extend Arcanist and add new commands. Except where otherwise noted, these workflows are generally agnostic to the underlying version control system and will work properly in git, Mercurial, or SVN repositories. = Installing Arcanist = Arcanist is meant to be installed on your local machine or development server -- whatever machine you're editing code on. It runs on: - Linux; - Other operating systems which are pretty similar to Linux, or which Linux is pretty similar to; - FreeBSD, a fine operating system held in great esteem by many; - Mac OS X (see @{article:Arcanist User Guide: Mac OS X}); and - Windows (see @{article:Arcanist User Guide: Windows}). Arcanist is written in PHP, so you need to install the PHP CLI first if you don't already have it. Arcanist should run on PHP 5.2 and newer. If you don't have PHP installed, you can download it from . To install Arcanist, pick an install directory and clone the code from GitHub: - some_install_path/ $ git clone https://github.com/phacility/libphutil.git some_install_path/ $ git clone https://github.com/phacility/arcanist.git -This should leave you with a directory structure like this - - some_install_path/ # Wherever you chose to install it. - arcanist/ # Arcanist-specific code and libraries. - libphutil/ # A shared library Arcanist depends upon. - Now add `some_install_path/arcanist/bin/` to your PATH environment variable. When you type "arc", you should see something like this: Usage Exception: No command provided. Try 'arc help'. If you get that far, you've done things correctly. If you get an error or have trouble getting this far, see these detailed guides: - On Windows: @{article:Arcanist User Guide: Windows} - On Mac OS X: @{article:Arcanist User Guide: Mac OS X} -You can later upgrade Arcanist and libphutil to the latest versions with -`arc upgrade`: +You can later upgrade Arcanist to the latest version with `arc upgrade`: $ arc upgrade == Installing Arcanist for a Team == Arcanist changes quickly, so it can be something of a headache to get it installed and keep people up to date. Here are some approaches you might be able to use: - Facebook does most development on development servers, which have a standard - environment and NFS mounts. Arcanist and libphutil themselves live on an + environment and NFS mounts. Arcanist lives on an NFS mount, and the default `.bashrc` adds them to the PATH. Updating the mount source updates everyone's versions, and new employees have a working `arc` when they first log in. - Another common approach is to write an install script as an action into existing build scripts, so users can run `make install-arc` or `ant install-arc` or similar. == Installing Tab Completion == If you use `bash`, you can set up tab completion by running this command: $ arc shell-complete This will install shell completion into your current shell. After installing, you may need to start a new shell (or open a new terminal window) to pick up the updated configuration. == Configuration == Some Arcanist commands can be configured. This configuration is read from three sources, in order: # A project can specify configuration in an `.arcconfig` file. This file is JSON, and can be updated using `arc set-config --local` or by editing it manually. # User configuration is read from `~/.arcconfig`. This file is JSON, and can be updated using `arc set-config`. # Host configuration is read from `/etc/arcconfig` (on Windows, the path is `C:\ProgramData\Phabricator\Arcanist\config`). Arcanist uses the first definition it encounters as the runtime setting. Existing settings can be printed with `arc get-config`. Use `arc help set-config` and `arc help get-config` for more information about reading and writing configuration. == Next Steps == Continue by: - setting up a new project for use with `arc`, with @{article:Arcanist User Guide: Configuring a New Project}; or - learning how to use `arc` to send changes for review with @{article:Arcanist User Guide: arc diff}. Advanced topics are also available. These are detailed guides to configuring technical features of `arc` that refine its behavior. You do not need to read them to get it working. - @{article:Arcanist User Guide: Commit Ranges} - @{article:Arcanist User Guide: Lint} - @{article:Arcanist User Guide: Customizing Existing Linters} - @{article:Arcanist User Guide: Customizing Lint, Unit Tests and Workflows} - @{article:Arcanist User Guide: Code Coverage} diff --git a/src/docs/user/userguide/arcanist_coverage.diviner b/src/docs/user/userguide/arcanist_coverage.diviner index a734e5dd80..cb25c0cc74 100644 --- a/src/docs/user/userguide/arcanist_coverage.diviner +++ b/src/docs/user/userguide/arcanist_coverage.diviner @@ -1,69 +1,69 @@ @title Arcanist User Guide: Code Coverage @group userguide Explains code coverage features in Arcanist and Phabricator. This is a configuration guide that helps you set up advanced features. If you're just getting started, you don't need to look at this yet. Instead, start with the @{article:Arcanist User Guide}. Before you can configure coverage features, you must set up unit test integration. For instructions, see @{article:Arcanist User Guide: Configuring a New Project} and @{article:Arcanist User Guide: Customizing Lint, Unit Tests and Workflows}. = Using Coverage Features = If your project has unit tests with coverage integration (see below for instructions on setting it up), you can use "arc" to show coverage reports. For example: arc unit --detailed-coverage src/some/file.php Depending on how your test engine is configured, this will run tests relevant to `src/some/file.php` and give you a detailed coverage report. If the test engine enables coverage by default, it will be uploaded to Differential and displayed in the right gutter when viewing diffs. -= Enabling Coverage for libphutil, Arcanist and Phabricator = += Enabling Coverage for Arcanist and Phabricator = -If you're contributing, libphutil, Arcanist and Phabricator support coverage if +If you're contributing, Arcanist and Phabricator support coverage if you install Xdebug: http://xdebug.org/ It should be sufficient to correctly install Xdebug; coverage information will be automatically enabled. = Building Coverage Support = To add coverage support to a unit test engine, just call `setCoverage()` when building @{class@arcanist:ArcanistUnitTestResult} objects. Provide a map of file names (relative to the working copy root) to coverage report strings. Coverage report strings look like this: NNNNNCCCNNNNNNNNCCCCCCNNNUUUNNNNN Each line in the file is represented by a character. Valid characters are: - **N** Not executable. This is a comment or whitespace which should be ignored when computing test coverage. - **C** Covered. This line has test coverage. - **U** Uncovered. This line is executable but has no test coverage. - **X** Unreachable. If your coverage analysis can detect unreachable code, you can report it here. This format is intended to be as simple as possible. A valid coverage result might look like this: array( 'src/example.php' => 'NNCNNNCNUNNNUNUNUNUNUNC', 'src/other.php' => 'NNUNNNUNCNNNUNUNCNCNCNU', ); You may also want to filter coverage information to the paths passed to the unit test engine. See @{class@arcanist:PhutilTestCase} and @{class@arcanist:PhutilUnitTestEngine} for an example of coverage integration in PHP using Xdebug. diff --git a/src/docs/user/userguide/arcanist_quick_start.diviner b/src/docs/user/userguide/arcanist_quick_start.diviner index 743afe4a11..25847ab8a6 100644 --- a/src/docs/user/userguide/arcanist_quick_start.diviner +++ b/src/docs/user/userguide/arcanist_quick_start.diviner @@ -1,82 +1,79 @@ @title Arcanist Quick Start @group userguide Quick guide to getting Arcanist working for a new project. This is a summary of steps to install Arcanist, configure a project for use with it, and run `arc` to send changes for review. For detailed instructions on installing Arcanist, see @{article:Arcanist User Guide}. OS specific guides are also available. - For Mac OS X, see @{article:Arcanist User Guide: Mac OS X}. - For Windows, see @{article:Arcanist User Guide: Windows}. = Installing Arcanist = First, install dependencies: - Install PHP. - Install Git. Then install Arcanist itself: - $ mkdir somewhere/ - $ cd somewhere/ - somewhere/ $ git clone https://github.com/phacility/libphutil.git somewhere/ $ git clone https://github.com/phacility/arcanist.git Add `arc` to your path: $ export PATH="$PATH:/somewhere/arcanist/bin/" This won't work for Windows, see @{article:Arcanist User Guide: Windows} for instructions. = Configure Your Project = For detailed instructions on project configuration, see @{article:Arcanist User Guide: Configuring a New Project}. Create a `.arcconfig` file in your project's working copy: $ cd yourproject/ yourproject/ $ $EDITOR .arcconfig yourproject/ $ cat .arcconfig { "phabricator.uri" : "https://phabricator.example.com/" } Set `phabricator.uri` to the URI for your Phabricator install (where `arc` should send changes to). NOTE: You should **commit this file** to the repository. = Install Arcanist Credentials = Credentials allow you to authenticate. You must have an account on Phabricator before you can perform this step. $ cd yourproject/ yourproject/ $ arc install-certificate ... Follow the instructions. This will link your user account on your local machine to your Phabricator account. = Send Changes For Review = For detailed instructions on using `arc diff`, see @{article:Arcanist User Guide: arc diff}. $ $EDITOR file.c $ arc diff = Next Steps = Continue by: - learning more about project configuration with @{article:Arcanist User Guide: Configuring a New Project}; or - learning more about `arc diff` with @{article:Arcanist User Guide: arc diff}; or - returning to @{article:Arcanist User Guide}. diff --git a/src/docs/user/userguide/conduit.diviner b/src/docs/user/userguide/conduit.diviner index 5784d8cd01..35daee505f 100644 --- a/src/docs/user/userguide/conduit.diviner +++ b/src/docs/user/userguide/conduit.diviner @@ -1,68 +1,67 @@ @title Conduit API Overview @group conduit Overview of the Conduit API. Overview ======== Conduit is the HTTP API for Phabricator. It is roughly JSON-RPC: you usually pass a JSON blob, and usually get a JSON blob back, although both call and result formats are flexible in some cases. API Clients =========== The primary ways to make Conduit calls are: **Web Console**: The {nav Conduit} application provides a web UI for exploring the API and making calls. This is the best starting point for learning about the API. See the next section for details. -`ConduitClient`: This is the official client available in `libphutil`, and -the one used by `arc`. +`ConduitClient`: This is the official client available in `arcanist`. `arc call-conduit`: You can use this `arc` command to execute low-level Conduit calls by piping JSON in to stdin. This can provide a simple way to explore the API, or a quick way to get API access from a script written in another language without needing a real client. `curl`: You can format a call with basic HTTP parameters and cURL. The console includes examples which show how to format calls. **Other Clients**: There are also clients available in other languages. You can check the [[ https://secure.phabricator.com/w/community_resources/ | Community Resources ]] page for links. API Console =========== The easiest way to begin exploring Conduit is by visiting {nav Conduit} in the web UI. The application provides an API console which you can use to explore available methods, make calls, read documentation, and see examples. The API console has details about how to construct calls and generate API tokens for authentication. Querying and Reading Objects ============================ For information on searching for objects and reading their properties and information, see @{article:Conduit API: Using Search Endpoints}. Creating and Editing Objects ============================ For information on creating, editing and updating objects, see @{article:Conduit API: Using Edit Endpoints}. Next Steps ========== Continue by: - reading recommendations on responding to API changes in @{article:Managing Conduit Changes}. diff --git a/src/docs/user/userguide/diffusion_managing.diviner b/src/docs/user/userguide/diffusion_managing.diviner index 138bc918bc..e3743526e9 100644 --- a/src/docs/user/userguide/diffusion_managing.diviner +++ b/src/docs/user/userguide/diffusion_managing.diviner @@ -1,451 +1,450 @@ @title Diffusion User Guide: Managing Repositories @group userguide Guide to configuring and managing repositories in Diffusion. Overview ======== After you create a new repository in Diffusion or select **Manage Repository** from the main screen if an existing repository, you'll be taken to the repository management interface for that repository. On this interface, you'll find many options which allow you to configure the behavior of a repository. This document walks through the options. Basics ====== The **Basics** section of the management interface allows you to configure the repository name, description, and identifiers. You can also activate or deactivate the repository here, and configure a few other miscellaneous settings. Basics: Name ============ The repository name is a human-readable primary name for the repository. It does not need to be unique Because the name is not unique and does not have any meaningful restrictions, it's fairly ambiguous and isn't very useful as an identifier. The other basic information (primarily callsigns and short names) gives you control over repository identifiers. Basics: Callsigns ================= Each repository can optionally be identified by a "callsign", which is a short uppercase string like "P" (for Phabricator) or "ARC" (for Arcanist). The primary goal of callsigns is to namespace commits to SVN repositories: if you use multiple SVN repositories, each repository has a revision 1, revision 2, etc., so referring to them by number alone is ambiguous. However, even for Git and Mercurial they impart additional information to human readers and allow parsers to detect that something is a commit name with high probability (and allow distinguishing between multiple copies of a repository). Configuring a callsign can make interacting with a commonly-used repository easier, but you may not want to bother assigning one to every repository if you have some similar, templated, or rarely-used repositories. If you choose to assign a callsign to a repository, it must be unique within an install but do not need to be globally unique, so you are free to use the single-letter callsigns for brevity. For example, Facebook uses "E" for the Engineering repository, "O" for the Ops repository, "Y" for a Yum package -repository, and so on, while Phabricator uses "P", "ARC", "PHU" for libphutil, -and "J" for Javelin. Keeping callsigns brief will make them easier to use, and -the use of one-character callsigns is encouraged if they are reasonably -evocative. +repository, and so on, while Phabricator uses "P" and Arcanist uses "ARC". +Keeping callsigns brief will make them easier to use, and the use of +one-character callsigns is encouraged if they are reasonably evocative. If you configure a callsign like `XYZ`, Phabricator will activate callsign URIs and activate the callsign identifier (like `rXYZ`) for the repository. These more human-readable identifiers can make things a little easier to interact with. Basics: Short Name ================== Each repository can optionally have a unique short name. Short names must be unique and have some minor restrictions to make sure they are unambiguous and appropriate for use as directory names and in URIs. Basics: Description =================== You may optionally provide a brief (or, at your discretion, excruciatingly long) human-readable description of the repository. This description will be shown on the main repository page. You can also create a `README` file at the repository root (or in any subdirectory) to provide information about the repository. These formats are supported: | File Name | Rendered As... |-------------------|--------------- | `README` | Plain Text | `README.txt` | Plain Text | `README.remarkup` | Remarkup | `README.md` | Remarkup | `README.rainbow` | Rainbow Basics: Encoding ================ Before content from the repository can be shown in the web UI or embedded in other contexts like email, it must be converted to UTF-8. Most source code is written in UTF-8 or a subset of UTF-8 (like plain ASCII) already, so everything will work fine. The majority of repositories do not need to adjust this setting. If your repository is primarily written in some other encoding, specify it here so Phabricator can convert from it properly when reading content to embed in a webpage or email. Basics: Dangerous Changes ========================= By default, repositories are protected against dangerous changes. Dangerous changes are operations which rewrite or destroy repository history (for example, by deleting or rewriting branches). Normally, these take the form of `git push --force` or similar. It is normally a good idea to leave this protection enabled because most scalable workflows rarely rewrite repository history and it's easy to make mistakes which are expensive to correct if this protection is disabled. If you do occasionally need to rewrite published history, you can treat this option like a safety: disable it, perform required rewrites, then enable it again. If you fully disable this at the repository level, you can still use Herald to selectively protect certain branches or grant this power to a limited set of users. This option is only available in Git and Mercurial, because it is impossible to make dangerous changes in Subversion. This option has no effect if a repository is not hosted because Phabricator can not prevent dangerous changes in a remote repository it is merely observing. Basics: Disable Publishing ========================== You can disable publishing for a repository. For more details on what this means, see @{article:Diffusion User Guide: Permanent Refs}. This is primarily useful if you need to perform major maintenance on a repository (like rewriting a large part of the repository history) and you don't want the maintenance to generate a large volume of email and notifications. You can disable publishing, apply major changes, wait for the new changes to import, and then reactivate publishing. Basics: Deactivate Repository ============================= Repositories can be deactivated. Deactivating a repository has these effects: - the repository will no longer be updated; - users will no longer be able to clone/fetch/checkout the repository; - users will no longer be able to push to the repository; and - the repository will be hidden from view in default queries. When repositories are created for the first time, they are deactivated. This gives you an opportunity to customize settings, like adjusting policies or configuring a URI to observe. You must activate a repository before it will start working normally. Basics: Delete Repository ========================= Repositories can not be deleted from the web UI, so this option only gives you information about how to delete a repository. Repositories can only be deleted from the command line, with `bin/remove`: ``` $ ./bin/remove destroy ``` This command will permanently destroy the repository. For more information about destroying things, see @{article:Permanently Destroying Data}. Policies ======== The **Policies** section of the management interface allows you to review and manage repository access policies. You can configure granular access policies for each repository to control who can view, clone, administrate, and push to the repository. Policies: View ============== The view policy for a repository controls who can view the repository from the web UI and clone, fetch, or check it out from Phabricator. Users who can view a repository can also access the "Manage" interface to review information about the repository and examine the edit history, but can not make any changes. Policies: Edit ============== The edit policy for a repository controls who can change repository settings using the "Manage" interface. In essence, this is permission to administrate the repository. You must be able to view a repository to edit it. You do not need this permission to push changes to a repository. Policies: Push ============== The push policy for a repository controls who can push changes to the repository. This policy has no effect if Phabricator is not hosting the repository, because it can not control who is allowed to make changes to a remote repository it is merely observing. You must also be able to view a repository to push to it. You do not need to be able to edit a repository to push to it. Further restrictions on who can push (and what they can push) can be configured for hosted repositories with Herald, which allows you to write more sophisticated rules that evaluate when Phabricator receives a push. To get started with Herald, see @{article:Herald User Guide}. Additionally, Git and Mercurial repositories have a setting which allows you to **Prevent Dangerous Changes**. This setting is enabled by default and will prevent any users from pushing changes which rewrite or destroy history. URIs ==== The **URIs** panel allows you to add and manage URIs which Phabricator will fetch from, serve from, and push to. These options are covered in detail in @{article:Diffusion User Guide: URIs}. Limits ====== The **Limits** panel allows you to configure limits and timeouts. **Filesize Limit**: Allows you to set a maximum filesize for any file in the repository. If a commit creates a larger file (or modifies an existing file so it becomes too large) it will be rejected. This option only applies to hosted repositories. This limit is primarily intended to make it more difficult to accidentally push very large files that shouldn't be version controlled (like logs, binaries, machine learning data, or media assets). Pushing huge datafiles by mistake can make the repository unwieldy by dramatically increasing how much data must be transferred over the network to clone it, and simply reverting the changes doesn't reduce the impact of this kind of mistake. **Clone/Fetch Timeout**: Configure the internal timeout for creating copies of this repository during operations like intracluster synchronization and Drydock working copy construction. This timeout does not affect external users. **Touch Limit**: Apply a limit to the maximum number of paths that any commit may touch. If a commit affects more paths than this limit, it will be rejected. This option only applies to hosted repositories. Users may work around this limit by breaking the commit into several smaller commits which each affect fewer paths. This limit is intended to offer a guard rail against users making silly mistakes that create obviously mistaken changes, like copying an entire repository into itself and pushing the result. This kind of change can take some effort to clean up if it becomes part of repository history. Note that if you move a file, both the old and new locations count as touched paths. You should generally configure this limit to be more than twice the number of files you anticipate any user ever legitimately wanting to move in a single commit. For example, a limit of `20000` will let users move up to 10,000 files in a single commit, but will reject users mistakenly trying to push a copy of another repository or a directory with a million logfiles or whatever other kind of creative nonsense they manage to dream up. Branches ======== The **Branches** panel allows you to configure how Phabricator interacts with branches. This panel is not available for Subversion repositories, because Subversion does not have formal branches. You can configure a **Default Branch**. This controls which branch is shown by default in the UI. If no branch is provided, Phabricator will use `master` in Git and `default` in Mercurial. **Fetch Refs**: In Git, if you are observing a remote repository, you can specify that you only want to fetch a subset of refs using "Fetch Refs". Normally, all refs (`refs/*`) are fetched. This means all branches, all tags, and all other refs. If you want to fetch only a few specific branches, you can list only those branches. For example, this will fetch only the branch "master": ``` refs/heads/master ``` You can fetch all branches and tags (but ignore other refs) like this: ``` refs/heads/* refs/tags/* ``` This may be useful if the remote is on a service like GitHub, GitLab, or Gerrit and uses custom refs (like `refs/pull/` or `refs/changes/`) to store metadata that you don't want to bring into Phabricator. **Permanent Refs**: To learn more about permanent refs, see: - @{article:Diffusion User Guide: Permanent Refs} By default, Phabricator considers all branches to be permanent refs. If you only want some branches to be treated as permanent refs, specify them here. When specifying branches, you should enter one branch name per line. You can use regular expressions to match branches by wrapping an expression in `regexp(...)`. For example: | Example | Effect | |---------|--------| | `master` | Only the `master` branch is a permanent ref. | `regexp(/^release-/)` | Branches are permanent if they start with `release-`. | `regexp(/^(?!temp-)/)` | Branches named `temp-` are not permanent. Staging Area ============ The **Staging Area** panel configures staging areas, used to make proposed changes available to build and continuous integration systems. For more details, see @{article:Harbormaster User Guide}. Automation ========== The **Automation** panel configures support for allowing Phabricator to make writes directly to the repository, so that it can perform operations like automatically landing revisions from the web UI. For details on repository automation, see @{article:Drydock User Guide: Repository Automation}. Symbols ====== The **Symbols** panel allows you to customize how symbols (like class and function names) are linked when viewing code in the repository, and when viewing revisions which propose code changes to the repository. To take advantage of this feature, you need to do additional work to build symbol indexes. For details on configuring and populating symbol indexes, see @{article:User Guide: Symbol Indexes}. Repository Identifiers and Names ================================ Repositories have several short identifiers which you can use to refer to the repository. For example, if you use command-line administrative tools to interact with a repository, you'll provide one of these identifiers: ``` $ ./bin/repository update ``` The identifiers available for a repository depend on which options are configured. Each repository may have several identifiers: - An **ID** identifier, like `R123`. This is available for all repositories. - A **callsign** identifier, like `rXY`. This is available for repositories with a callsign. - A **short name** identifier, like `xylophone`. This is available for repositories with a short name. All three identifiers can be used to refer to the repository in cases where the intent is unambiguous, but only the first two forms work in ambiguous contexts. For example, if you type `R123` or `rXY` into a comment, Phabricator will recognize them as references to the repository. If you type `xylophone`, it assumes you mean the word "xylophone". Only the `R123` identifier is immutable: the others can be changed later by adjusting the callsign or short name for the repository. Commit Identifiers ================== Diffusion uses repository identifiers and information about the commit itself to generate globally unique identifiers for each commit, like `rE12345`. Each commit may have several identifiers: - A repository **ID** identifier, like `R123:abcdef123...`. - A repository **callsign** identifier, like `rXYZabcdef123...`. This only works if a repository has a callsign. - Any unique prefix of the commit hash. Git and Mercurial use commit hashes to identify commits, and Phabricator will recognize a commit if the hash prefix is unique and sufficiently long. Commit hashes qualified with a repository identifier must be at least 5 characters long; unqualified commit hashes must be at least 7 characters long. In Subversion, commit identifiers are sequential integers and prefixes can not be used to identify them. When rendering the name of a Git or Mercurial commit hash, Phabricator tends to shorten it to 12 characters. This "short length" is relatively long compared to Git itself (which often uses 7 characters). See this post on the LKML for a historical explanation of Git's occasional internal use of 7-character hashes: https://lkml.org/lkml/2010/10/28/287 Because 7-character hashes are likely to collide for even moderately large repositories, Diffusion generally uses either a 12-character prefix (which makes collisions very unlikely) or the full 40-character hash (which makes collisions astronomically unlikely). Next Steps ========== Continue by: - returning to the @{article:Diffusion User Guide}. diff --git a/src/docs/user/userguide/diffusion_symbols.diviner b/src/docs/user/userguide/diffusion_symbols.diviner index f5da8aefe0..7d14ad92b2 100644 --- a/src/docs/user/userguide/diffusion_symbols.diviner +++ b/src/docs/user/userguide/diffusion_symbols.diviner @@ -1,97 +1,97 @@ @title Diffusion User Guide: Symbol Indexes @group userguide Guide to configuring and using the symbol index. = Overview = Phabricator can maintain a symbol index, which keeps track of where classes and functions are defined in the codebase. Once you set up indexing, you can use the index to do things like: - jump to symbol definitions from Differential code reviews and Diffusion code browsing by ctrl-clicking (cmd-click on Mac) symbols - search for symbols from the quick-search - let the IRC bot answer questions like "Where is SomeClass?" NOTE: Because this feature depends on the syntax highlighter, it will work better for some languages than others. It currently works fairly well for PHP, but your mileage may vary for other languages. = Populating the Index = To populate the index, you need to write a script which identifies symbols in your codebase and set up a cronjob which pipes its output to: ./scripts/symbols/import_repository_symbols.php Phabricator includes a script which can identify symbols in PHP projects: ./scripts/symbols/generate_php_symbols.php Phabricator also includes a script which can identify symbols in any programming language that has classes and/or functions, and is supported by Exuberant Ctags (http://ctags.sourceforge.net): ./scripts/symbols/generate_ctags_symbols.php If you want to identify symbols from another language, you need to write a script which can export them (for example, maybe by parsing a `ctags` file). The output format of the script should be one symbol per line: For example: ExampleClass exampleMethod function php 13 /src/classes/ExampleClass.php Context is, broadly speaking, the scope or namespace where the symbol is defined. For object-oriented languages, this is probably a class name. The symbols with that context are class constants, methods, properties, nested classes, etc. When printing symbols without a context (those that are defined globally, for instance), the `` field should be empty (that is, the line should start with a space). Your script should enumerate all the symbols in your project, and provide paths from the project root (where ".arcconfig" is) beginning with a "/". You can look at `generate_php_symbols.php` for an example of how you might write such a script, and run this command to see its output: $ cd phabricator/ $ find . -type f -name '*.php' | ./scripts/symbols/generate_php_symbols.php To actually build the symbol index, pipe this data to the `import_repository_symbols.php` script, providing the repository callsign: $ ./scripts/symbols/import_repository_symbols.php REPO < symbols_data Then just set up a cronjob to run that however often you like. You can test that the import worked by querying for symbols using the Conduit method `diffusion.findsymbols`. Some features (like that method, and the IRC bot integration) will start working immediately. Others will require more configuration. = Advanced Configuration = You can configure some more options by going to {nav Diffusion > (Select repository) > Edit Repository > Edit Symbols}, and filling out these fields: - **Indexed Languages**: Fill in all the languages you've built indexes for. You can leave this blank for "All languages". - **Uses Symbols From**: If this project depends on other repositories, add the other repositories which symbols should be looked for here. For example, - Phabricator lists "Arcanist" and "libphutil" because it uses classes and - functions from these repositories. + Phabricator lists "Arcanist" because it uses classes and functions defined + in `arcanist/`. == External Symbols == By @{article@phabcontrib:Adding New Classes}, you can teach Phabricator about symbols from the outside world. Extend @{class:DiffusionExternalSymbolsSource}; Once loaded, your new implementation will be used any time a symbol is queried. See @{class:DiffusionPhpExternalSymbolsSource} and @{class:DiffusionPythonExternalSymbolsSource} for example implementations. diff --git a/src/docs/user/userguide/drydock_hosts.diviner b/src/docs/user/userguide/drydock_hosts.diviner index 8bfed7dc60..1b8f22cce1 100644 --- a/src/docs/user/userguide/drydock_hosts.diviner +++ b/src/docs/user/userguide/drydock_hosts.diviner @@ -1,126 +1,126 @@ @title Drydock Blueprints: Hosts @group userguide Guide to configuring Drydock host blueprints. Overview ======== IMPORTANT: Drydock is not a mature application and may be difficult to configure and use for now. To give Drydock access to machines so it can perform work, you'll configure **host blueprints**. These blueprints tell Drydock where to find machines (or how to build machines) and how to connect to them. Once Drydock has access to hosts it can use them to build more interesting and complex types of resources, like repository working copies. Drydock currently supports these kinds of host blueprints: - **Almanac Hosts**: Gives Drydock access to a predefined list of hosts. Drydock may support additional blueprints in the future. Security ======== Drydock can be used to run semi-trusted and untrusted code, and you may want to isolate specific processes or classes of processes from one another. See @{article:Drydock User Guide: Security} for discussion of security concerns and guidance on how to make isolation tradeoffs. General Considerations ====================== **You must install software on hosts.** Drydock does not currently handle installing software on hosts. You'll need to make sure any hosts are configured properly with any software you need, and have tools like `git`, `hg` or `svn` that may be required to interact with working copies. -You do **not** need to install PHP, arcanist, libphutil or Phabricator on the +You do **not** need to install PHP, arcanist, or Phabricator on the hosts unless you are specifically running `arc` commands. **You must configure authentication.** Drydock also does not handle credentials for VCS operations. If you're interacting with repositories hosted on Phabricator, the simplest way to set this up is something like this: - Create a new bot user in Phabricator. - In {nav Settings > SSH Public Keys}, add a public key or generate a keypair. - Put the private key on your build hosts as `~/.ssh/id_rsa` for whatever user you're connecting with. This will let processes on the host access Phabricator as the bot user, and use the bot user's permissions to pull and push changes. If you're using hosted repositories from an external service, you can follow similar steps for that service. Note that any processes running under the given user account will have access to the private key, so you should give the bot the smallest acceptable level of permissions if you're running semi-trusted or untrusted code like unit tests. **You must create a `/var/drydock` directory.** This is hard-coded in Drydock for now, so you need to create it on the hosts. This can be a symlink to a different location if you prefer. Almanac Hosts ============= The **Almanac Hosts** blueprint type gives Drydock access to a predefined list of hosts which you configure in the Almanac application. This is the simplest type of blueprint to set up. For more information about Almanac, see @{article:Almanac User Guide}. For example, suppose you have `build001.mycompany.com` and `build002.mycompany.com`, and want to configure Drydock to be able to use these hosts. To do this: **Create Almanac Devices**: Create a device record in Almanac for each your hosts. {nav Almanac > Devices > Create Device} Enter the device names (like `build001.mycompany.com`). After creating the devices, use {nav Add Interface} to configure the ports and IP addresses that Drydock should connect to over SSH (normally, this is port `22`). **Create an Almanac Service**: In the Almanac application, create a new service to define the pool of devices you want to use. {nav Almanac > Services > Create Service} Choose the service type **Drydock: Resource Pool**. This will allow Drydock to use the devices that are bound to the service. Now, use {nav Add Binding} to bind all of the devices to the service. You can add more hosts to the pool later by binding additional devices, and Drydock will automatically start using them. Likewise, you can remove bindings to take hosts out of service. **Create a Drydock Blueprint**: Now, create a new blueprint in Drydock. {nav Drydock > Blueprints > New Blueprint} Choose the **Almanac Hosts** blueprint type. In **Almanac Services**, select the service you previously created. For **Credentials**, select an SSH private key you want Drydock to use to connect to the hosts. Drydock should now be able to build resources from these hosts. Next Steps ========== Continue by: - returning to @{article:Drydock Blueprints}. diff --git a/src/docs/user/userguide/events.diviner b/src/docs/user/userguide/events.diviner index ea66448c8a..e18578288b 100644 --- a/src/docs/user/userguide/events.diviner +++ b/src/docs/user/userguide/events.diviner @@ -1,218 +1,218 @@ @title Events User Guide: Installing Event Listeners @group userguide Using Phabricator event listeners to customize behavior. = Overview = (WARNING) The event system is an artifact of a bygone era. Use of the event system is strongly discouraged. We have been removing events since 2013 and will continue to remove events in the future. Phabricator and Arcanist allow you to install custom runtime event listeners which can react to certain things happening (like a Maniphest Task being edited or a user creating a new Differential Revision) and run custom code to perform logging, synchronize with other systems, or modify workflows. These listeners are PHP classes which you install beside Phabricator or Arcanist, and which Phabricator loads at runtime and runs in-process. They require somewhat more effort upfront than simple configuration switches, but are the most direct and powerful way to respond to events. = Installing Event Listeners (Phabricator) = To install event listeners in Phabricator, follow these steps: - - Write a listener class which extends @{class@libphutil:PhutilEventListener}. + - Write a listener class which extends @{class@arcanist:PhutilEventListener}. - Add it to a libphutil library, or create a new library (for instructions, see @{article@phabcontrib:Adding New Classes}. - Configure Phabricator to load the library by adding it to `load-libraries` in the Phabricator config. - Configure Phabricator to install the event listener by adding the class name to `events.listeners` in the Phabricator config. You can verify your listener is registered in the "Events" tab of DarkConsole. It should appear at the top under "Registered Event Listeners". You can also see any events the page emitted there. For details on DarkConsole, see @{article:Using DarkConsole}. = Installing Event Listeners (Arcanist) = To install event listeners in Arcanist, follow these steps: - - Write a listener class which extends @{class@libphutil:PhutilEventListener}. + - Write a listener class which extends @{class@arcanist:PhutilEventListener}. - Add it to a libphutil library, or create a new library (for instructions, see @{article@phabcontrib:Adding New Classes}. - Configure Phabricator to load the library by adding it to `load` in the Arcanist config (e.g., `.arcconfig`, or user/global config). - Configure Arcanist to install the event listener by adding the class name to `events.listeners` in the Arcanist config. You can verify your listener is registered by running any `arc` command with `--trace`. You should see output indicating your class was registered as an event listener. = Example Listener = Phabricator includes an example event listener, @{class:PhabricatorExampleEventListener}, which may be useful as a starting point in developing your own listeners. This listener listens for a test event that is emitted by the script `scripts/util/emit_test_event.php`. If you run this script normally, it should output something like this: $ ./scripts/util/emit_test_event.php Emitting event... Done. This is because there are no listeners for the event, so nothing reacts to it when it is emitted. You can add the example listener by either adding it to your `events.listeners` configuration or with the `--listen` command-line flag: $ ./scripts/util/emit_test_event.php --listen PhabricatorExampleEventListener Installing 'PhabricatorExampleEventListener'... Emitting event... PhabricatorExampleEventListener got test event at 1341344566 Done. This time, the listener was installed and had its callback invoked when the test event was emitted. = Available Events = You can find a list of all Phabricator events in @{class:PhabricatorEventType}. == All Events == The special constant `PhutilEventType::TYPE_ALL` will let you listen for all events. Normally, you want to listen only to specific events, but if you're writing a generic handler you can listen to all events with this constant rather than by enumerating each event. == Arcanist Events == Arcanist event constants are listed in @{class@arcanist:ArcanistEventType}. All Arcanist events have this data available: - `workflow` The active @{class@arcanist:ArcanistWorkflow}. == Arcanist: Commit: Will Commit SVN == The constant for this event is `ArcanistEventType::TYPE_COMMIT_WILLCOMMITSVN`. This event is dispatched before an `svn commit` occurs and allows you to modify the commit message. Data available on this event: - `message` The text of the message. == Arcanist: Diff: Will Build Message == The constant for this event is `ArcanistEventType::TYPE_DIFF_WILLBUILDMESSAGE`. This event is dispatched before an editable message is presented to the user, and allows you to, e.g., fill in default values for fields. Data available on this event: - `fields` A map of field values to be compiled into a message. == Arcanist: Diff: Was Created == The constant for this event is `ArcanistEventType::TYPE_DIFF_WASCREATED`. This event is dispatched after a diff is created. It is currently only useful for collecting timing information. No data is available on this event. == Arcanist: Revision: Will Create Revision == The constant for this event is `ArcanistEventType::TYPE_REVISION_WILLCREATEREVISION`. This event is dispatched before a revision is created. It allows you to modify fields to, e.g., edit revision titles. Data available on this event: - `specification` Parameters that will be used to invoke the `differential.createrevision` Conduit call. == Differential: Will Mark Generated == The constant for this event is `PhabricatorEventType::TYPE_DIFFERENTIAL_WILLMARKGENERATED`. This event is dispatched before Differential decides if a file is generated (and doesn't need to be reviewed) or not. Data available on this event: - `corpus` Body of the file. - `is_generated` Boolean indicating if this file should be treated as generated. == Diffusion: Did Discover Commit == The constant for this event is `PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT`. This event is dispatched when the daemons discover a commit for the first time. This event happens very early in the pipeline, and not all commit information will be available yet. Data available on this event: - `commit` The @{class:PhabricatorRepositoryCommit} that was discovered. - `repository` The @{class:PhabricatorRepository} the commit was discovered in. == Test: Did Run Test == The constant for this event is `PhabricatorEventType::TYPE_TEST_DIDRUNTEST`. This is a test event for testing event listeners. See above for details. == UI: Did Render Actions == The constant for this event is `PhabricatorEventType::TYPE_UI_DIDRENDERACTIONS`. This event is dispatched after a @{class:PhabricatorActionListView} is built by the UI. It allows you to add new actions that your application may provide, like "Fax this Object". Data available on this event: - `object` The object which actions are being rendered for. - `actions` The current list of available actions. NOTE: This event is unstable and subject to change. = Debugging Listeners = If you're having problems with your listener, try these steps: - If you're getting an error about Phabricator being unable to find the listener class, make sure you've added it to a libphutil library and configured Phabricator to load the library with `load-libraries`. - Make sure the listener is registered. It should appear in the "Events" tab of DarkConsole. If it's not there, you may have forgotten to add it to `events.listeners`. - Make sure it calls `listen()` on the right events in its `register()` method. If you don't listen for the events you're interested in, you won't get a callback. - Make sure the events you're listening for are actually happening. If they occur on a normal page they should appear in the "Events" tab of DarkConsole. If they occur on a POST, you could add a `phlog()` to the source code near the event and check your error log to make sure the code ran. - You can check if your callback is getting invoked by adding `phlog()` with a message and checking the error log. - You can try listening to `PhutilEventType::TYPE_ALL` instead of a specific event type to get all events, to narrow down whether problems are caused by the types of events you're listening to. - You can edit the `emit_test_event.php` script to emit other types of events instead, to test that your listener reacts to them properly. You might have to use fake data, but this gives you an easy way to test the at least the basics. - For scripts, you can run under `--trace` to see which events are emitted and how many handlers are listening to each event. = Next Steps = Continue by: - taking a look at @{class:PhabricatorExampleEventListener}; or - building a library with @{article:libphutil Libraries User Guide}. diff --git a/src/docs/user/userguide/utf8.diviner b/src/docs/user/userguide/utf8.diviner index a604599e9f..b6742f0c36 100644 --- a/src/docs/user/userguide/utf8.diviner +++ b/src/docs/user/userguide/utf8.diviner @@ -1,80 +1,38 @@ @title User Guide: UTF-8 and Character Encoding @group userguide How Phabricator handles character encodings. = Overview = Phabricator stores all internal text data as UTF-8, processes all text data as UTF-8, outputs in UTF-8, and expects all inputs to be UTF-8. Principally, this means that you should write your source code in UTF-8. In most cases this does not require you to change anything, because ASCII text is a subset of UTF-8. If you have a repository with source files that do not have UTF-8, you have two options: - Convert all files in the repository to ASCII or UTF-8 (see "Detecting and Repairing Files" below). This is recommended, especially if the encoding problems are accidental. - Configure Phabricator to convert files into UTF-8 from whatever encoding your repository is in when it needs to (see "Support for Alternate Encodings" below). This is not completely supported, and repositories with files that have multiple encodings are not supported. -= Detecting and Repairing Files = - -It is recommended that you write source files only in ASCII text, but -Phabricator fully supports UTF-8 source files. - -If you have a project which isn't valid UTF-8 because a few files have random -binary nonsense in them, there is a script in libphutil which can help you -identify and fix them: - - project/ $ libphutil/scripts/utils/utf8.php - -Generally, run this script on all source files with "-t" to find files with bad -byte ranges, and then run it without "-t" on each file to identify where there -are problems. For example: - - project/ $ find . -type f -name '*.c' -print0 | xargs -0 -n256 ./utf8 -t - ./hello_world.c - -If this script exits without output, you're in good shape and all the files that -were identified are valid UTF-8. If it found some problems, you need to repair -them. You can identify the specific problems by omitting the "-t" flag: - - project/ $ ./utf8.php hello_world.c - FAIL hello_world.c - - 3 main() - 4 { - 5 printf ("Hello World<0xE9><0xD6>!\n"); - 6 } - 7 - -This shows the offending bytes on line 5 (in the actual console display, they'll -be highlighted). Often a codebase will mostly be valid UTF-8 but have a few -scattered files that have other things in them, like curly quotes which someone -copy-pasted from Word into a comment. In these cases, you can just manually -identify and fix the problems pretty easily. - -If you have a prohibitively large number of UTF-8 issues in your source code, -Phabricator doesn't include any default tools to help you process them in a -systematic way. You could hack up `utf8.php` as a starting point, or use other -tools to batch-process your source files. - = Support for Alternate Encodings = Phabricator has some support for encodings other than UTF-8. NOTE: Alternate encodings are not completely supported, and a few features will not work correctly. Codebases with files that have multiple different encodings (for example, some files in ISO-8859-1 and some files in Shift-JIS) are not supported at all. To use an alternate encoding, edit the repository in Diffusion and specify the encoding to use. Optionally, you can use the `--encoding` flag when running `arc`, or set `encoding` in your `.arcconfig`. diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php index 4faae5c83b..480a9d8614 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php @@ -1,89 +1,90 @@ array( 'taskClass' => 'text64', 'leaseOwner' => 'text64?', 'leaseExpires' => 'epoch?', 'failureCount' => 'uint32', 'failureTime' => 'epoch?', 'priority' => 'uint32', 'objectPHID' => 'phid?', 'containerPHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_object' => array( 'columns' => array('objectPHID'), ), 'key_container' => array( 'columns' => array('containerPHID'), ), ), ) + parent::getConfiguration(); } final public function setExecutionException($execution_exception) { $this->executionException = $execution_exception; return $this; } final public function getExecutionException() { return $this->executionException; } final public function setData($data) { $this->data = $data; return $this; } final public function getData() { return $this->data; } final public function isArchived() { return ($this instanceof PhabricatorWorkerArchiveTask); } final public function getWorkerInstance() { $id = $this->getID(); $class = $this->getTaskClass(); try { - // NOTE: If the class does not exist, libphutil will throw an exception. + // NOTE: If the class does not exist, the autoloader will throw an + // exception. class_exists($class); } catch (PhutilMissingSymbolException $ex) { throw new PhabricatorWorkerPermanentFailureException( pht( "Task class '%s' does not exist!", $class)); } if (!is_subclass_of($class, 'PhabricatorWorker')) { throw new PhabricatorWorkerPermanentFailureException( pht( "Task class '%s' does not extend %s.", $class, 'PhabricatorWorker')); } return newv($class, array($this->getData())); } } diff --git a/src/infrastructure/storage/lisk/LiskDAO.php b/src/infrastructure/storage/lisk/LiskDAO.php index 03b735d315..13b2f8d319 100644 --- a/src/infrastructure/storage/lisk/LiskDAO.php +++ b/src/infrastructure/storage/lisk/LiskDAO.php @@ -1,1912 +1,1912 @@ setName('Sawyer') * ->setBreed('Pug') * ->save(); * * Note that **Lisk automatically builds getters and setters for all of your * object's protected properties** via @{method:__call}. If you want to add * custom behavior to your getters or setters, you can do so by overriding the * @{method:readField} and @{method:writeField} methods. * * Calling @{method:save} will persist the object to the database. After calling * @{method:save}, you can call @{method:getID} to retrieve the object's ID. * * To load objects by ID, use the @{method:load} method: * * $dog = id(new Dog())->load($id); * * This will load the Dog record with ID $id into $dog, or `null` if no such * record exists (@{method:load} is an instance method rather than a static * method because PHP does not support late static binding, at least until PHP * 5.3). * * To update an object, change its properties and save it: * * $dog->setBreed('Lab')->save(); * * To delete an object, call @{method:delete}: * * $dog->delete(); * * That's Lisk CRUD in a nutshell. * * = Queries = * * Often, you want to load a bunch of objects, or execute a more specialized * query. Use @{method:loadAllWhere} or @{method:loadOneWhere} to do this: * * $pugs = $dog->loadAllWhere('breed = %s', 'Pug'); * $sawyer = $dog->loadOneWhere('name = %s', 'Sawyer'); * - * These methods work like @{function@libphutil:queryfx}, but only take half of + * These methods work like @{function@arcanist:queryfx}, but only take half of * a query (the part after the WHERE keyword). Lisk will handle the connection, * columns, and object construction; you are responsible for the rest of it. * @{method:loadAllWhere} returns a list of objects, while * @{method:loadOneWhere} returns a single object (or `null`). * * There's also a @{method:loadRelatives} method which helps to prevent the 1+N * queries problem. * * = Managing Transactions = * * Lisk uses a transaction stack, so code does not generally need to be aware * of the transactional state of objects to implement correct transaction * semantics: * * $obj->openTransaction(); * $obj->save(); * $other->save(); * // ... * $other->openTransaction(); * $other->save(); * $another->save(); * if ($some_condition) { * $other->saveTransaction(); * } else { * $other->killTransaction(); * } * // ... * $obj->saveTransaction(); * * Assuming ##$obj##, ##$other## and ##$another## live on the same database, * this code will work correctly by establishing savepoints. * * Selects whose data are used later in the transaction should be included in * @{method:beginReadLocking} or @{method:beginWriteLocking} block. * * @task conn Managing Connections * @task config Configuring Lisk * @task load Loading Objects * @task info Examining Objects * @task save Writing Objects * @task hook Hooks and Callbacks * @task util Utilities * @task xaction Managing Transactions * @task isolate Isolation for Unit Testing */ abstract class LiskDAO extends Phobject implements AphrontDatabaseTableRefInterface { const CONFIG_IDS = 'id-mechanism'; const CONFIG_TIMESTAMPS = 'timestamps'; const CONFIG_AUX_PHID = 'auxiliary-phid'; const CONFIG_SERIALIZATION = 'col-serialization'; const CONFIG_BINARY = 'binary'; const CONFIG_COLUMN_SCHEMA = 'col-schema'; const CONFIG_KEY_SCHEMA = 'key-schema'; const CONFIG_NO_TABLE = 'no-table'; const CONFIG_NO_MUTATE = 'no-mutate'; const SERIALIZATION_NONE = 'id'; const SERIALIZATION_JSON = 'json'; const SERIALIZATION_PHP = 'php'; const IDS_AUTOINCREMENT = 'ids-auto'; const IDS_COUNTER = 'ids-counter'; const IDS_MANUAL = 'ids-manual'; const COUNTER_TABLE_NAME = 'lisk_counter'; private static $processIsolationLevel = 0; private static $transactionIsolationLevel = 0; private $ephemeral = false; private $forcedConnection; private static $connections = array(); protected $id; protected $phid; protected $dateCreated; protected $dateModified; /** * Build an empty object. * * @return obj Empty object. */ public function __construct() { $id_key = $this->getIDKey(); if ($id_key) { $this->$id_key = null; } } /* -( Managing Connections )----------------------------------------------- */ /** * Establish a live connection to a database service. This method should * return a new connection. Lisk handles connection caching and management; * do not perform caching deeper in the stack. * * @param string Mode, either 'r' (reading) or 'w' (reading and writing). * @return AphrontDatabaseConnection New database connection. * @task conn */ abstract protected function establishLiveConnection($mode); /** * Return a namespace for this object's connections in the connection cache. * Generally, the database name is appropriate. Two connections are considered * equivalent if they have the same connection namespace and mode. * * @return string Connection namespace for cache * @task conn */ protected function getConnectionNamespace() { return $this->getDatabaseName(); } abstract protected function getDatabaseName(); /** * Get an existing, cached connection for this object. * * @param mode Connection mode. * @return AphrontDatabaseConnection|null Connection, if it exists in cache. * @task conn */ protected function getEstablishedConnection($mode) { $key = $this->getConnectionNamespace().':'.$mode; if (isset(self::$connections[$key])) { return self::$connections[$key]; } return null; } /** * Store a connection in the connection cache. * * @param mode Connection mode. * @param AphrontDatabaseConnection Connection to cache. * @return this * @task conn */ protected function setEstablishedConnection( $mode, AphrontDatabaseConnection $connection, $force_unique = false) { $key = $this->getConnectionNamespace().':'.$mode; if ($force_unique) { $key .= ':unique'; while (isset(self::$connections[$key])) { $key .= '!'; } } self::$connections[$key] = $connection; return $this; } /** * Force an object to use a specific connection. * * This overrides all connection management and forces the object to use * a specific connection when interacting with the database. * * @param AphrontDatabaseConnection Connection to force this object to use. * @task conn */ public function setForcedConnection(AphrontDatabaseConnection $connection) { $this->forcedConnection = $connection; return $this; } /* -( Configuring Lisk )--------------------------------------------------- */ /** * Change Lisk behaviors, like ID configuration and timestamps. If you want * to change these behaviors, you should override this method in your child * class and change the options you're interested in. For example: * * protected function getConfiguration() { * return array( * Lisk_DataAccessObject::CONFIG_EXAMPLE => true, * ) + parent::getConfiguration(); * } * * The available options are: * * CONFIG_IDS * Lisk objects need to have a unique identifying ID. The three mechanisms * available for generating this ID are IDS_AUTOINCREMENT (default, assumes * the ID column is an autoincrement primary key), IDS_MANUAL (you are taking * full responsibility for ID management), or IDS_COUNTER (see below). * * InnoDB does not persist the value of `auto_increment` across restarts, * and instead initializes it to `MAX(id) + 1` during startup. This means it * may reissue the same autoincrement ID more than once, if the row is deleted * and then the database is restarted. To avoid this, you can set an object to * use a counter table with IDS_COUNTER. This will generally behave like * IDS_AUTOINCREMENT, except that the counter value will persist across * restarts and inserts will be slightly slower. If a database stores any * DAOs which use this mechanism, you must create a table there with this * schema: * * CREATE TABLE lisk_counter ( * counterName VARCHAR(64) COLLATE utf8_bin PRIMARY KEY, * counterValue BIGINT UNSIGNED NOT NULL * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; * * CONFIG_TIMESTAMPS * Lisk can automatically handle keeping track of a `dateCreated' and * `dateModified' column, which it will update when it creates or modifies * an object. If you don't want to do this, you may disable this option. * By default, this option is ON. * * CONFIG_AUX_PHID * This option can be enabled by being set to some truthy value. The meaning * of this value is defined by your PHID generation mechanism. If this option * is enabled, a `phid' property will be populated with a unique PHID when an * object is created (or if it is saved and does not currently have one). You * need to override generatePHID() and hook it into your PHID generation * mechanism for this to work. By default, this option is OFF. * * CONFIG_SERIALIZATION * You can optionally provide a column serialization map that will be applied * to values when they are written to the database. For example: * * self::CONFIG_SERIALIZATION => array( * 'complex' => self::SERIALIZATION_JSON, * ) * * This will cause Lisk to JSON-serialize the 'complex' field before it is * written, and unserialize it when it is read. * * CONFIG_BINARY * You can optionally provide a map of columns to a flag indicating that * they store binary data. These columns will not raise an error when * handling binary writes. * * CONFIG_COLUMN_SCHEMA * Provide a map of columns to schema column types. * * CONFIG_KEY_SCHEMA * Provide a map of key names to key specifications. * * CONFIG_NO_TABLE * Allows you to specify that this object does not actually have a table in * the database. * * CONFIG_NO_MUTATE * Provide a map of columns which should not be included in UPDATE statements. * If you have some columns which are always written to explicitly and should * never be overwritten by a save(), you can specify them here. This is an * advanced, specialized feature and there are usually better approaches for * most locking/contention problems. * * @return dictionary Map of configuration options to values. * * @task config */ protected function getConfiguration() { return array( self::CONFIG_IDS => self::IDS_AUTOINCREMENT, self::CONFIG_TIMESTAMPS => true, ); } /** * Determine the setting of a configuration option for this class of objects. * * @param const Option name, one of the CONFIG_* constants. * @return mixed Option value, if configured (null if unavailable). * * @task config */ public function getConfigOption($option_name) { static $options = null; if (!isset($options)) { $options = $this->getConfiguration(); } return idx($options, $option_name); } /* -( Loading Objects )---------------------------------------------------- */ /** * Load an object by ID. You need to invoke this as an instance method, not * a class method, because PHP doesn't have late static binding (until * PHP 5.3.0). For example: * * $dog = id(new Dog())->load($dog_id); * * @param int Numeric ID identifying the object to load. * @return obj|null Identified object, or null if it does not exist. * * @task load */ public function load($id) { if (is_object($id)) { $id = (string)$id; } if (!$id || (!is_int($id) && !ctype_digit($id))) { return null; } return $this->loadOneWhere( '%C = %d', $this->getIDKeyForUse(), $id); } /** * Loads all of the objects, unconditionally. * * @return dict Dictionary of all persisted objects of this type, keyed * on object ID. * * @task load */ public function loadAll() { return $this->loadAllWhere('1 = 1'); } /** * Load all objects which match a WHERE clause. You provide everything after * the 'WHERE'; Lisk handles everything up to it. For example: * * $old_dogs = id(new Dog())->loadAllWhere('age > %d', 7); * * The pattern and arguments are as per queryfx(). * * @param string queryfx()-style SQL WHERE clause. * @param ... Zero or more conversions. * @return dict Dictionary of matching objects, keyed on ID. * * @task load */ public function loadAllWhere($pattern /* , $arg, $arg, $arg ... */) { $args = func_get_args(); $data = call_user_func_array( array($this, 'loadRawDataWhere'), $args); return $this->loadAllFromArray($data); } /** * Load a single object identified by a 'WHERE' clause. You provide * everything after the 'WHERE', and Lisk builds the first half of the * query. See loadAllWhere(). This method is similar, but returns a single * result instead of a list. * * @param string queryfx()-style SQL WHERE clause. * @param ... Zero or more conversions. * @return obj|null Matching object, or null if no object matches. * * @task load */ public function loadOneWhere($pattern /* , $arg, $arg, $arg ... */) { $args = func_get_args(); $data = call_user_func_array( array($this, 'loadRawDataWhere'), $args); if (count($data) > 1) { throw new AphrontCountQueryException( pht( 'More than one result from %s!', __FUNCTION__.'()')); } $data = reset($data); if (!$data) { return null; } return $this->loadFromArray($data); } protected function loadRawDataWhere($pattern /* , $args... */) { $conn = $this->establishConnection('r'); if ($conn->isReadLocking()) { $lock_clause = qsprintf($conn, 'FOR UPDATE'); } else if ($conn->isWriteLocking()) { $lock_clause = qsprintf($conn, 'LOCK IN SHARE MODE'); } else { $lock_clause = qsprintf($conn, ''); } $args = func_get_args(); $args = array_slice($args, 1); $pattern = 'SELECT * FROM %R WHERE '.$pattern.' %Q'; array_unshift($args, $this); array_push($args, $lock_clause); array_unshift($args, $pattern); return call_user_func_array(array($conn, 'queryData'), $args); } /** * Reload an object from the database, discarding any changes to persistent * properties. This is primarily useful after entering a transaction but * before applying changes to an object. * * @return this * * @task load */ public function reload() { if (!$this->getID()) { throw new Exception( pht("Unable to reload object that hasn't been loaded!")); } $result = $this->loadOneWhere( '%C = %d', $this->getIDKeyForUse(), $this->getID()); if (!$result) { throw new AphrontObjectMissingQueryException(); } return $this; } /** * Initialize this object's properties from a dictionary. Generally, you * load single objects with loadOneWhere(), but sometimes it may be more * convenient to pull data from elsewhere directly (e.g., a complicated * join via @{method:queryData}) and then load from an array representation. * * @param dict Dictionary of properties, which should be equivalent to * selecting a row from the table or calling * @{method:getProperties}. * @return this * * @task load */ public function loadFromArray(array $row) { static $valid_properties = array(); $map = array(); foreach ($row as $k => $v) { // We permit (but ignore) extra properties in the array because a // common approach to building the array is to issue a raw SELECT query // which may include extra explicit columns or joins. // This pathway is very hot on some pages, so we're inlining a cache // and doing some microoptimization to avoid a strtolower() call for each // assignment. The common path (assigning a valid property which we've // already seen) always incurs only one empty(). The second most common // path (assigning an invalid property which we've already seen) costs // an empty() plus an isset(). if (empty($valid_properties[$k])) { if (isset($valid_properties[$k])) { // The value is set but empty, which means it's false, so we've // already determined it's not valid. We don't need to check again. continue; } $valid_properties[$k] = $this->hasProperty($k); if (!$valid_properties[$k]) { continue; } } $map[$k] = $v; } $this->willReadData($map); foreach ($map as $prop => $value) { $this->$prop = $value; } $this->didReadData(); return $this; } /** * Initialize a list of objects from a list of dictionaries. Usually you * load lists of objects with @{method:loadAllWhere}, but sometimes that * isn't flexible enough. One case is if you need to do joins to select the * right objects: * * function loadAllWithOwner($owner) { * $data = $this->queryData( * 'SELECT d.* * FROM owner o * JOIN owner_has_dog od ON o.id = od.ownerID * JOIN dog d ON od.dogID = d.id * WHERE o.id = %d', * $owner); * return $this->loadAllFromArray($data); * } * * This is a lot messier than @{method:loadAllWhere}, but more flexible. * * @param list List of property dictionaries. * @return dict List of constructed objects, keyed on ID. * * @task load */ public function loadAllFromArray(array $rows) { $result = array(); $id_key = $this->getIDKey(); foreach ($rows as $row) { $obj = clone $this; if ($id_key && isset($row[$id_key])) { $row_id = $row[$id_key]; if (isset($result[$row_id])) { throw new Exception( pht( 'Rows passed to "loadAllFromArray(...)" include two or more '. 'rows with the same ID ("%s"). Rows must have unique IDs. '. 'An underlying query may be missing a GROUP BY.', $row_id)); } $result[$row_id] = $obj->loadFromArray($row); } else { $result[] = $obj->loadFromArray($row); } } return $result; } /* -( Examining Objects )-------------------------------------------------- */ /** * Set unique ID identifying this object. You normally don't need to call this * method unless with `IDS_MANUAL`. * * @param mixed Unique ID. * @return this * @task save */ public function setID($id) { static $id_key = null; if ($id_key === null) { $id_key = $this->getIDKeyForUse(); } $this->$id_key = $id; return $this; } /** * Retrieve the unique ID identifying this object. This value will be null if * the object hasn't been persisted and you didn't set it manually. * * @return mixed Unique ID. * * @task info */ public function getID() { static $id_key = null; if ($id_key === null) { $id_key = $this->getIDKeyForUse(); } return $this->$id_key; } public function getPHID() { return $this->phid; } /** * Test if a property exists. * * @param string Property name. * @return bool True if the property exists. * @task info */ public function hasProperty($property) { return (bool)$this->checkProperty($property); } /** * Retrieve a list of all object properties. This list only includes * properties that are declared as protected, and it is expected that * all properties returned by this function should be persisted to the * database. * Properties that should not be persisted must be declared as private. * * @return dict Dictionary of normalized (lowercase) to canonical (original * case) property names. * * @task info */ protected function getAllLiskProperties() { static $properties = null; if (!isset($properties)) { $class = new ReflectionClass(get_class($this)); $properties = array(); foreach ($class->getProperties(ReflectionProperty::IS_PROTECTED) as $p) { $properties[strtolower($p->getName())] = $p->getName(); } $id_key = $this->getIDKey(); if ($id_key != 'id') { unset($properties['id']); } if (!$this->getConfigOption(self::CONFIG_TIMESTAMPS)) { unset($properties['datecreated']); unset($properties['datemodified']); } if ($id_key != 'phid' && !$this->getConfigOption(self::CONFIG_AUX_PHID)) { unset($properties['phid']); } } return $properties; } /** * Check if a property exists on this object. * * @return string|null Canonical property name, or null if the property * does not exist. * * @task info */ protected function checkProperty($property) { static $properties = null; if ($properties === null) { $properties = $this->getAllLiskProperties(); } $property = strtolower($property); if (empty($properties[$property])) { return null; } return $properties[$property]; } /** * Get or build the database connection for this object. * * @param string 'r' for read, 'w' for read/write. * @param bool True to force a new connection. The connection will not * be retrieved from or saved into the connection cache. * @return AphrontDatabaseConnection Lisk connection object. * * @task info */ public function establishConnection($mode, $force_new = false) { if ($mode != 'r' && $mode != 'w') { throw new Exception( pht( "Unknown mode '%s', should be 'r' or 'w'.", $mode)); } if ($this->forcedConnection) { return $this->forcedConnection; } if (self::shouldIsolateAllLiskEffectsToCurrentProcess()) { $mode = 'isolate-'.$mode; $connection = $this->getEstablishedConnection($mode); if (!$connection) { $connection = $this->establishIsolatedConnection($mode); $this->setEstablishedConnection($mode, $connection); } return $connection; } if (self::shouldIsolateAllLiskEffectsToTransactions()) { // If we're doing fixture transaction isolation, force the mode to 'w' // so we always get the same connection for reads and writes, and thus // can see the writes inside the transaction. $mode = 'w'; } // TODO: There is currently no protection on 'r' queries against writing. $connection = null; if (!$force_new) { if ($mode == 'r') { // If we're requesting a read connection but already have a write // connection, reuse the write connection so that reads can take place // inside transactions. $connection = $this->getEstablishedConnection('w'); } if (!$connection) { $connection = $this->getEstablishedConnection($mode); } } if (!$connection) { $connection = $this->establishLiveConnection($mode); if (self::shouldIsolateAllLiskEffectsToTransactions()) { $connection->openTransaction(); } $this->setEstablishedConnection( $mode, $connection, $force_unique = $force_new); } return $connection; } /** * Convert this object into a property dictionary. This dictionary can be * restored into an object by using @{method:loadFromArray} (unless you're * using legacy features with CONFIG_CONVERT_CAMELCASE, but in that case you * should just go ahead and die in a fire). * * @return dict Dictionary of object properties. * * @task info */ protected function getAllLiskPropertyValues() { $map = array(); foreach ($this->getAllLiskProperties() as $p) { // We may receive a warning here for properties we've implicitly added // through configuration; squelch it. $map[$p] = @$this->$p; } return $map; } /* -( Writing Objects )---------------------------------------------------- */ /** * Make an object read-only. * * Making an object ephemeral indicates that you will be changing state in * such a way that you would never ever want it to be written back to the * storage. */ public function makeEphemeral() { $this->ephemeral = true; return $this; } private function isEphemeralCheck() { if ($this->ephemeral) { throw new LiskEphemeralObjectException(); } } /** * Persist this object to the database. In most cases, this is the only * method you need to call to do writes. If the object has not yet been * inserted this will do an insert; if it has, it will do an update. * * @return this * * @task save */ public function save() { if ($this->shouldInsertWhenSaved()) { return $this->insert(); } else { return $this->update(); } } /** * Save this object, forcing the query to use REPLACE regardless of object * state. * * @return this * * @task save */ public function replace() { $this->isEphemeralCheck(); return $this->insertRecordIntoDatabase('REPLACE'); } /** * Save this object, forcing the query to use INSERT regardless of object * state. * * @return this * * @task save */ public function insert() { $this->isEphemeralCheck(); return $this->insertRecordIntoDatabase('INSERT'); } /** * Save this object, forcing the query to use UPDATE regardless of object * state. * * @return this * * @task save */ public function update() { $this->isEphemeralCheck(); $this->willSaveObject(); $data = $this->getAllLiskPropertyValues(); // Remove columns flagged as nonmutable from the update statement. $no_mutate = $this->getConfigOption(self::CONFIG_NO_MUTATE); if ($no_mutate) { foreach ($no_mutate as $column) { unset($data[$column]); } } $this->willWriteData($data); $map = array(); foreach ($data as $k => $v) { $map[$k] = $v; } $conn = $this->establishConnection('w'); $binary = $this->getBinaryColumns(); foreach ($map as $key => $value) { if (!empty($binary[$key])) { $map[$key] = qsprintf($conn, '%C = %nB', $key, $value); } else { $map[$key] = qsprintf($conn, '%C = %ns', $key, $value); } } $id = $this->getID(); $conn->query( 'UPDATE %R SET %LQ WHERE %C = '.(is_int($id) ? '%d' : '%s'), $this, $map, $this->getIDKeyForUse(), $id); // We can't detect a missing object because updating an object without // changing any values doesn't affect rows. We could jiggle timestamps // to catch this for objects which track them if we wanted. $this->didWriteData(); return $this; } /** * Delete this object, permanently. * * @return this * * @task save */ public function delete() { $this->isEphemeralCheck(); $this->willDelete(); $conn = $this->establishConnection('w'); $conn->query( 'DELETE FROM %R WHERE %C = %d', $this, $this->getIDKeyForUse(), $this->getID()); $this->didDelete(); return $this; } /** * Internal implementation of INSERT and REPLACE. * * @param const Either "INSERT" or "REPLACE", to force the desired mode. * @return this * * @task save */ protected function insertRecordIntoDatabase($mode) { $this->willSaveObject(); $data = $this->getAllLiskPropertyValues(); $conn = $this->establishConnection('w'); $id_mechanism = $this->getConfigOption(self::CONFIG_IDS); switch ($id_mechanism) { case self::IDS_AUTOINCREMENT: // If we are using autoincrement IDs, let MySQL assign the value for the // ID column, if it is empty. If the caller has explicitly provided a // value, use it. $id_key = $this->getIDKeyForUse(); if (empty($data[$id_key])) { unset($data[$id_key]); } break; case self::IDS_COUNTER: // If we are using counter IDs, assign a new ID if we don't already have // one. $id_key = $this->getIDKeyForUse(); if (empty($data[$id_key])) { $counter_name = $this->getTableName(); $id = self::loadNextCounterValue($conn, $counter_name); $this->setID($id); $data[$id_key] = $id; } break; case self::IDS_MANUAL: break; default: throw new Exception(pht('Unknown %s mechanism!', 'CONFIG_IDs')); } $this->willWriteData($data); $columns = array_keys($data); $binary = $this->getBinaryColumns(); foreach ($data as $key => $value) { try { if (!empty($binary[$key])) { $data[$key] = qsprintf($conn, '%nB', $value); } else { $data[$key] = qsprintf($conn, '%ns', $value); } } catch (AphrontParameterQueryException $parameter_exception) { throw new PhutilProxyException( pht( "Unable to insert or update object of class %s, field '%s' ". "has a non-scalar value.", get_class($this), $key), $parameter_exception); } } switch ($mode) { case 'INSERT': $verb = qsprintf($conn, 'INSERT'); break; case 'REPLACE': $verb = qsprintf($conn, 'REPLACE'); break; default: throw new Exception( pht( 'Insert mode verb "%s" is not recognized, use INSERT or REPLACE.', $mode)); } $conn->query( '%Q INTO %R (%LC) VALUES (%LQ)', $verb, $this, $columns, $data); // Only use the insert id if this table is using auto-increment ids if ($id_mechanism === self::IDS_AUTOINCREMENT) { $this->setID($conn->getInsertID()); } $this->didWriteData(); return $this; } /** * Method used to determine whether to insert or update when saving. * * @return bool true if the record should be inserted */ protected function shouldInsertWhenSaved() { $key_type = $this->getConfigOption(self::CONFIG_IDS); if ($key_type == self::IDS_MANUAL) { throw new Exception( pht( 'You are using manual IDs. You must override the %s method '. 'to properly detect when to insert a new record.', __FUNCTION__.'()')); } else { return !$this->getID(); } } /* -( Hooks and Callbacks )------------------------------------------------ */ /** * Retrieve the database table name. By default, this is the class name. * * @return string Table name for object storage. * * @task hook */ public function getTableName() { return get_class($this); } /** * Retrieve the primary key column, "id" by default. If you can not * reasonably name your ID column "id", override this method. * * @return string Name of the ID column. * * @task hook */ public function getIDKey() { return 'id'; } protected function getIDKeyForUse() { $id_key = $this->getIDKey(); if (!$id_key) { throw new Exception( pht( 'This DAO does not have a single-part primary key. The method you '. 'called requires a single-part primary key.')); } return $id_key; } /** * Generate a new PHID, used by CONFIG_AUX_PHID. * * @return phid Unique, newly allocated PHID. * * @task hook */ public function generatePHID() { $type = $this->getPHIDType(); return PhabricatorPHID::generateNewPHID($type); } public function getPHIDType() { throw new PhutilMethodNotImplementedException(); } /** * Hook to apply serialization or validation to data before it is written to * the database. See also @{method:willReadData}. * * @task hook */ protected function willWriteData(array &$data) { $this->applyLiskDataSerialization($data, false); } /** * Hook to perform actions after data has been written to the database. * * @task hook */ protected function didWriteData() {} /** * Hook to make internal object state changes prior to INSERT, REPLACE or * UPDATE. * * @task hook */ protected function willSaveObject() { $use_timestamps = $this->getConfigOption(self::CONFIG_TIMESTAMPS); if ($use_timestamps) { if (!$this->getDateCreated()) { $this->setDateCreated(time()); } $this->setDateModified(time()); } if ($this->getConfigOption(self::CONFIG_AUX_PHID) && !$this->getPHID()) { $this->setPHID($this->generatePHID()); } } /** * Hook to apply serialization or validation to data as it is read from the * database. See also @{method:willWriteData}. * * @task hook */ protected function willReadData(array &$data) { $this->applyLiskDataSerialization($data, $deserialize = true); } /** * Hook to perform an action on data after it is read from the database. * * @task hook */ protected function didReadData() {} /** * Hook to perform an action before the deletion of an object. * * @task hook */ protected function willDelete() {} /** * Hook to perform an action after the deletion of an object. * * @task hook */ protected function didDelete() {} /** * Reads the value from a field. Override this method for custom behavior * of @{method:getField} instead of overriding getField directly. * * @param string Canonical field name * @return mixed Value of the field * * @task hook */ protected function readField($field) { if (isset($this->$field)) { return $this->$field; } return null; } /** * Writes a value to a field. Override this method for custom behavior of * setField($value) instead of overriding setField directly. * * @param string Canonical field name * @param mixed Value to write * * @task hook */ protected function writeField($field, $value) { $this->$field = $value; } /* -( Manging Transactions )----------------------------------------------- */ /** * Increase transaction stack depth. * * @return this */ public function openTransaction() { $this->establishConnection('w')->openTransaction(); return $this; } /** * Decrease transaction stack depth, saving work. * * @return this */ public function saveTransaction() { $this->establishConnection('w')->saveTransaction(); return $this; } /** * Decrease transaction stack depth, discarding work. * * @return this */ public function killTransaction() { $this->establishConnection('w')->killTransaction(); return $this; } /** * Begins read-locking selected rows with SELECT ... FOR UPDATE, so that * other connections can not read them (this is an enormous oversimplification * of FOR UPDATE semantics; consult the MySQL documentation for details). To * end read locking, call @{method:endReadLocking}. For example: * * $beach->openTransaction(); * $beach->beginReadLocking(); * * $beach->reload(); * $beach->setGrainsOfSand($beach->getGrainsOfSand() + 1); * $beach->save(); * * $beach->endReadLocking(); * $beach->saveTransaction(); * * @return this * @task xaction */ public function beginReadLocking() { $this->establishConnection('w')->beginReadLocking(); return $this; } /** * Ends read-locking that began at an earlier @{method:beginReadLocking} call. * * @return this * @task xaction */ public function endReadLocking() { $this->establishConnection('w')->endReadLocking(); return $this; } /** * Begins write-locking selected rows with SELECT ... LOCK IN SHARE MODE, so * that other connections can not update or delete them (this is an * oversimplification of LOCK IN SHARE MODE semantics; consult the * MySQL documentation for details). To end write locking, call * @{method:endWriteLocking}. * * @return this * @task xaction */ public function beginWriteLocking() { $this->establishConnection('w')->beginWriteLocking(); return $this; } /** * Ends write-locking that began at an earlier @{method:beginWriteLocking} * call. * * @return this * @task xaction */ public function endWriteLocking() { $this->establishConnection('w')->endWriteLocking(); return $this; } /* -( Isolation )---------------------------------------------------------- */ /** * @task isolate */ public static function beginIsolateAllLiskEffectsToCurrentProcess() { self::$processIsolationLevel++; } /** * @task isolate */ public static function endIsolateAllLiskEffectsToCurrentProcess() { self::$processIsolationLevel--; if (self::$processIsolationLevel < 0) { throw new Exception( pht('Lisk process isolation level was reduced below 0.')); } } /** * @task isolate */ public static function shouldIsolateAllLiskEffectsToCurrentProcess() { return (bool)self::$processIsolationLevel; } /** * @task isolate */ private function establishIsolatedConnection($mode) { $config = array(); return new AphrontIsolatedDatabaseConnection($config); } /** * @task isolate */ public static function beginIsolateAllLiskEffectsToTransactions() { if (self::$transactionIsolationLevel === 0) { self::closeAllConnections(); } self::$transactionIsolationLevel++; } /** * @task isolate */ public static function endIsolateAllLiskEffectsToTransactions() { self::$transactionIsolationLevel--; if (self::$transactionIsolationLevel < 0) { throw new Exception( pht('Lisk transaction isolation level was reduced below 0.')); } else if (self::$transactionIsolationLevel == 0) { foreach (self::$connections as $key => $conn) { if ($conn) { $conn->killTransaction(); } } self::closeAllConnections(); } } /** * @task isolate */ public static function shouldIsolateAllLiskEffectsToTransactions() { return (bool)self::$transactionIsolationLevel; } /** * Close any connections with no recent activity. * * Long-running processes can use this method to clean up connections which * have not been used recently. * * @param int Close connections with no activity for this many seconds. * @return void */ public static function closeInactiveConnections($idle_window) { $connections = self::$connections; $now = PhabricatorTime::getNow(); foreach ($connections as $key => $connection) { // If the connection is not idle, never consider it inactive. if (!$connection->isIdle()) { continue; } $last_active = $connection->getLastActiveEpoch(); $idle_duration = ($now - $last_active); if ($idle_duration <= $idle_window) { continue; } self::closeConnection($key); } } public static function closeAllConnections() { $connections = self::$connections; foreach ($connections as $key => $connection) { self::closeConnection($key); } } public static function closeIdleConnections() { $connections = self::$connections; foreach ($connections as $key => $connection) { if (!$connection->isIdle()) { continue; } self::closeConnection($key); } } private static function closeConnection($key) { if (empty(self::$connections[$key])) { throw new Exception( pht( 'No database connection with connection key "%s" exists!', $key)); } $connection = self::$connections[$key]; unset(self::$connections[$key]); $connection->close(); } /* -( Utilities )---------------------------------------------------------- */ /** * Applies configured serialization to a dictionary of values. * * @task util */ protected function applyLiskDataSerialization(array &$data, $deserialize) { $serialization = $this->getConfigOption(self::CONFIG_SERIALIZATION); if ($serialization) { foreach (array_intersect_key($serialization, $data) as $col => $format) { switch ($format) { case self::SERIALIZATION_NONE: break; case self::SERIALIZATION_PHP: if ($deserialize) { $data[$col] = unserialize($data[$col]); } else { $data[$col] = serialize($data[$col]); } break; case self::SERIALIZATION_JSON: if ($deserialize) { $data[$col] = json_decode($data[$col], true); } else { $data[$col] = phutil_json_encode($data[$col]); } break; default: throw new Exception( pht("Unknown serialization format '%s'.", $format)); } } } } /** * Black magic. Builds implied get*() and set*() for all properties. * * @param string Method name. * @param list Argument vector. * @return mixed get*() methods return the property value. set*() methods * return $this. * @task util */ public function __call($method, $args) { // NOTE: PHP has a bug that static variables defined in __call() are shared // across all children classes. Call a different method to work around this // bug. return $this->call($method, $args); } /** * @task util */ final protected function call($method, $args) { // NOTE: This method is very performance-sensitive (many thousands of calls // per page on some pages), and thus has some silliness in the name of // optimizations. static $dispatch_map = array(); if ($method[0] === 'g') { if (isset($dispatch_map[$method])) { $property = $dispatch_map[$method]; } else { if (substr($method, 0, 3) !== 'get') { throw new Exception(pht("Unable to resolve method '%s'!", $method)); } $property = substr($method, 3); if (!($property = $this->checkProperty($property))) { throw new Exception(pht('Bad getter call: %s', $method)); } $dispatch_map[$method] = $property; } return $this->readField($property); } if ($method[0] === 's') { if (isset($dispatch_map[$method])) { $property = $dispatch_map[$method]; } else { if (substr($method, 0, 3) !== 'set') { throw new Exception(pht("Unable to resolve method '%s'!", $method)); } $property = substr($method, 3); $property = $this->checkProperty($property); if (!$property) { throw new Exception(pht('Bad setter call: %s', $method)); } $dispatch_map[$method] = $property; } $this->writeField($property, $args[0]); return $this; } throw new Exception(pht("Unable to resolve method '%s'.", $method)); } /** * Warns against writing to undeclared property. * * @task util */ public function __set($name, $value) { // Hack for policy system hints, see PhabricatorPolicyRule for notes. if ($name != '_hashKey') { phlog( pht( 'Wrote to undeclared property %s.', get_class($this).'::$'.$name)); } $this->$name = $value; } /** * Increments a named counter and returns the next value. * * @param AphrontDatabaseConnection Database where the counter resides. * @param string Counter name to create or increment. * @return int Next counter value. * * @task util */ public static function loadNextCounterValue( AphrontDatabaseConnection $conn_w, $counter_name) { // NOTE: If an insert does not touch an autoincrement row or call // LAST_INSERT_ID(), MySQL normally does not change the value of // LAST_INSERT_ID(). This can cause a counter's value to leak to a // new counter if the second counter is created after the first one is // updated. To avoid this, we insert LAST_INSERT_ID(1), to ensure the // LAST_INSERT_ID() is always updated and always set correctly after the // query completes. queryfx( $conn_w, 'INSERT INTO %T (counterName, counterValue) VALUES (%s, LAST_INSERT_ID(1)) ON DUPLICATE KEY UPDATE counterValue = LAST_INSERT_ID(counterValue + 1)', self::COUNTER_TABLE_NAME, $counter_name); return $conn_w->getInsertID(); } /** * Returns the current value of a named counter. * * @param AphrontDatabaseConnection Database where the counter resides. * @param string Counter name to read. * @return int|null Current value, or `null` if the counter does not exist. * * @task util */ public static function loadCurrentCounterValue( AphrontDatabaseConnection $conn_r, $counter_name) { $row = queryfx_one( $conn_r, 'SELECT counterValue FROM %T WHERE counterName = %s', self::COUNTER_TABLE_NAME, $counter_name); if (!$row) { return null; } return (int)$row['counterValue']; } /** * Overwrite a named counter, forcing it to a specific value. * * If the counter does not exist, it is created. * * @param AphrontDatabaseConnection Database where the counter resides. * @param string Counter name to create or overwrite. * @return void * * @task util */ public static function overwriteCounterValue( AphrontDatabaseConnection $conn_w, $counter_name, $counter_value) { queryfx( $conn_w, 'INSERT INTO %T (counterName, counterValue) VALUES (%s, %d) ON DUPLICATE KEY UPDATE counterValue = VALUES(counterValue)', self::COUNTER_TABLE_NAME, $counter_name, $counter_value); } private function getBinaryColumns() { return $this->getConfigOption(self::CONFIG_BINARY); } public function getSchemaColumns() { $custom_map = $this->getConfigOption(self::CONFIG_COLUMN_SCHEMA); if (!$custom_map) { $custom_map = array(); } $serialization = $this->getConfigOption(self::CONFIG_SERIALIZATION); if (!$serialization) { $serialization = array(); } $serialization_map = array( self::SERIALIZATION_JSON => 'text', self::SERIALIZATION_PHP => 'bytes', ); $binary_map = $this->getBinaryColumns(); $id_mechanism = $this->getConfigOption(self::CONFIG_IDS); if ($id_mechanism == self::IDS_AUTOINCREMENT) { $id_type = 'auto'; } else { $id_type = 'id'; } $builtin = array( 'id' => $id_type, 'phid' => 'phid', 'viewPolicy' => 'policy', 'editPolicy' => 'policy', 'epoch' => 'epoch', 'dateCreated' => 'epoch', 'dateModified' => 'epoch', ); $map = array(); foreach ($this->getAllLiskProperties() as $property) { // First, use types specified explicitly in the table configuration. if (array_key_exists($property, $custom_map)) { $map[$property] = $custom_map[$property]; continue; } // If we don't have an explicit type, try a builtin type for the // column. $type = idx($builtin, $property); if ($type) { $map[$property] = $type; continue; } // If the column has serialization, we can infer the column type. if (isset($serialization[$property])) { $type = idx($serialization_map, $serialization[$property]); if ($type) { $map[$property] = $type; continue; } } if (isset($binary_map[$property])) { $map[$property] = 'bytes'; continue; } if ($property === 'spacePHID') { $map[$property] = 'phid?'; continue; } // If the column is named `somethingPHID`, infer it is a PHID. if (preg_match('/[a-z]PHID$/', $property)) { $map[$property] = 'phid'; continue; } // If the column is named `somethingID`, infer it is an ID. if (preg_match('/[a-z]ID$/', $property)) { $map[$property] = 'id'; continue; } // We don't know the type of this column. $map[$property] = PhabricatorConfigSchemaSpec::DATATYPE_UNKNOWN; } return $map; } public function getSchemaKeys() { $custom_map = $this->getConfigOption(self::CONFIG_KEY_SCHEMA); if (!$custom_map) { $custom_map = array(); } $default_map = array(); foreach ($this->getAllLiskProperties() as $property) { switch ($property) { case 'id': $default_map['PRIMARY'] = array( 'columns' => array('id'), 'unique' => true, ); break; case 'phid': $default_map['key_phid'] = array( 'columns' => array('phid'), 'unique' => true, ); break; case 'spacePHID': $default_map['key_space'] = array( 'columns' => array('spacePHID'), ); break; } } return $custom_map + $default_map; } public function getColumnMaximumByteLength($column) { $map = $this->getSchemaColumns(); if (!isset($map[$column])) { throw new Exception( pht( 'Object (of class "%s") does not have a column "%s".', get_class($this), $column)); } $data_type = $map[$column]; return id(new PhabricatorStorageSchemaSpec()) ->getMaximumByteLengthForDataType($data_type); } public function getSchemaPersistence() { return null; } /* -( AphrontDatabaseTableRefInterface )----------------------------------- */ public function getAphrontRefDatabaseName() { return $this->getDatabaseName(); } public function getAphrontRefTableName() { return $this->getTableName(); } } diff --git a/support/startup/PhabricatorStartup.php b/support/startup/PhabricatorStartup.php index bf66f8839b..b687f27bb0 100644 --- a/support/startup/PhabricatorStartup.php +++ b/support/startup/PhabricatorStartup.php @@ -1,788 +1,788 @@ setEncoding($encoding); } $input = ''; do { $bytes = $stream->readData(); if ($bytes === null) { break; } $input .= $bytes; } while (true); self::$rawInput = $input; } return self::$rawInput; } /* -( Startup Hooks )------------------------------------------------------ */ /** * @param float Request start time, from `microtime(true)`. * @task hook */ public static function didStartup($start_time) { self::$startTime = $start_time; self::$phases = array(); self::$accessLog = null; self::$requestPath = null; static $registered; if (!$registered) { // NOTE: This protects us against multiple calls to didStartup() in the // same request, but also against repeated requests to the same // interpreter state, which we may implement in the future. register_shutdown_function(array(__CLASS__, 'didShutdown')); $registered = true; } self::setupPHP(); self::verifyPHP(); // If we've made it this far, the environment isn't completely broken so // we can switch over to relying on our own exception recovery mechanisms. ini_set('display_errors', 0); self::connectRateLimits(); self::normalizeInput(); self::readRequestPath(); self::beginOutputCapture(); } /** * @task hook */ public static function didShutdown() { // Disconnect any active rate limits before we shut down. If we don't do // this, requests which exit early will lock a slot in any active // connection limits, and won't count for rate limits. self::disconnectRateLimits(array()); $event = error_get_last(); if (!$event) { return; } switch ($event['type']) { case E_ERROR: case E_PARSE: case E_COMPILE_ERROR: break; default: return; } $msg = ">>> UNRECOVERABLE FATAL ERROR <<<\n\n"; if ($event) { // Even though we should be emitting this as text-plain, escape things // just to be sure since we can't really be sure what the program state // is when we get here. $msg .= htmlspecialchars( $event['message']."\n\n".$event['file'].':'.$event['line'], ENT_QUOTES, 'UTF-8'); } // flip dem tables $msg .= "\n\n\n"; $msg .= "\xe2\x94\xbb\xe2\x94\x81\xe2\x94\xbb\x20\xef\xb8\xb5\x20\xc2\xaf". "\x5c\x5f\x28\xe3\x83\x84\x29\x5f\x2f\xc2\xaf\x20\xef\xb8\xb5\x20". "\xe2\x94\xbb\xe2\x94\x81\xe2\x94\xbb"; self::didFatal($msg); } public static function loadCoreLibraries() { $phabricator_root = dirname(dirname(dirname(__FILE__))); $libraries_root = dirname($phabricator_root); $root = null; if (!empty($_SERVER['PHUTIL_LIBRARY_ROOT'])) { $root = $_SERVER['PHUTIL_LIBRARY_ROOT']; } ini_set( 'include_path', $libraries_root.PATH_SEPARATOR.ini_get('include_path')); $ok = @include_once $root.'arcanist/src/init/init-library.php'; if (!$ok) { self::didFatal( 'Unable to load the "Arcanist" library. Put "arcanist/" next to '. '"phabricator/" on disk.'); } // Load Phabricator itself using the absolute path, so we never end up doing // anything surprising (loading index.php and libraries from different // directories). phutil_load_library($phabricator_root.'/src'); } /* -( Output Capture )----------------------------------------------------- */ public static function beginOutputCapture() { if (self::$capturingOutput) { self::didFatal('Already capturing output!'); } self::$capturingOutput = true; ob_start(); } public static function endOutputCapture() { if (!self::$capturingOutput) { return null; } self::$capturingOutput = false; return ob_get_clean(); } /* -( Debug Time Limit )--------------------------------------------------- */ /** * Set a time limit (in seconds) for the current script. After time expires, * the script fatals. * * This works like `max_execution_time`, but prints out a useful stack trace * when the time limit expires. This is primarily intended to make it easier * to debug pages which hang by allowing extraction of a stack trace: set a * short debug limit, then use the trace to figure out what's happening. * * The limit is implemented with a tick function, so enabling it implies * some accounting overhead. * * @param int Time limit in seconds. * @return void */ public static function setDebugTimeLimit($limit) { self::$debugTimeLimit = $limit; static $initialized; if (!$initialized) { declare(ticks=1); register_tick_function(array(__CLASS__, 'onDebugTick')); } } /** * Callback tick function used by @{method:setDebugTimeLimit}. * * Fatals with a useful stack trace after the time limit expires. * * @return void */ public static function onDebugTick() { $limit = self::$debugTimeLimit; if (!$limit) { return; } $elapsed = (microtime(true) - self::getStartTime()); if ($elapsed > $limit) { $frames = array(); foreach (debug_backtrace() as $frame) { $file = isset($frame['file']) ? $frame['file'] : '-'; $file = basename($file); $line = isset($frame['line']) ? $frame['line'] : '-'; $class = isset($frame['class']) ? $frame['class'].'->' : null; $func = isset($frame['function']) ? $frame['function'].'()' : '?'; $frames[] = "{$file}:{$line} {$class}{$func}"; } self::didFatal( "Request aborted by debug time limit after {$limit} seconds.\n\n". "STACK TRACE\n". implode("\n", $frames)); } } /* -( In Case of Apocalypse )---------------------------------------------- */ /** * Fatal the request completely in response to an exception, sending a plain * text message to the client. Calls @{method:didFatal} internally. * * @param string Brief description of the exception context, like * `"Rendering Exception"`. * @param Throwable The exception itself. * @param bool True if it's okay to show the exception's stack trace * to the user. The trace will always be logged. * @return exit This method **does not return**. * * @task apocalypse */ public static function didEncounterFatalException( $note, $ex, $show_trace) { $message = '['.$note.'/'.get_class($ex).'] '.$ex->getMessage(); $full_message = $message; $full_message .= "\n\n"; $full_message .= $ex->getTraceAsString(); if ($show_trace) { $message = $full_message; } self::didFatal($message, $full_message); } /** * Fatal the request completely, sending a plain text message to the client. * * @param string Plain text message to send to the client. * @param string Plain text message to send to the error log. If not * provided, the client message is used. You can pass a more * detailed message here (e.g., with stack traces) to avoid * showing it to users. * @return exit This method **does not return**. * * @task apocalypse */ public static function didFatal($message, $log_message = null) { if ($log_message === null) { $log_message = $message; } self::endOutputCapture(); $access_log = self::$accessLog; if ($access_log) { // We may end up here before the access log is initialized, e.g. from // verifyPHP(). $access_log->setData( array( 'c' => 500, )); $access_log->write(); } header( 'Content-Type: text/plain; charset=utf-8', $replace = true, $http_error = 500); error_log($log_message); echo $message."\n"; exit(1); } /* -( Validation )--------------------------------------------------------- */ /** * @task validation */ private static function setupPHP() { error_reporting(E_ALL | E_STRICT); self::$oldMemoryLimit = ini_get('memory_limit'); ini_set('memory_limit', -1); // If we have libxml, disable the incredibly dangerous entity loader. if (function_exists('libxml_disable_entity_loader')) { libxml_disable_entity_loader(true); } // See T13060. If the locale for this process (the parent process) is not // a UTF-8 locale we can encounter problems when launching subprocesses // which receive UTF-8 parameters in their command line argument list. @setlocale(LC_ALL, 'en_US.UTF-8'); $config_map = array( // See PHI1894. Keep "args" in exception backtraces. 'zend.exception_ignore_args' => 0, // See T13100. We'd like the regex engine to fail, rather than segfault, // if handed a pathological regular expression. 'pcre.backtrack_limit' => 10000, 'pcre.recusion_limit' => 10000, // NOTE: Arcanist applies a similar set of startup options for CLI // environments in "init-script.php". Changes here may also be // appropriate to apply there. ); foreach ($config_map as $config_key => $config_value) { ini_set($config_key, $config_value); } } /** * @task validation */ public static function getOldMemoryLimit() { return self::$oldMemoryLimit; } /** * @task validation */ private static function normalizeInput() { // Replace superglobals with unfiltered versions, disrespect php.ini (we // filter ourselves). // NOTE: We don't filter INPUT_SERVER because we don't want to overwrite // changes made in "preamble.php". // NOTE: WE don't filter INPUT_POST because we may be constructing it // lazily if "enable_post_data_reading" is disabled. $filter = array( INPUT_GET, INPUT_ENV, INPUT_COOKIE, ); foreach ($filter as $type) { $filtered = filter_input_array($type, FILTER_UNSAFE_RAW); if (!is_array($filtered)) { continue; } switch ($type) { case INPUT_GET: $_GET = array_merge($_GET, $filtered); break; case INPUT_COOKIE: $_COOKIE = array_merge($_COOKIE, $filtered); break; case INPUT_ENV; $env = array_merge($_ENV, $filtered); $_ENV = self::filterEnvSuperglobal($env); break; } } self::rebuildRequest(); } /** * @task validation */ public static function rebuildRequest() { // Rebuild $_REQUEST, respecting order declared in ".ini" files. $order = ini_get('request_order'); if (!$order) { $order = ini_get('variables_order'); } if (!$order) { // $_REQUEST will be empty, so leave it alone. return; } $_REQUEST = array(); for ($ii = 0; $ii < strlen($order); $ii++) { switch ($order[$ii]) { case 'G': $_REQUEST = array_merge($_REQUEST, $_GET); break; case 'P': $_REQUEST = array_merge($_REQUEST, $_POST); break; case 'C': $_REQUEST = array_merge($_REQUEST, $_COOKIE); break; default: // $_ENV and $_SERVER never go into $_REQUEST. break; } } } /** * Adjust `$_ENV` before execution. * * Adjustments here primarily impact the environment as seen by subprocesses. * The environment is forwarded explicitly by @{class:ExecFuture}. * * @param map Input `$_ENV`. * @return map Suitable `$_ENV`. * @task validation */ private static function filterEnvSuperglobal(array $env) { // In some configurations, we may get "argc" and "argv" set in $_ENV. // These are not real environmental variables, and "argv" may have an array // value which can not be forwarded to subprocesses. Remove these from the // environment if they are present. unset($env['argc']); unset($env['argv']); return $env; } /** * @task validation */ private static function verifyPHP() { $required_version = '5.2.3'; if (version_compare(PHP_VERSION, $required_version) < 0) { self::didFatal( "You are running PHP version '".PHP_VERSION."', which is older than ". "the minimum version, '{$required_version}'. Update to at least ". "'{$required_version}'."); } if (function_exists('get_magic_quotes_gpc')) { if (@get_magic_quotes_gpc()) { self::didFatal( 'Your server is configured with the PHP language feature '. '"magic_quotes_gpc" enabled.'. "\n\n". 'This feature is "highly discouraged" by PHP\'s developers, and '. 'has been removed entirely in PHP8.'. "\n\n". 'You must disable "magic_quotes_gpc" to run Phabricator. Consult '. 'the PHP manual for instructions.'); } } if (extension_loaded('apc')) { $apc_version = phpversion('apc'); $known_bad = array( '3.1.14' => true, '3.1.15' => true, '3.1.15-dev' => true, ); if (isset($known_bad[$apc_version])) { self::didFatal( "You have APC {$apc_version} installed. This version of APC is ". "known to be bad, and does not work with Phabricator (it will ". "cause Phabricator to fatal unrecoverably with nonsense errors). ". "Downgrade to version 3.1.13."); } } if (isset($_SERVER['HTTP_PROXY'])) { self::didFatal( 'This HTTP request included a "Proxy:" header, poisoning the '. 'environment (CVE-2016-5385 / httpoxy). Declining to process this '. 'request. For details, see: https://phurl.io/u/httpoxy'); } } /** * @task request-path */ private static function readRequestPath() { // See T13575. The request path may be provided in: // // - the "$_GET" parameter "__path__" (normal for Apache and nginx); or // - the "$_SERVER" parameter "REQUEST_URI" (normal for the PHP builtin // webserver). // // Locate it wherever it is, and store it for later use. Note that writing // to "$_REQUEST" here won't always work, because later code may rebuild // "$_REQUEST" from other sources. if (isset($_REQUEST['__path__']) && strlen($_REQUEST['__path__'])) { self::setRequestPath($_REQUEST['__path__']); return; } // Compatibility with PHP 5.4+ built-in web server. if (php_sapi_name() == 'cli-server') { $path = parse_url($_SERVER['REQUEST_URI']); self::setRequestPath($path['path']); return; } if (!isset($_REQUEST['__path__'])) { self::didFatal( "Request parameter '__path__' is not set. Your rewrite rules ". "are not configured correctly."); } if (!strlen($_REQUEST['__path__'])) { self::didFatal( "Request parameter '__path__' is set, but empty. Your rewrite rules ". "are not configured correctly. The '__path__' should always ". "begin with a '/'."); } } /** * @task request-path */ public static function getRequestPath() { $path = self::$requestPath; if ($path === null) { self::didFatal( 'Request attempted to access request path, but no request path is '. 'available for this request. You may be calling web request code '. 'from a non-request context, or your webserver may not be passing '. 'a request path to Phabricator in a format that it understands.'); } return $path; } /** * @task request-path */ public static function setRequestPath($path) { self::$requestPath = $path; } /* -( Rate Limiting )------------------------------------------------------ */ /** * Add a new client limits. * * @param PhabricatorClientLimit New limit. * @return PhabricatorClientLimit The limit. */ public static function addRateLimit(PhabricatorClientLimit $limit) { self::$limits[] = $limit; return $limit; } /** * Apply configured rate limits. * * If any limit is exceeded, this method terminates the request. * * @return void * @task ratelimit */ private static function connectRateLimits() { $limits = self::$limits; $reason = null; $connected = array(); foreach ($limits as $limit) { $reason = $limit->didConnect(); $connected[] = $limit; if ($reason !== null) { break; } } // If we're killing the request here, disconnect any limits that we // connected to try to keep the accounting straight. if ($reason !== null) { foreach ($connected as $limit) { $limit->didDisconnect(array()); } self::didRateLimit($reason); } } /** * Tear down rate limiting and allow limits to score the request. * * @param map Additional, freeform request state. * @return void * @task ratelimit */ public static function disconnectRateLimits(array $request_state) { $limits = self::$limits; // Remove all limits before disconnecting them so this works properly if // it runs twice. (We run this automatically as a shutdown handler.) self::$limits = array(); foreach ($limits as $limit) { $limit->didDisconnect($request_state); } } /** * Emit an HTTP 429 "Too Many Requests" response (indicating that the user * has exceeded application rate limits) and exit. * * @return exit This method **does not return**. * @task ratelimit */ private static function didRateLimit($reason) { header( 'Content-Type: text/plain; charset=utf-8', $replace = true, $http_error = 429); echo $reason; exit(1); } /* -( Startup Timers )----------------------------------------------------- */ /** * Record the beginning of a new startup phase. * * For phases which occur before @{class:PhabricatorStartup} loads, save the * time and record it with @{method:recordStartupPhase} after the class is * available. * * @param string Phase name. * @task phases */ public static function beginStartupPhase($phase) { self::recordStartupPhase($phase, microtime(true)); } /** * Record the start time of a previously executed startup phase. * * For startup phases which occur after @{class:PhabricatorStartup} loads, * use @{method:beginStartupPhase} instead. This method can be used to * record a time before the class loads, then hand it over once the class * becomes available. * * @param string Phase name. * @param float Phase start time, from `microtime(true)`. * @task phases */ public static function recordStartupPhase($phase, $time) { self::$phases[$phase] = $time; } /** * Get information about startup phase timings. * * Sometimes, performance problems can occur before we start the profiler. * Since the profiler can't examine these phases, it isn't useful in * understanding their performance costs. * * Instead, the startup process marks when it enters various phases using * @{method:beginStartupPhase}. A later call to this method can retrieve this * information, which can be examined to gain greater insight into where * time was spent. The output is still crude, but better than nothing. * * @task phases */ public static function getPhases() { return self::$phases; } } diff --git a/webroot/rsrc/externals/javelin/docs/concepts/behaviors.diviner b/webroot/rsrc/externals/javelin/docs/concepts/behaviors.diviner index 0ff27d912a..f3cea9cda6 100644 --- a/webroot/rsrc/externals/javelin/docs/concepts/behaviors.diviner +++ b/webroot/rsrc/externals/javelin/docs/concepts/behaviors.diviner @@ -1,181 +1,181 @@ @title Concepts: Behaviors @group concepts Javelin behaviors help you glue pieces of code together. = Overview = Javelin behaviors provide a place for you to put glue code. For instance, when a page loads, you often need to instantiate objects, or set up event listeners, or alert the user that they've won a hog, or create a dependency between two objects, or modify the DOM, etc. Sometimes there's enough code involved here or a particular setup step happens often enough that it makes sense to write a class, but sometimes it's just a few lines of one-off glue. Behaviors give you a structured place to put this glue so that it's consistently organized and can benefit from Javelin infrastructure. = Behavior Basics = Behaviors are defined with @{function:JX.behavior}: lang=js JX.behavior('win-a-hog', function(config, statics) { alert("YOU WON A HOG NAMED " + config.hogName + "!"); }); They are called with @{function:JX.initBehaviors}: lang=js JX.initBehaviors({ "win-a-hog" : [{hogName : "Ethel"}] }); Normally, you don't construct the @{function:JX.initBehaviors} call yourself, but instead use a server-side library which manages behavior initialization for you. For example, using the PHP library: lang=php $config = array('hogName' => 'Ethel'); JavelinHelper::initBehaviors('win-a-hog', $config); Regardless, this will alert the user that they've won a hog (named Ethel, which is a good name for a hog) when they load the page. The callback you pass to @{function:JX.behavior} should have this signature: lang=js function(config, statics) { // ... } The function will be invoked once for each configuration dictionary passed to @{function:JX.initBehaviors}, and the dictionary will be passed as the `config` parameter. For example, to alert the user that they've won two hogs: lang=js JX.initBehaviors({ "win-a-hog" : [{hogName : "Ethel"}, {hogName: "Beatrice"}] }); This will invoke the function twice, once for each `config` dictionary. Usually, you invoke a behavior multiple times if you have several similar controls on a page, like multiple @{class:JX.Tokenizer}s. An initially empty object will be passed in the `statics` parameter, but changes to this object will persist across invocations of the behavior. For example: lang=js JX.initBehaviors('win-a-hog', function(config, statics) { statics.hogsWon = (statics.hogsWon || 0) + 1; if (statics.hogsWon == 1) { alert("YOU WON A HOG! YOUR HOG IS NAMED " + config.hogName + "!"); } else { alert("YOU WON ANOTHER HOG!!! THIS ONE IS NAMED " + config.hogName + "!"); } } One way to think about behaviors are that they take the anonymous function passed to @{function:JX.behavior} and put it in a private Javelin namespace, which you access with @{function:JX.initBehavior}. Another way to think about them is that you are defining methods which represent the entirety of the API exposed by the document. The recommended approach to glue code is that the server interact with Javascript on the client //only// by invoking behaviors, so the set of available behaviors represent the complete set of legal interactions available to the server. = History and Rationale = This section explains why behaviors exist and how they came about. You can understand and use them without knowing any of this, but it may be useful or interesting. In early 2007, Facebook often solved the "glue code" problem through the use of global functions and DOM Level 0 event handlers, by manually building HTML tags in PHP: lang=php echo ''. 'Click here to win!'. ''; (This example produces a link which the user can click to be alerted they have won a hog, which is slightly different from the automatic alert in the other examples in this document. Some subtle distinctions are ignored or glossed over here because they are not important to understanding behaviors.) This has a wide array of technical and architectural problems: - Correctly escaping parameters is cumbersome and difficult. - It resists static analysis, and is difficult to even grep for. You can't easily package, minify, or determine dependencies for the piece of JS in the result string. - DOM Level 0 events have a host of issues in a complex application environment. - The JS global namespace becomes polluted with application glue functions. - The server and client are tightly and relatively arbitrarily coupled, since many of these handlers called multiple functions or had logic in the strings. There is no structure to the coupling, so many callers relied on the full power of arbitrary JS execution. - It's utterly hideous. -In 2007/2008, we introduced @{function@libphutil:jsprintf} and a function called +In 2007/2008, we introduced @{function@arcanist:jsprintf} and a function called onloadRegister() to solve some of the obvious problems: lang=php onloadRegister('win_a_hog(%s);', $hog_name); This registers the snippet for invocation after DOMContentReady fires. This API makes escaping manageable, and was combined with recommendations to structure code like this in order to address some of the other problems: lang=php $id = uniq_id(); echo 'Click here to win!'; onloadRegister('new WinAHogController(%s, %s);', $id, $hog_name); By 2010 (particularly with the introduction of XHP) the API had become more sophisticated, but this is basically how most of Facebook's glue code still works as of mid-2011. If you view the source of any page, you'll see a bunch of `onloadRegister()` calls in the markup which are generated like this. This mitigates many of the problems but is still fairly awkward. Escaping is easier, but still possible to get wrong. Stuff is a bit easier to grep for, but not much. You can't get very far with static analysis unless you get very complex. Coupling between the languages has been reduced but not eliminated. And now you have a bunch of classes which only really have glue code in them. Javelin behaviors provide a more structured solution to some of these problems: - All your Javascript code is in Javascript files, not embedded in strings in in some host language on the server side. - You can use static analysis and minification tools normally. - Provided you use a reasonable server-side library, you can't get escaping wrong. - Coupling is reduced because server only passes data to the client, never code. - The server declares client dependencies explicitly, not implicitly inside a string literal. Behaviors are also relatively easy to grep for. - Behaviors exist in a private, structured namespace instead of the global namespace. - Separation between the document's layout and behavior is a consequence of the structure of behaviors. - The entire interface the server may invoke against can be readily inferred. Note that Javelin does provide @{function:JX.onload}, which behaves like `onloadRegister()`. However, its use is discouraged. The two major downsides to the behavior design appear to be: - They have a higher setup cost than the ad-hoc methods, but Javelin philosophically places a very low value on this. - Because there's a further setup cost to migrate an existing behavior into a class, behaviors sometimes grow little by little until they are too big, have more than just glue code, and should have been refactored into a real class some time ago. This is a pretty high-level drawback and is manageable through awareness of the risk and code review. diff --git a/webroot/rsrc/externals/javelin/docs/facebook.diviner b/webroot/rsrc/externals/javelin/docs/facebook.diviner deleted file mode 100644 index 628ec5cfdb..0000000000 --- a/webroot/rsrc/externals/javelin/docs/facebook.diviner +++ /dev/null @@ -1,82 +0,0 @@ -@title Javelin at Facebook -@group facebook - -Information specific to Javelin at Facebook. - -= Building Support Scripts = - -Javelin now ships with the source to build several libfbjs-based binaries, which -serve to completely sever its dependencies on trunk: - - - `javelinsymbols`: used for lint - - `jsast`: used for documentation generation - - `jsxmin`: used to crush packages - -To build these, first build libfbjs: - - javelin/ $ cd externals/libfbjs - javelin/externals/libfbjs/ $ CXX=/usr/bin/g++ make - -Note that **you must specify CXX explicitly because the default CXX is broken**. - -Now you should be able to build the individual binaries: - - javelin/ $ cd support/javelinsymbols - javelin/support/javelinsymbols $ CXX=/usr/bin/g++ make - - javelin/ $ cd support/jsast - javelin/support/jsast $ CXX=/usr/bin/g++ make - - javelin/ $ cd support/jsxmin - javelin/support/jsxmin $ CXX=/usr/bin/g++ make - -= Synchronizing Javelin = - -To synchronize Javelin **from** Facebook trunk, run the synchronize script: - - javelin/ $ ./scripts/sync-from-facebook.php ~/www - -...where `~/www` is the root you want to pull Javelin files from. The script -will copy files out of `html/js/javelin` and build packages, and leave the -results in your working copy. From there you can review changes and commit, and -then push, diff, or send a pull request. - -To synchronize Javelin **to** Facebook trunk, run the, uh, reverse-synchronize -script: - - javelin/ $ ./scripts/sync-to-facebook.php ~/www - -...where `~/www` is the root you want to push Javelin files to. The script -will copy files out of the working copy into your `www` and leave you with a -dirty `www`. From there you can review changes. - -Once Facebook moves to pure git for `www` we can probably just submodule -Javelin into it and get rid of all this nonsense, but the mixed SVN/git -environment makes that difficult until then. - -= Building Documentation = - -Check out `diviner` and `libphutil` from Facebook github, and put them in a -directory with `javelin`: - - somewhere/ $ ls - diviner/ - javelin/ - libphutil/ - somewhere/ $ - -Now run `diviner` on `javelin`: - - somewhere/ $ cd javelin - somewhere/javelin/ $ ../diviner/bin/diviner . - [DivinerArticleEngine] Generating documentation for 48 files... - [JavelinDivinerEngine] Generating documentation for 74 files... - somewhere/javelin/ $ - -Documentation is now available in `javelin/docs/`. - -= Editing javelinjs.com = - -The source for javelinjs.com lives in `javelin/support/webroot/`. The site -itself is served off the phabricator.com host. You need access to that host to -push it.