diff --git a/resources/sql/autopatches/20180322.lock.01.identifier.sql b/resources/sql/autopatches/20180322.lock.01.identifier.sql new file mode 100644 index 0000000000..b115a691fa --- /dev/null +++ b/resources/sql/autopatches/20180322.lock.01.identifier.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_repository.repository_pushevent + ADD requestIdentifier VARBINARY(12); + +ALTER TABLE {$NAMESPACE}_repository.repository_pushevent + ADD UNIQUE KEY `key_request` (requestIdentifier); diff --git a/scripts/repository/commit_hook.php b/scripts/repository/commit_hook.php index 51abcb6c89..07d6d7cfa2 100755 --- a/scripts/repository/commit_hook.php +++ b/scripts/repository/commit_hook.php @@ -1,229 +1,234 @@ #!/usr/bin/env php 1) { $context = $argv[1]; $context = explode(':', $context, 2); $argv[1] = $context[0]; if (count($context) > 1) { $_ENV['PHABRICATOR_INSTANCE'] = $context[1]; putenv('PHABRICATOR_INSTANCE='.$context[1]); } } $root = dirname(dirname(dirname(__FILE__))); require_once $root.'/scripts/__init_script__.php'; if ($argc < 2) { throw new Exception(pht('usage: commit-hook ')); } $engine = new DiffusionCommitHookEngine(); $repository = id(new PhabricatorRepositoryQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withIdentifiers(array($argv[1])) ->needProjectPHIDs(true) ->executeOne(); if (!$repository) { throw new Exception(pht('No such repository "%s"!', $argv[1])); } if (!$repository->isHosted()) { // In Mercurial, the "pretxnchangegroup" hook fires for both pulls and // pushes. Normally we only install the hook for hosted repositories, but // if a hosted repository is later converted into an observed repository we // can end up with an observed repository that has the hook installed. // If we're running hooks from an observed repository, just exit without // taking action. For more discussion, see PHI24. return 0; } $engine->setRepository($repository); $args = new PhutilArgumentParser($argv); $args->parsePartial( array( array( 'name' => 'hook-mode', 'param' => 'mode', 'help' => pht('Hook execution mode.'), ), )); $argv = array_merge( array($argv[0]), $args->getUnconsumedArgumentVector()); // Figure out which user is writing the commit. $hook_mode = $args->getArg('hook-mode'); if ($hook_mode !== null) { $known_modes = array( 'svn-revprop' => true, ); if (empty($known_modes[$hook_mode])) { throw new Exception( pht( 'Invalid Hook Mode: This hook was invoked in "%s" mode, but this '. 'is not a recognized hook mode. Valid modes are: %s.', $hook_mode, implode(', ', array_keys($known_modes)))); } } $is_svnrevprop = ($hook_mode == 'svn-revprop'); if ($is_svnrevprop) { // For now, we let these through if the repository allows dangerous changes // and prevent them if it doesn't. See T11208 for discussion. $revprop_key = $argv[5]; if ($repository->shouldAllowDangerousChanges()) { $err = 0; } else { $err = 1; $console = PhutilConsole::getConsole(); $console->writeErr( pht( "DANGEROUS CHANGE: Dangerous change protection is enabled for this ". "repository, so you can not change revision properties (you are ". "attempting to edit \"%s\").\n". "Edit the repository configuration before making dangerous changes.", $revprop_key)); } exit($err); } else if ($repository->isGit() || $repository->isHg()) { $username = getenv(DiffusionCommitHookEngine::ENV_USER); if (!strlen($username)) { throw new Exception( pht( 'No Direct Pushes: You are pushing directly to a repository hosted '. 'by Phabricator. This will not work. See "No Direct Pushes" in the '. 'documentation for more information.')); } if ($repository->isHg()) { // We respond to several different hooks in Mercurial. $engine->setMercurialHook($argv[2]); } } else if ($repository->isSVN()) { // NOTE: In Subversion, the entire environment gets wiped so we can't read // DiffusionCommitHookEngine::ENV_USER. Instead, we've set "--tunnel-user" to // specify the correct user; read this user out of the commit log. if ($argc < 4) { throw new Exception(pht('usage: commit-hook ')); } $svn_repo = $argv[2]; $svn_txn = $argv[3]; list($username) = execx('svnlook author -t %s %s', $svn_txn, $svn_repo); $username = rtrim($username, "\n"); $engine->setSubversionTransactionInfo($svn_txn, $svn_repo); } else { throw new Exception(pht('Unknown repository type.')); } $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($username)) ->executeOne(); if (!$user) { throw new Exception(pht('No such user "%s"!', $username)); } $engine->setViewer($user); // Read stdin for the hook engine. if ($repository->isHg()) { // Mercurial leaves stdin open, so we can't just read it until EOF. $stdin = ''; } else { // Git and Subversion write data into stdin and then close it. Read the // data. $stdin = @file_get_contents('php://stdin'); if ($stdin === false) { throw new Exception(pht('Failed to read stdin!')); } } $engine->setStdin($stdin); $engine->setOriginalArgv(array_slice($argv, 2)); $remote_address = getenv(DiffusionCommitHookEngine::ENV_REMOTE_ADDRESS); if (strlen($remote_address)) { $engine->setRemoteAddress($remote_address); } $remote_protocol = getenv(DiffusionCommitHookEngine::ENV_REMOTE_PROTOCOL); if (strlen($remote_protocol)) { $engine->setRemoteProtocol($remote_protocol); } +$request_identifier = getenv(DiffusionCommitHookEngine::ENV_REQUEST); +if (strlen($request_identifier)) { + $engine->setRequestIdentifier($request_identifier); +} + try { $err = $engine->execute(); } catch (DiffusionCommitHookRejectException $ex) { $console = PhutilConsole::getConsole(); if (PhabricatorEnv::getEnvConfig('phabricator.serious-business')) { $preamble = pht('*** PUSH REJECTED BY COMMIT HOOK ***'); } else { $preamble = pht(<<writeErr("%s\n\n", $preamble); $console->writeErr("%s\n\n", $ex->getMessage()); $err = 1; } exit($err); diff --git a/scripts/ssh/ssh-exec.php b/scripts/ssh/ssh-exec.php index 2ff1bdc198..0f2275cda8 100755 --- a/scripts/ssh/ssh-exec.php +++ b/scripts/ssh/ssh-exec.php @@ -1,306 +1,313 @@ #!/usr/bin/env php setData( + array( + 'Q' => $request_identifier, + )); + $args = new PhutilArgumentParser($argv); $args->setTagline(pht('execute SSH requests')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'phabricator-ssh-user', 'param' => 'username', 'help' => pht( 'If the request authenticated with a user key, the name of the '. 'user.'), ), array( 'name' => 'phabricator-ssh-device', 'param' => 'name', 'help' => pht( 'If the request authenticated with a device key, the name of the '. 'device.'), ), array( 'name' => 'phabricator-ssh-key', 'param' => 'id', 'help' => pht( 'The ID of the SSH key which authenticated this request. This is '. 'used to allow logs to report when specific keys were used, to make '. 'it easier to manage credentials.'), ), array( 'name' => 'ssh-command', 'param' => 'command', 'help' => pht( 'Provide a command to execute. This makes testing this script '. 'easier. When running normally, the command is read from the '. 'environment (%s), which is populated by sshd.', 'SSH_ORIGINAL_COMMAND'), ), )); try { $remote_address = null; $ssh_client = getenv('SSH_CLIENT'); if ($ssh_client) { // This has the format " ". Grab the IP. $remote_address = head(explode(' ', $ssh_client)); $ssh_log->setData( array( 'r' => $remote_address, )); } $key_id = $args->getArg('phabricator-ssh-key'); if ($key_id) { $ssh_log->setData( array( 'k' => $key_id, )); } $user_name = $args->getArg('phabricator-ssh-user'); $device_name = $args->getArg('phabricator-ssh-device'); $user = null; $device = null; $is_cluster_request = false; if ($user_name && $device_name) { throw new Exception( pht( 'The %s and %s flags are mutually exclusive. You can not '. 'authenticate as both a user ("%s") and a device ("%s"). '. 'Specify one or the other, but not both.', '--phabricator-ssh-user', '--phabricator-ssh-device', $user_name, $device_name)); } else if (strlen($user_name)) { $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($user_name)) ->executeOne(); if (!$user) { throw new Exception( pht( 'Invalid username ("%s"). There is no user with this username.', $user_name)); } id(new PhabricatorAuthSessionEngine()) ->willServeRequestForUser($user); } else if (strlen($device_name)) { if (!$remote_address) { throw new Exception( pht( 'Unable to identify remote address from the %s environment '. 'variable. Device authentication is accepted only from trusted '. 'sources.', 'SSH_CLIENT')); } if (!PhabricatorEnv::isClusterAddress($remote_address)) { throw new Exception( pht( 'This request originates from outside of the Phabricator cluster '. 'address range. Requests signed with a trusted device key must '. 'originate from trusted hosts.')); } $device = id(new AlmanacDeviceQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withNames(array($device_name)) ->executeOne(); if (!$device) { throw new Exception( pht( 'Invalid device name ("%s"). There is no device with this name.', $device_name)); } // We're authenticated as a device, but we're going to read the user out of // the command below. $is_cluster_request = true; } else { throw new Exception( pht( 'This script must be invoked with either the %s or %s flag.', '--phabricator-ssh-user', '--phabricator-ssh-device')); } if ($args->getArg('ssh-command')) { $original_command = $args->getArg('ssh-command'); } else { $original_command = getenv('SSH_ORIGINAL_COMMAND'); } $original_argv = id(new PhutilShellLexer()) ->splitArguments($original_command); if ($device) { // If we're authenticating as a device, the first argument may be a // "@username" argument to act as a particular user. $first_argument = head($original_argv); if (preg_match('/^@/', $first_argument)) { $act_as_name = array_shift($original_argv); $act_as_name = substr($act_as_name, 1); $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($act_as_name)) ->executeOne(); if (!$user) { throw new Exception( pht( 'Device request identifies an acting user with an invalid '. 'username ("%s"). There is no user with this username.', $act_as_name)); } } else { $user = PhabricatorUser::getOmnipotentUser(); } } if ($user->isOmnipotent()) { $user_name = 'device/'.$device->getName(); } else { $user_name = $user->getUsername(); } $ssh_log->setData( array( 'u' => $user_name, 'P' => $user->getPHID(), )); if (!$device) { if (!$user->canEstablishSSHSessions()) { throw new Exception( pht( 'Your account ("%s") does not have permission to establish SSH '. 'sessions. Visit the web interface for more information.', $user_name)); } } $workflows = id(new PhutilClassMapQuery()) ->setAncestorClass('PhabricatorSSHWorkflow') ->setUniqueMethod('getName') ->execute(); if (!$original_argv) { throw new Exception( pht( "Welcome to Phabricator.\n\n". "You are logged in as %s.\n\n". "You haven't specified a command to run. This means you're requesting ". "an interactive shell, but Phabricator does not provide an ". "interactive shell over SSH.\n\n". "Usually, you should run a command like `%s` or `%s` ". "rather than connecting directly with SSH.\n\n". "Supported commands are: %s.", $user_name, 'git clone', 'hg push', implode(', ', array_keys($workflows)))); } $log_argv = implode(' ', $original_argv); $log_argv = id(new PhutilUTF8StringTruncator()) ->setMaximumCodepoints(128) ->truncateString($log_argv); $ssh_log->setData( array( 'C' => $original_argv[0], 'U' => $log_argv, )); $command = head($original_argv); $parseable_argv = $original_argv; array_unshift($parseable_argv, 'phabricator-ssh-exec'); $parsed_args = new PhutilArgumentParser($parseable_argv); if (empty($workflows[$command])) { throw new Exception(pht('Invalid command.')); } $workflow = $parsed_args->parseWorkflows($workflows); $workflow->setSSHUser($user); $workflow->setOriginalArguments($original_argv); $workflow->setIsClusterRequest($is_cluster_request); + $workflow->setRequestIdentifier($request_identifier); $sock_stdin = fopen('php://stdin', 'r'); if (!$sock_stdin) { throw new Exception(pht('Unable to open stdin.')); } $sock_stdout = fopen('php://stdout', 'w'); if (!$sock_stdout) { throw new Exception(pht('Unable to open stdout.')); } $sock_stderr = fopen('php://stderr', 'w'); if (!$sock_stderr) { throw new Exception(pht('Unable to open stderr.')); } $socket_channel = new PhutilSocketChannel( $sock_stdin, $sock_stdout); $error_channel = new PhutilSocketChannel(null, $sock_stderr); $metrics_channel = new PhutilMetricsChannel($socket_channel); $workflow->setIOChannel($metrics_channel); $workflow->setErrorChannel($error_channel); $rethrow = null; try { $err = $workflow->execute($parsed_args); $metrics_channel->flush(); $error_channel->flush(); } catch (Exception $ex) { $rethrow = $ex; } // Always write this if we got as far as building a metrics channel. $ssh_log->setData( array( 'i' => $metrics_channel->getBytesRead(), 'o' => $metrics_channel->getBytesWritten(), )); if ($rethrow) { throw $rethrow; } } catch (Exception $ex) { fwrite(STDERR, "phabricator-ssh-exec: ".$ex->getMessage()."\n"); $err = 1; } $ssh_log->setData( array( 'c' => $err, 'T' => (int)(1000000 * (microtime(true) - $ssh_start_time)), )); exit($err); diff --git a/src/applications/config/option/PhabricatorAccessLogConfigOptions.php b/src/applications/config/option/PhabricatorAccessLogConfigOptions.php index 9ae60825ea..2b9c4bbc75 100644 --- a/src/applications/config/option/PhabricatorAccessLogConfigOptions.php +++ b/src/applications/config/option/PhabricatorAccessLogConfigOptions.php @@ -1,136 +1,140 @@ pht('The controller or workflow which handled the request.'), 'c' => pht('The HTTP response code or process exit code.'), 'D' => pht('The request date.'), 'e' => pht('Epoch timestamp.'), 'h' => pht("The webserver's host name."), 'p' => pht('The PID of the server process.'), 'r' => pht('The remote IP.'), 'T' => pht('The request duration, in microseconds.'), 'U' => pht('The request path, or request target.'), 'm' => pht('For conduit, the Conduit method which was invoked.'), 'u' => pht('The logged-in username, if one is logged in.'), 'P' => pht('The logged-in user PHID, if one is logged in.'), 'i' => pht('Request input, in bytes.'), 'o' => pht('Request output, in bytes.'), 'I' => pht('Cluster instance name, if configured.'), ); $http_map = $common_map + array( 'R' => pht('The HTTP referrer.'), 'M' => pht('The HTTP method.'), ); $ssh_map = $common_map + array( 's' => pht('The system user.'), 'S' => pht('The system sudo user.'), 'k' => pht('ID of the SSH key used to authenticate the request.'), + + // TODO: This is a reasonable thing to support in the HTTP access + // log, too. + 'Q' => pht('A random, unique string which identifies the request.'), ); $http_desc = pht( 'Format for the HTTP access log. Use `%s` to set the path. '. 'Available variables are:', 'log.access.path'); $http_desc .= "\n\n"; $http_desc .= $this->renderMapHelp($http_map); $ssh_desc = pht( 'Format for the SSH access log. Use %s to set the path. '. 'Available variables are:', 'log.ssh.path'); $ssh_desc .= "\n\n"; $ssh_desc .= $this->renderMapHelp($ssh_map); return array( $this->newOption('log.access.path', 'string', null) ->setLocked(true) ->setSummary(pht('Access log location.')) ->setDescription( pht( "To enable the Phabricator access log, specify a path. The ". "Phabricator access than normal HTTP access logs (for instance, ". "it can show logged-in users, controllers, and other application ". "data).\n\n". "If not set, no log will be written.")) ->addExample( null, pht('Disable access log.')) ->addExample( '/var/log/phabricator/access.log', pht('Write access log here.')), $this->newOption( 'log.access.format', // NOTE: This is 'wild' intead of 'string' so "\t" and such can be // specified. 'wild', "[%D]\t%p\t%h\t%r\t%u\t%C\t%m\t%U\t%R\t%c\t%T") ->setLocked(true) ->setSummary(pht('Access log format.')) ->setDescription($http_desc), $this->newOption('log.ssh.path', 'string', null) ->setLocked(true) ->setSummary(pht('SSH log location.')) ->setDescription( pht( "To enable the Phabricator SSH log, specify a path. The ". "access log can provide more detailed information about SSH ". "access than a normal SSH log (for instance, it can show ". "logged-in users, commands, and other application data).\n\n". "If not set, no log will be written.")) ->addExample( null, pht('Disable SSH log.')) ->addExample( '/var/log/phabricator/ssh.log', pht('Write SSH log here.')), $this->newOption( 'log.ssh.format', 'wild', "[%D]\t%p\t%h\t%r\t%s\t%S\t%u\t%C\t%U\t%c\t%T\t%i\t%o") ->setLocked(true) ->setSummary(pht('SSH log format.')) ->setDescription($ssh_desc), ); } private function renderMapHelp(array $map) { $desc = ''; foreach ($map as $key => $kdesc) { $desc .= " - `%".$key."` ".$kdesc."\n"; } $desc .= "\n"; $desc .= pht( "If a variable isn't available (for example, %%m appears in the file ". "format but the request is not a Conduit request), it will be rendered ". "as '-'"); $desc .= "\n\n"; $desc .= pht( "Note that the default format is subject to change in the future, so ". "if you rely on the log's format, specify it explicitly."); return $desc; } } diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index a0769a51f0..cc4526dbdc 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -1,1323 +1,1343 @@ remoteProtocol = $remote_protocol; return $this; } public function getRemoteProtocol() { return $this->remoteProtocol; } public function setRemoteAddress($remote_address) { $this->remoteAddress = $remote_address; return $this; } public function getRemoteAddress() { return $this->remoteAddress; } + public function setRequestIdentifier($request_identifier) { + $this->requestIdentifier = $request_identifier; + return $this; + } + + public function getRequestIdentifier() { + return $this->requestIdentifier; + } + public function setSubversionTransactionInfo($transaction, $repository) { $this->subversionTransaction = $transaction; $this->subversionRepository = $repository; return $this; } public function setStdin($stdin) { $this->stdin = $stdin; return $this; } public function getStdin() { return $this->stdin; } public function setOriginalArgv(array $original_argv) { $this->originalArgv = $original_argv; return $this; } public function getOriginalArgv() { return $this->originalArgv; } public function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setMercurialHook($mercurial_hook) { $this->mercurialHook = $mercurial_hook; return $this; } public function getMercurialHook() { return $this->mercurialHook; } /* -( Hook Execution )----------------------------------------------------- */ public function execute() { $ref_updates = $this->findRefUpdates(); $all_updates = $ref_updates; $caught = null; try { try { $this->rejectDangerousChanges($ref_updates); } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting dangerous changes, flag everything that we've // seen as rejected so it's clear that none of it was accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_DANGEROUS; throw $ex; } $this->applyHeraldRefRules($ref_updates, $all_updates); $content_updates = $this->findContentUpdates($ref_updates); try { $this->rejectEnormousChanges($content_updates); } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting enormous changes, flag everything. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ENORMOUS; throw $ex; } $all_updates = array_merge($all_updates, $content_updates); $this->applyHeraldContentRules($content_updates, $all_updates); // Run custom scripts in `hook.d/` directories. $this->applyCustomHooks($all_updates); // If we make it this far, we're accepting these changes. Mark all the // logs as accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ACCEPT; } catch (Exception $ex) { // We'll throw this again in a minute, but we want to save all the logs // first. $caught = $ex; } // Save all the logs no matter what the outcome was. $event = $this->newPushEvent(); $event->setRejectCode($this->rejectCode); $event->setRejectDetails($this->rejectDetails); $event->openTransaction(); $event->save(); foreach ($all_updates as $update) { $update->setPushEventPHID($event->getPHID()); $update->save(); } $event->saveTransaction(); if ($caught) { throw $caught; } // If this went through cleanly, detect pushes which are actually imports // of an existing repository rather than an addition of new commits. If // this push is importing a bunch of stuff, set the importing flag on // the repository. It will be cleared once we fully process everything. if ($this->isInitialImport($all_updates)) { $repository = $this->getRepository(); $repository->markImporting(); } if ($this->emailPHIDs) { // If Herald rules triggered email to users, queue a worker to send the // mail. We do this out-of-process so that we block pushes as briefly // as possible. // (We do need to pull some commit info here because the commit objects // may not exist yet when this worker runs, which could be immediately.) PhabricatorWorker::scheduleTask( 'PhabricatorRepositoryPushMailWorker', array( 'eventPHID' => $event->getPHID(), 'emailPHIDs' => array_values($this->emailPHIDs), 'info' => $this->loadCommitInfoForWorker($all_updates), ), array( 'priority' => PhabricatorWorker::PRIORITY_ALERTS, )); } return 0; } private function findRefUpdates() { $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionRefUpdates(); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } private function rejectDangerousChanges(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $repository = $this->getRepository(); if ($repository->shouldAllowDangerousChanges()) { return; } $flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; foreach ($ref_updates as $ref_update) { if (!$ref_update->hasChangeFlags($flag_dangerous)) { // This is not a dangerous change. continue; } // We either have a branch deletion or a non fast-forward branch update. // Format a message and reject the push. $message = pht( "DANGEROUS CHANGE: %s\n". "Dangerous change protection is enabled for this repository.\n". "Edit the repository configuration before making dangerous changes.", $ref_update->getDangerousChangeDescription()); throw new DiffusionCommitHookRejectException($message); } } private function findContentUpdates(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionContentUpdates($ref_updates); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } /* -( Herald )------------------------------------------------------------- */ private function applyHeraldRefRules( array $ref_updates, array $all_updates) { $this->applyHeraldRules( $ref_updates, new HeraldPreCommitRefAdapter(), $all_updates); } private function applyHeraldContentRules( array $content_updates, array $all_updates) { $this->applyHeraldRules( $content_updates, new HeraldPreCommitContentAdapter(), $all_updates); } private function applyHeraldRules( array $updates, HeraldAdapter $adapter_template, array $all_updates) { if (!$updates) { return; } $viewer = $this->getViewer(); $adapter_template ->setHookEngine($this) ->setActingAsPHID($viewer->getPHID()); $engine = new HeraldEngine(); $rules = null; $blocking_effect = null; $blocked_update = null; $blocking_xscript = null; foreach ($updates as $update) { $adapter = id(clone $adapter_template) ->setPushLog($update); if ($rules === null) { $rules = $engine->loadRulesForAdapter($adapter); } $effects = $engine->applyRules($rules, $adapter); $engine->applyEffects($effects, $adapter, $rules); $xscript = $engine->getTranscript(); // Store any PHIDs we want to send email to for later. foreach ($adapter->getEmailPHIDs() as $email_phid) { $this->emailPHIDs[$email_phid] = $email_phid; } $block_action = DiffusionBlockHeraldAction::ACTIONCONST; if ($blocking_effect === null) { foreach ($effects as $effect) { if ($effect->getAction() == $block_action) { $blocking_effect = $effect; $blocked_update = $update; $blocking_xscript = $xscript; break; } } } } if ($blocking_effect) { $rule = $blocking_effect->getRule(); $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_HERALD; $this->rejectDetails = $rule->getPHID(); $message = $blocking_effect->getTarget(); if (!strlen($message)) { $message = pht('(None.)'); } $blocked_ref_name = coalesce( $blocked_update->getRefName(), $blocked_update->getRefNewShort()); $blocked_name = $blocked_update->getRefType().'/'.$blocked_ref_name; throw new DiffusionCommitHookRejectException( pht( "This push was rejected by Herald push rule %s.\n". " Change: %s\n". " Rule: %s\n". " Reason: %s\n". "Transcript: %s", $rule->getMonogram(), $blocked_name, $rule->getName(), $message, PhabricatorEnv::getProductionURI( '/herald/transcript/'.$blocking_xscript->getID().'/'))); } } public function loadViewerProjectPHIDsForHerald() { // This just caches the viewer's projects so we don't need to load them // over and over again when applying Herald rules. if ($this->heraldViewerProjects === null) { $this->heraldViewerProjects = id(new PhabricatorProjectQuery()) ->setViewer($this->getViewer()) ->withMemberPHIDs(array($this->getViewer()->getPHID())) ->execute(); } return mpull($this->heraldViewerProjects, 'getPHID'); } /* -( Git )---------------------------------------------------------------- */ private function findGitRefUpdates() { $ref_updates = array(); // First, parse stdin, which lists all the ref changes. The input looks // like this: // // $stdin = $this->getStdin(); $lines = phutil_split_lines($stdin, $retain_endings = false); foreach ($lines as $line) { $parts = explode(' ', $line, 3); if (count($parts) != 3) { throw new Exception(pht('Expected "old new ref", got "%s".', $line)); } $ref_old = $parts[0]; $ref_new = $parts[1]; $ref_raw = $parts[2]; if (preg_match('(^refs/heads/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; $ref_raw = substr($ref_raw, strlen('refs/heads/')); } else if (preg_match('(^refs/tags/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG; $ref_raw = substr($ref_raw, strlen('refs/tags/')); } else { throw new Exception( pht( "Unable to identify the reftype of '%s'. Rejecting push.", $ref_raw)); } $ref_update = $this->newPushLog() ->setRefType($ref_type) ->setRefName($ref_raw) ->setRefOld($ref_old) ->setRefNew($ref_new); $ref_updates[] = $ref_update; } $this->findGitMergeBases($ref_updates); $this->findGitChangeFlags($ref_updates); return $ref_updates; } private function findGitMergeBases(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $futures = array(); foreach ($ref_updates as $key => $ref_update) { // If the old hash is "00000...", the ref is being created (either a new // branch, or a new tag). If the new hash is "00000...", the ref is being // deleted. If both are nonempty, the ref is being updated. For updates, // we'll figure out the `merge-base` of the old and new objects here. This // lets us reject non-FF changes cheaply; later, we'll figure out exactly // which commits are new. $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); if (($ref_old === self::EMPTY_HASH) || ($ref_new === self::EMPTY_HASH)) { continue; } $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'merge-base %s %s', $ref_old, $ref_new); } $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { // If 'old' and 'new' have no common ancestors (for example, a force push // which completely rewrites a ref), `git merge-base` will exit with // an error and no output. It would be nice to find a positive test // for this instead, but I couldn't immediately come up with one. See // T4224. Assume this means there are no ancestors. list($err, $stdout) = $future->resolve(); if ($err) { $merge_base = null; } else { $merge_base = rtrim($stdout, "\n"); } $ref_update = $ref_updates[$key]; $ref_update->setMergeBase($merge_base); } return $ref_updates; } private function findGitChangeFlags(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); foreach ($ref_updates as $key => $ref_update) { $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); $ref_type = $ref_update->getRefType(); $ref_flags = 0; $dangerous = null; if (($ref_old === self::EMPTY_HASH) && ($ref_new === self::EMPTY_HASH)) { // This happens if you try to delete a tag or branch which does not // exist by pushing directly to the ref. Git will warn about it but // allow it. Just call it a delete, without flagging it as dangerous. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else if ($ref_old === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($ref_new === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push deletes the branch '%s'.", $ref_update->getRefName()); } } else { $merge_base = $ref_update->getMergeBase(); if ($merge_base == $ref_old) { // This is a fast-forward update to an existing branch. // These are safe. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; // For now, we don't consider deleting or moving tags to be a // "dangerous" update. It's way harder to get wrong and should be easy // to recover from once we have better logging. Only add the dangerous // flag if this ref is a branch. if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push updates the branch '%s' ". "from '%s' to '%s', but this is not a fast-forward. Pushes ". "which rewrite published branch history are dangerous.", $ref_update->getRefName(), $ref_update->getRefOldShort(), $ref_update->getRefNewShort()); } } } $ref_update->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } } return $ref_updates; } private function findGitContentUpdates(array $ref_updates) { $flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; $futures = array(); foreach ($ref_updates as $key => $ref_update) { if ($ref_update->hasChangeFlags($flag_delete)) { // Deleting a branch or tag can never create any new commits. continue; } // NOTE: This piece of magic finds all new commits, by walking backward // from the new value to the value of *any* existing ref in the // repository. Particularly, this will cover the cases of a new branch, a // completely moved tag, etc. $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'log --format=%s %s --not --all', '%H', $ref_update->getRefNew()); } $content_updates = array(); $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { list($stdout) = $future->resolvex(); if (!strlen(trim($stdout))) { // This change doesn't have any new commits. One common case of this // is creating a new tag which points at an existing commit. continue; } $commits = phutil_split_lines($stdout, $retain_newlines = false); // If we're looking at a branch, mark all of the new commits as on that // branch. It's only possible for these commits to be on updated branches, // since any other branch heads are necessarily behind them. $branch_name = null; $ref_update = $ref_updates[$key]; $type_branch = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; if ($ref_update->getRefType() == $type_branch) { $branch_name = $ref_update->getRefName(); } foreach ($commits as $commit) { if ($branch_name) { $this->gitCommits[$commit][] = $branch_name; } $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } } return $content_updates; } /* -( Custom )------------------------------------------------------------- */ private function applyCustomHooks(array $updates) { $args = $this->getOriginalArgv(); $stdin = $this->getStdin(); $console = PhutilConsole::getConsole(); $env = array( self::ENV_REPOSITORY => $this->getRepository()->getPHID(), self::ENV_USER => $this->getViewer()->getUsername(), + self::ENV_REQUEST => $this->getRequestIdentifier(), self::ENV_REMOTE_PROTOCOL => $this->getRemoteProtocol(), self::ENV_REMOTE_ADDRESS => $this->getRemoteAddress(), ); $repository = $this->getRepository(); $env += $repository->getPassthroughEnvironmentalVariables(); $directories = $repository->getHookDirectories(); foreach ($directories as $directory) { $hooks = $this->getExecutablesInDirectory($directory); sort($hooks); foreach ($hooks as $hook) { // NOTE: We're explicitly running the hooks in sequential order to // make this more predictable. $future = id(new ExecFuture('%s %Ls', $hook, $args)) ->setEnv($env, $wipe_process_env = false) ->write($stdin); list($err, $stdout, $stderr) = $future->resolve(); if (!$err) { // This hook ran OK, but echo its output in case there was something // informative. $console->writeOut('%s', $stdout); $console->writeErr('%s', $stderr); continue; } $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_EXTERNAL; $this->rejectDetails = basename($hook); throw new DiffusionCommitHookRejectException( pht( "This push was rejected by custom hook script '%s':\n\n%s%s", basename($hook), $stdout, $stderr)); } } } private function getExecutablesInDirectory($directory) { $executables = array(); if (!Filesystem::pathExists($directory)) { return $executables; } foreach (Filesystem::listDirectory($directory) as $path) { $full_path = $directory.DIRECTORY_SEPARATOR.$path; if (!is_executable($full_path)) { // Don't include non-executable files. continue; } if (basename($full_path) == 'README') { // Don't include README, even if it is marked as executable. It almost // certainly got caught in the crossfire of a sweeping `chmod`, since // users do this with some frequency. continue; } $executables[] = $full_path; } return $executables; } /* -( Mercurial )---------------------------------------------------------- */ private function findMercurialRefUpdates() { $hook = $this->getMercurialHook(); switch ($hook) { case 'pretxnchangegroup': return $this->findMercurialChangegroupRefUpdates(); case 'prepushkey': return $this->findMercurialPushKeyRefUpdates(); default: throw new Exception(pht('Unrecognized hook "%s"!', $hook)); } } private function findMercurialChangegroupRefUpdates() { $hg_node = getenv('HG_NODE'); if (!$hg_node) { throw new Exception( pht( 'Expected %s in environment!', 'HG_NODE')); } // NOTE: We need to make sure this is passed to subprocesses, or they won't // be able to see new commits. Mercurial uses this as a marker to determine // whether the pending changes are visible or not. $_ENV['HG_PENDING'] = getenv('HG_PENDING'); $repository = $this->getRepository(); $futures = array(); foreach (array('old', 'new') as $key) { $futures[$key] = $repository->getLocalCommandFuture( 'heads --template %s', '{node}\1{branch}\2'); } // Wipe HG_PENDING out of the old environment so we see the pre-commit // state of the repository. $futures['old']->updateEnv('HG_PENDING', null); $futures['commits'] = $repository->getLocalCommandFuture( 'log --rev %s --template %s', hgsprintf('%s:%s', $hg_node, 'tip'), '{node}\1{branch}\2'); // Resolve all of the futures now. We don't need the 'commits' future yet, // but it simplifies the logic to just get it out of the way. foreach (new FutureIterator($futures) as $future) { $future->resolve(); } list($commit_raw) = $futures['commits']->resolvex(); $commit_map = $this->parseMercurialCommits($commit_raw); $this->mercurialCommits = $commit_map; // NOTE: `hg heads` exits with an error code and no output if the repository // has no heads. Most commonly this happens on a new repository. We know // we can run `hg` successfully since the `hg log` above didn't error, so // just ignore the error code. list($err, $old_raw) = $futures['old']->resolve(); $old_refs = $this->parseMercurialHeads($old_raw); list($err, $new_raw) = $futures['new']->resolve(); $new_refs = $this->parseMercurialHeads($new_raw); $all_refs = array_keys($old_refs + $new_refs); $ref_updates = array(); foreach ($all_refs as $ref) { $old_heads = idx($old_refs, $ref, array()); $new_heads = idx($new_refs, $ref, array()); sort($old_heads); sort($new_heads); if (!$old_heads && !$new_heads) { // This should never be possible, as it makes no sense. Explode. throw new Exception( pht( 'Mercurial repository has no new or old heads for branch "%s" '. 'after push. This makes no sense; rejecting change.', $ref)); } if ($old_heads === $new_heads) { // No changes to this branch, so skip it. continue; } $stray_heads = array(); $head_map = array(); if ($old_heads && !$new_heads) { // This is a branch deletion with "--close-branch". foreach ($old_heads as $old_head) { $head_map[$old_head] = array(self::EMPTY_HASH); } } else if (count($old_heads) > 1) { // HORRIBLE: In Mercurial, branches can have multiple heads. If the // old branch had multiple heads, we need to figure out which new // heads descend from which old heads, so we can tell whether you're // actively creating new heads (dangerous) or just working in a // repository that's already full of garbage (strongly discouraged but // not as inherently dangerous). These cases should be very uncommon. // NOTE: We're only looking for heads on the same branch. The old // tip of the branch may be the branchpoint for other branches, but that // is OK. $dfutures = array(); foreach ($old_heads as $old_head) { $dfutures[$old_head] = $repository->getLocalCommandFuture( 'log --branch %s --rev %s --template %s', $ref, hgsprintf('(descendants(%s) and head())', $old_head), '{node}\1'); } foreach (new FutureIterator($dfutures) as $future_head => $dfuture) { list($stdout) = $dfuture->resolvex(); $descendant_heads = array_filter(explode("\1", $stdout)); if ($descendant_heads) { // This old head has at least one descendant in the push. $head_map[$future_head] = $descendant_heads; } else { // This old head has no descendants, so it is being deleted. $head_map[$future_head] = array(self::EMPTY_HASH); } } // Now, find all the new stray heads this push creates, if any. These // are new heads which do not descend from the old heads. $seen = array_fuse(array_mergev($head_map)); foreach ($new_heads as $new_head) { if ($new_head === self::EMPTY_HASH) { // If a branch head is being deleted, don't insert it as an add. continue; } if (empty($seen[$new_head])) { $head_map[self::EMPTY_HASH][] = $new_head; } } } else if ($old_heads) { $head_map[head($old_heads)] = $new_heads; } else { $head_map[self::EMPTY_HASH] = $new_heads; } foreach ($head_map as $old_head => $child_heads) { foreach ($child_heads as $new_head) { if ($new_head === $old_head) { continue; } $ref_flags = 0; $dangerous = null; if ($old_head == self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } $deletes_existing_head = ($new_head == self::EMPTY_HASH); $splits_existing_head = (count($child_heads) > 1); $creates_duplicate_head = ($old_head == self::EMPTY_HASH) && (count($head_map) > 1); if ($splits_existing_head || $creates_duplicate_head) { $readable_child_heads = array(); foreach ($child_heads as $child_head) { $readable_child_heads[] = substr($child_head, 0, 12); } $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; if ($splits_existing_head) { // We're splitting an existing head into two or more heads. // This is dangerous, and a super bad idea. Note that we're only // raising this if you're actively splitting a branch head. If a // head split in the past, we don't consider appends to it // to be dangerous. $dangerous = pht( "The change you're attempting to push splits the head of ". "branch '%s' into multiple heads: %s. This is inadvisable ". "and dangerous.", $ref, implode(', ', $readable_child_heads)); } else { // We're adding a second (or more) head to a branch. The new // head is not a descendant of any old head. $dangerous = pht( "The change you're attempting to push creates new, divergent ". "heads for the branch '%s': %s. This is inadvisable and ". "dangerous.", $ref, implode(', ', $readable_child_heads)); } } if ($deletes_existing_head) { // TODO: Somewhere in here we should be setting CHANGEFLAG_REWRITE // if we are also creating at least one other head to replace // this one. // NOTE: In Git, this is a dangerous change, but it is not dangerous // in Mercurial. Mercurial branches are version controlled, and // Mercurial does not prompt you for any special flags when pushing // a `--close-branch` commit by default. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH) ->setRefName($ref) ->setRefOld($old_head) ->setRefNew($new_head) ->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } $ref_updates[] = $ref_update; } } } return $ref_updates; } private function findMercurialPushKeyRefUpdates() { $key_namespace = getenv('HG_NAMESPACE'); if ($key_namespace === 'phases') { // Mercurial changes commit phases as part of normal push operations. We // just ignore these, as they don't seem to represent anything // interesting. return array(); } $key_name = getenv('HG_KEY'); $key_old = getenv('HG_OLD'); if (!strlen($key_old)) { $key_old = null; } $key_new = getenv('HG_NEW'); if (!strlen($key_new)) { $key_new = null; } if ($key_namespace !== 'bookmarks') { throw new Exception( pht( "Unknown Mercurial key namespace '%s', with key '%s' (%s -> %s). ". "Rejecting push.", $key_namespace, $key_name, coalesce($key_old, pht('null')), coalesce($key_new, pht('null')))); } if ($key_old === $key_new) { // We get a callback when the bookmark doesn't change. Just ignore this, // as it's a no-op. return array(); } $ref_flags = 0; $merge_base = null; if ($key_old === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($key_new === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else { list($merge_base_raw) = $this->getRepository()->execxLocalCommand( 'log --template %s --rev %s', '{node}', hgsprintf('ancestor(%s, %s)', $key_old, $key_new)); if (strlen(trim($merge_base_raw))) { $merge_base = trim($merge_base_raw); } if ($merge_base && ($merge_base === $key_old)) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; } } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BOOKMARK) ->setRefName($key_name) ->setRefOld(coalesce($key_old, self::EMPTY_HASH)) ->setRefNew(coalesce($key_new, self::EMPTY_HASH)) ->setChangeFlags($ref_flags); return array($ref_update); } private function findMercurialContentUpdates(array $ref_updates) { $content_updates = array(); foreach ($this->mercurialCommits as $commit => $branches) { $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } return $content_updates; } private function parseMercurialCommits($raw) { $commits_lines = explode("\2", $raw); $commits_lines = array_filter($commits_lines); $commit_map = array(); foreach ($commits_lines as $commit_line) { list($node, $branch) = explode("\1", $commit_line); $commit_map[$node] = array($branch); } return $commit_map; } private function parseMercurialHeads($raw) { $heads_map = $this->parseMercurialCommits($raw); $heads = array(); foreach ($heads_map as $commit => $branches) { foreach ($branches as $branch) { $heads[$branch][] = $commit; } } return $heads; } /* -( Subversion )--------------------------------------------------------- */ private function findSubversionRefUpdates() { // Subversion doesn't have any kind of mutable ref metadata. return array(); } private function findSubversionContentUpdates(array $ref_updates) { list($youngest) = execx( 'svnlook youngest %s', $this->subversionRepository); $ref_new = (int)$youngest + 1; $ref_flags = 0; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; $ref_content = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($ref_new) ->setChangeFlags($ref_flags); return array($ref_content); } /* -( Internals )---------------------------------------------------------- */ private function newPushLog() { // NOTE: We generate PHIDs up front so the Herald transcripts can pick them // up. $phid = id(new PhabricatorRepositoryPushLog())->generatePHID(); $device = AlmanacKeys::getLiveDevice(); if ($device) { $device_phid = $device->getPHID(); } else { $device_phid = null; } return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer()) ->setPHID($phid) ->setDevicePHID($device_phid) ->setRepositoryPHID($this->getRepository()->getPHID()) ->attachRepository($this->getRepository()) - ->setEpoch(time()); + ->setEpoch(PhabricatorTime::getNow()); } private function newPushEvent() { $viewer = $this->getViewer(); - return PhabricatorRepositoryPushEvent::initializeNewEvent($viewer) + + $event = PhabricatorRepositoryPushEvent::initializeNewEvent($viewer) ->setRepositoryPHID($this->getRepository()->getPHID()) ->setRemoteAddress($this->getRemoteAddress()) ->setRemoteProtocol($this->getRemoteProtocol()) - ->setEpoch(time()); + ->setEpoch(PhabricatorTime::getNow()); + + $identifier = $this->getRequestIdentifier(); + if (strlen($identifier)) { + $event->setRequestIdentifier($identifier); + } + + return $event; } private function rejectEnormousChanges(array $content_updates) { $repository = $this->getRepository(); if ($repository->shouldAllowEnormousChanges()) { return; } foreach ($content_updates as $update) { $identifier = $update->getRefNew(); try { $changesets = $this->loadChangesetsForCommit($identifier); $this->changesets[$identifier] = $changesets; } catch (Exception $ex) { $this->changesets[$identifier] = $ex; $message = pht( 'ENORMOUS CHANGE'. "\n". 'Enormous change protection is enabled for this repository, but '. 'you are pushing an enormous change ("%s"). Edit the repository '. 'configuration before making enormous changes.'. "\n\n". "Content Exception: %s", $identifier, $ex->getMessage()); throw new DiffusionCommitHookRejectException($message); } } } private function loadChangesetsForCommit($identifier) { $byte_limit = HeraldCommitAdapter::getEnormousByteLimit(); $time_limit = HeraldCommitAdapter::getEnormousTimeLimit(); $vcs = $this->getRepository()->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // For git and hg, we can use normal commands. $drequest = DiffusionRequest::newFromDictionary( array( 'repository' => $this->getRepository(), 'user' => $this->getViewer(), 'commit' => $identifier, )); $raw_diff = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest) ->setTimeout($time_limit) ->setByteLimit($byte_limit) ->setLinesOfContext(0) ->executeInline(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // TODO: This diff has 3 lines of context, which produces slightly // incorrect "added file content" and "removed file content" results. // This may also choke on binaries, but "svnlook diff" does not support // the "--diff-cmd" flag. // For subversion, we need to use `svnlook`. $future = new ExecFuture( 'svnlook diff -t %s %s', $this->subversionTransaction, $this->subversionRepository); $future->setTimeout($time_limit); $future->setStdoutSizeLimit($byte_limit); $future->setStderrSizeLimit($byte_limit); list($raw_diff) = $future->resolvex(); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } if (strlen($raw_diff) >= $byte_limit) { throw new Exception( pht( 'The raw text of this change ("%s") is enormous (larger than %s '. 'bytes).', $identifier, new PhutilNumber($byte_limit))); } if (!strlen($raw_diff)) { // If the commit is actually empty, just return no changesets. return array(); } $parser = new ArcanistDiffParser(); $changes = $parser->parseDiff($raw_diff); $diff = DifferentialDiff::newEphemeralFromRawChanges( $changes); return $diff->getChangesets(); } public function getChangesetsForCommit($identifier) { if (isset($this->changesets[$identifier])) { $cached = $this->changesets[$identifier]; if ($cached instanceof Exception) { throw $cached; } return $cached; } return $this->loadChangesetsForCommit($identifier); } public function loadCommitRefForCommit($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return id(new DiffusionLowLevelCommitQuery()) ->setRepository($repository) ->withIdentifier($identifier) ->execute(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // For subversion, we need to use `svnlook`. list($message) = execx( 'svnlook log -t %s %s', $this->subversionTransaction, $this->subversionRepository); return id(new DiffusionCommitRef()) ->setMessage($message); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } } public function loadBranches($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return idx($this->gitCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // NOTE: This will be "the branch the commit was made to", not // "a list of all branch heads which descend from the commit". // This is consistent with Mercurial, but possibly confusing. return idx($this->mercurialCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // Subversion doesn't have branches. return array(); } } private function loadCommitInfoForWorker(array $all_updates) { $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; $map = array(); foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $map[$update->getRefNew()] = array(); } foreach ($map as $identifier => $info) { $ref = $this->loadCommitRefForCommit($identifier); $map[$identifier] += array( 'summary' => $ref->getSummary(), 'branches' => $this->loadBranches($identifier), ); } return $map; } private function isInitialImport(array $all_updates) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // There is no meaningful way to import history into Subversion by // pushing. return false; default: break; } // Now, apply a heuristic to guess whether this is a normal commit or // an initial import. We guess something is an initial import if: // // - the repository is currently empty; and // - it pushes more than 7 commits at once. // // The number "7" is chosen arbitrarily as seeming reasonable. We could // also look at author data (do the commits come from multiple different // authors?) and commit date data (is the oldest commit more than 48 hours // old), but we don't have immediate access to those and this simple // heuristic might be good enough. $commit_count = 0; $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $commit_count++; } if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) { // If this pushes a very small number of commits, assume it's an // initial commit or stack of a few initial commits. return false; } $any_commits = id(new DiffusionCommitQuery()) ->setViewer($this->getViewer()) ->withRepository($repository) ->setLimit(1) ->execute(); if ($any_commits) { // If the repository already has commits, this isn't an import. return false; } return true; } } diff --git a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php index e40d8e1f51..baf1749252 100644 --- a/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionSSHWorkflow.php @@ -1,280 +1,285 @@ repository) { throw new Exception(pht('Repository is not available yet!')); } return $this->repository; } private function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getArgs() { return $this->args; } public function getEnvironment() { $env = array( DiffusionCommitHookEngine::ENV_USER => $this->getSSHUser()->getUsername(), DiffusionCommitHookEngine::ENV_REMOTE_PROTOCOL => 'ssh', ); + $identifier = $this->getRequestIdentifier(); + if ($identifier !== null) { + $env[DiffusionCommitHookEngine::ENV_REQUEST] = $identifier; + } + $remote_address = $this->getSSHRemoteAddress(); if ($remote_address !== null) { $env[DiffusionCommitHookEngine::ENV_REMOTE_ADDRESS] = $remote_address; } return $env; } /** * Identify and load the affected repository. */ abstract protected function identifyRepository(); abstract protected function executeRepositoryOperations(); abstract protected function raiseWrongVCSException( PhabricatorRepository $repository); protected function getBaseRequestPath() { return $this->baseRequestPath; } protected function writeError($message) { $this->getErrorChannel()->write($message); return $this; } protected function getCurrentDeviceName() { $device = AlmanacKeys::getLiveDevice(); if ($device) { return $device->getName(); } return php_uname('n'); } protected function getTargetDeviceName() { // TODO: This should use the correct device identity. $uri = new PhutilURI($this->proxyURI); return $uri->getDomain(); } protected function shouldProxy() { return (bool)$this->proxyURI; } protected function getProxyCommand() { $uri = new PhutilURI($this->proxyURI); $username = AlmanacKeys::getClusterSSHUser(); if ($username === null) { throw new Exception( pht( 'Unable to determine the username to connect with when trying '. 'to proxy an SSH request within the Phabricator cluster.')); } $port = $uri->getPort(); $host = $uri->getDomain(); $key_path = AlmanacKeys::getKeyPath('device.key'); if (!Filesystem::pathExists($key_path)) { throw new Exception( pht( 'Unable to proxy this SSH request within the cluster: this device '. 'is not registered and has a missing device key (expected to '. 'find key at "%s").', $key_path)); } $options = array(); $options[] = '-o'; $options[] = 'StrictHostKeyChecking=no'; $options[] = '-o'; $options[] = 'UserKnownHostsFile=/dev/null'; // This is suppressing "added
to the list of known hosts" // messages, which are confusing and irrelevant when they arise from // proxied requests. It might also be suppressing lots of useful errors, // of course. Ideally, we would enforce host keys eventually. $options[] = '-o'; $options[] = 'LogLevel=quiet'; // NOTE: We prefix the command with "@username", which the far end of the // connection will parse in order to act as the specified user. This // behavior is only available to cluster requests signed by a trusted // device key. return csprintf( 'ssh %Ls -l %s -i %s -p %s %s -- %s %Ls', $options, $username, $key_path, $port, $host, '@'.$this->getSSHUser()->getUsername(), $this->getOriginalArguments()); } final public function execute(PhutilArgumentParser $args) { $this->args = $args; $viewer = $this->getSSHUser(); $have_diffusion = PhabricatorApplication::isClassInstalledForViewer( 'PhabricatorDiffusionApplication', $viewer); if (!$have_diffusion) { throw new Exception( pht( 'You do not have permission to access the Diffusion application, '. 'so you can not interact with repositories over SSH.')); } $repository = $this->identifyRepository(); $this->setRepository($repository); $is_cluster_request = $this->getIsClusterRequest(); $uri = $repository->getAlmanacServiceURI( $viewer, $is_cluster_request, array( 'ssh', )); if ($uri) { $this->proxyURI = $uri; } try { return $this->executeRepositoryOperations(); } catch (Exception $ex) { $this->writeError(get_class($ex).': '.$ex->getMessage()); return 1; } } protected function loadRepositoryWithPath($path, $vcs) { $viewer = $this->getSSHUser(); $info = PhabricatorRepository::parseRepositoryServicePath($path, $vcs); if ($info === null) { throw new Exception( pht( 'Unrecognized repository path "%s". Expected a path like "%s", '. '"%s", or "%s".', $path, '/diffusion/X/', '/diffusion/123/', '/source/thaumaturgy.git')); } $identifier = $info['identifier']; $base = $info['base']; $this->baseRequestPath = $base; $repository = id(new PhabricatorRepositoryQuery()) ->setViewer($viewer) ->withIdentifiers(array($identifier)) ->needURIs(true) ->executeOne(); if (!$repository) { throw new Exception( pht('No repository "%s" exists!', $identifier)); } $protocol = PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH; if (!$repository->canServeProtocol($protocol, false)) { throw new Exception( pht( 'This repository ("%s") is not available over SSH.', $repository->getDisplayName())); } if ($repository->getVersionControlSystem() != $vcs) { $this->raiseWrongVCSException($repository); } return $repository; } protected function requireWriteAccess($protocol_command = null) { if ($this->hasWriteAccess === true) { return; } $repository = $this->getRepository(); $viewer = $this->getSSHUser(); if ($viewer->isOmnipotent()) { throw new Exception( pht( 'This request is authenticated as a cluster device, but is '. 'performing a write. Writes must be performed with a real '. 'user account.')); } $protocol = PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH; if ($repository->canServeProtocol($protocol, true)) { $can_push = PhabricatorPolicyFilter::hasCapability( $viewer, $repository, DiffusionPushCapability::CAPABILITY); if (!$can_push) { throw new Exception( pht('You do not have permission to push to this repository.')); } } else { if ($protocol_command !== null) { throw new Exception( pht( 'This repository is read-only over SSH (tried to execute '. 'protocol command "%s").', $protocol_command)); } else { throw new Exception( pht('This repository is read-only over SSH.')); } } $this->hasWriteAccess = true; return $this->hasWriteAccess; } protected function shouldSkipReadSynchronization() { $viewer = $this->getSSHUser(); // Currently, the only case where devices interact over SSH without // assuming user credentials is when synchronizing before a read. These // synchronizing reads do not themselves need to be synchronized. if ($viewer->isOmnipotent()) { return true; } return false; } protected function newPullEvent() { $viewer = $this->getSSHUser(); $repository = $this->getRepository(); $remote_address = $this->getSSHRemoteAddress(); return id(new PhabricatorRepositoryPullEvent()) ->setEpoch(PhabricatorTime::getNow()) ->setRemoteAddress($remote_address) ->setRemoteProtocol(PhabricatorRepositoryPullEvent::PROTOCOL_SSH) ->setPullerPHID($viewer->getPHID()) ->setRepositoryPHID($repository->getPHID()); } } diff --git a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php index 8ad76987f9..5bf2c84aeb 100644 --- a/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php +++ b/src/applications/repository/query/PhabricatorRepositoryPushLogSearchEngine.php @@ -1,239 +1,243 @@ newQuery(); if ($map['repositoryPHIDs']) { $query->withRepositoryPHIDs($map['repositoryPHIDs']); } if ($map['pusherPHIDs']) { $query->withPusherPHIDs($map['pusherPHIDs']); } if ($map['createdStart'] || $map['createdEnd']) { $query->withEpochBetween( $map['createdStart'], $map['createdEnd']); } return $query; } protected function buildCustomSearchFields() { return array( id(new PhabricatorSearchDatasourceField()) ->setDatasource(new DiffusionRepositoryDatasource()) ->setKey('repositoryPHIDs') ->setAliases(array('repository', 'repositories', 'repositoryPHID')) ->setLabel(pht('Repositories')) ->setDescription( pht('Search for pull logs for specific repositories.')), id(new PhabricatorUsersSearchField()) ->setKey('pusherPHIDs') ->setAliases(array('pusher', 'pushers', 'pusherPHID')) ->setLabel(pht('Pushers')) ->setDescription( pht('Search for pull logs by specific users.')), id(new PhabricatorSearchDateField()) ->setLabel(pht('Created After')) ->setKey('createdStart'), id(new PhabricatorSearchDateField()) ->setLabel(pht('Created Before')) ->setKey('createdEnd'), ); } protected function getURI($path) { return '/diffusion/pushlog/'.$path; } protected function getBuiltinQueryNames() { return array( 'all' => pht('All Push Logs'), ); } public function buildSavedQueryFromBuiltin($query_key) { $query = $this->newSavedQuery(); $query->setQueryKey($query_key); switch ($query_key) { case 'all': return $query; } return parent::buildSavedQueryFromBuiltin($query_key); } protected function renderResultList( array $logs, PhabricatorSavedQuery $query, array $handles) { $table = id(new DiffusionPushLogListView()) ->setViewer($this->requireViewer()) ->setLogs($logs); return id(new PhabricatorApplicationSearchResultView()) ->setTable($table); } protected function newExportFields() { $viewer = $this->requireViewer(); $fields = array( $fields[] = id(new PhabricatorIDExportField()) ->setKey('pushID') ->setLabel(pht('Push ID')), + $fields[] = id(new PhabricatorStringExportField()) + ->setKey('unique') + ->setLabel(pht('Unique')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('protocol') ->setLabel(pht('Protocol')), $fields[] = id(new PhabricatorPHIDExportField()) ->setKey('repositoryPHID') ->setLabel(pht('Repository PHID')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('repository') ->setLabel(pht('Repository')), $fields[] = id(new PhabricatorPHIDExportField()) ->setKey('pusherPHID') ->setLabel(pht('Pusher PHID')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('pusher') ->setLabel(pht('Pusher')), $fields[] = id(new PhabricatorPHIDExportField()) ->setKey('devicePHID') ->setLabel(pht('Device PHID')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('device') ->setLabel(pht('Device')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('type') ->setLabel(pht('Ref Type')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('name') ->setLabel(pht('Ref Name')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('old') ->setLabel(pht('Ref Old')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('new') ->setLabel(pht('Ref New')), $fields[] = id(new PhabricatorIntExportField()) ->setKey('flags') ->setLabel(pht('Flags')), $fields[] = id(new PhabricatorStringListExportField()) ->setKey('flagNames') ->setLabel(pht('Flag Names')), $fields[] = id(new PhabricatorIntExportField()) ->setKey('result') ->setLabel(pht('Result')), $fields[] = id(new PhabricatorStringExportField()) ->setKey('resultName') ->setLabel(pht('Result Name')), ); if ($viewer->getIsAdmin()) { $fields[] = id(new PhabricatorStringExportField()) ->setKey('remoteAddress') ->setLabel(pht('Remote Address')); } return $fields; } protected function newExportData(array $logs) { $viewer = $this->requireViewer(); $phids = array(); foreach ($logs as $log) { $phids[] = $log->getPusherPHID(); $phids[] = $log->getDevicePHID(); $phids[] = $log->getPushEvent()->getRepositoryPHID(); } $handles = $viewer->loadHandles($phids); $flag_map = PhabricatorRepositoryPushLog::getFlagDisplayNames(); $reject_map = PhabricatorRepositoryPushLog::getRejectCodeDisplayNames(); $export = array(); foreach ($logs as $log) { $event = $log->getPushEvent(); $repository_phid = $event->getRepositoryPHID(); if ($repository_phid) { $repository_name = $handles[$repository_phid]->getName(); } else { $repository_name = null; } $pusher_phid = $log->getPusherPHID(); if ($pusher_phid) { $pusher_name = $handles[$pusher_phid]->getName(); } else { $pusher_name = null; } $device_phid = $log->getDevicePHID(); if ($device_phid) { $device_name = $handles[$device_phid]->getName(); } else { $device_name = null; } $flags = $log->getChangeFlags(); $flag_names = array(); foreach ($flag_map as $flag_key => $flag_name) { if (($flags & $flag_key) === $flag_key) { $flag_names[] = $flag_name; } } $result = $event->getRejectCode(); $result_name = idx($reject_map, $result, pht('Unknown ("%s")', $result)); $map = array( 'pushID' => $event->getID(), + 'unique' => $event->getRequestIdentifier(), 'protocol' => $event->getRemoteProtocol(), 'repositoryPHID' => $repository_phid, 'repository' => $repository_name, 'pusherPHID' => $pusher_phid, 'pusher' => $pusher_name, 'devicePHID' => $device_phid, 'device' => $device_name, 'type' => $log->getRefType(), 'name' => $log->getRefName(), 'old' => $log->getRefOld(), 'new' => $log->getRefNew(), 'flags' => $flags, 'flagNames' => $flag_names, 'result' => $result, 'resultName' => $result_name, ); if ($viewer->getIsAdmin()) { $map['remoteAddress'] = $event->getRemoteAddress(); } $export[] = $map; } return $export; } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php b/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php index 2bc751ffca..369af15635 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php @@ -1,92 +1,98 @@ setPusherPHID($viewer->getPHID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_TIMESTAMPS => false, self::CONFIG_COLUMN_SCHEMA => array( + 'requestIdentifier' => 'bytes12?', 'remoteAddress' => 'ipaddress?', 'remoteProtocol' => 'text32?', 'rejectCode' => 'uint32', 'rejectDetails' => 'text64?', ), self::CONFIG_KEY_SCHEMA => array( 'key_repository' => array( 'columns' => array('repositoryPHID'), ), + 'key_request' => array( + 'columns' => array('requestIdentifier'), + 'unique' => true, + ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryPushEventPHIDType::TYPECONST); } public function attachRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->assertAttached($this->repository); } public function attachLogs(array $logs) { $this->logs = $logs; return $this; } public function getLogs() { return $this->assertAttached($this->logs); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getRepository()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getRepository()->hasAutomaticCapability($capability, $viewer); } public function describeAutomaticCapability($capability) { return pht( "A repository's push events are visible to users who can see the ". "repository."); } } diff --git a/src/infrastructure/ssh/PhabricatorSSHWorkflow.php b/src/infrastructure/ssh/PhabricatorSSHWorkflow.php index 6b6222304c..f7739f1332 100644 --- a/src/infrastructure/ssh/PhabricatorSSHWorkflow.php +++ b/src/infrastructure/ssh/PhabricatorSSHWorkflow.php @@ -1,113 +1,123 @@ errorChannel = $error_channel; return $this; } public function getErrorChannel() { return $this->errorChannel; } public function setSSHUser(PhabricatorUser $ssh_user) { $this->sshUser = $ssh_user; return $this; } public function getSSHUser() { return $this->sshUser; } public function setIOChannel(PhutilChannel $channel) { $this->iochannel = $channel; return $this; } public function getIOChannel() { return $this->iochannel; } public function readAllInput() { $channel = $this->getIOChannel(); while ($channel->update()) { PhutilChannel::waitForAny(array($channel)); if (!$channel->isOpenForReading()) { break; } } return $channel->read(); } public function writeIO($data) { $this->getIOChannel()->write($data); return $this; } public function writeErrorIO($data) { $this->getErrorChannel()->write($data); return $this; } protected function newPassthruCommand() { return id(new PhabricatorSSHPassthruCommand()) ->setErrorChannel($this->getErrorChannel()); } public function setIsClusterRequest($is_cluster_request) { $this->isClusterRequest = $is_cluster_request; return $this; } public function getIsClusterRequest() { return $this->isClusterRequest; } public function setOriginalArguments(array $original_arguments) { $this->originalArguments = $original_arguments; return $this; } public function getOriginalArguments() { return $this->originalArguments; } + public function setRequestIdentifier($request_identifier) { + $this->requestIdentifier = $request_identifier; + return $this; + } + + public function getRequestIdentifier() { + return $this->requestIdentifier; + } + public function getSSHRemoteAddress() { $ssh_client = getenv('SSH_CLIENT'); if (!strlen($ssh_client)) { return null; } // TODO: When commands are proxied, the original remote address should // also be proxied. // This has the format " ". Grab the IP. $remote_address = head(explode(' ', $ssh_client)); try { $address = PhutilIPAddress::newAddress($remote_address); } catch (Exception $ex) { return null; } return $address->getAddress(); } }