diff --git a/scripts/ssh/ssh-exec.php b/scripts/ssh/ssh-exec.php index 0f2275cda8..eea2db4712 100755 --- a/scripts/ssh/ssh-exec.php +++ b/scripts/ssh/ssh-exec.php @@ -1,313 +1,313 @@ #!/usr/bin/env php setData( array( 'Q' => $request_identifier, )); $args = new PhutilArgumentParser($argv); $args->setTagline(pht('execute SSH requests')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'phabricator-ssh-user', 'param' => 'username', 'help' => pht( 'If the request authenticated with a user key, the name of the '. 'user.'), ), array( 'name' => 'phabricator-ssh-device', 'param' => 'name', 'help' => pht( 'If the request authenticated with a device key, the name of the '. 'device.'), ), array( 'name' => 'phabricator-ssh-key', 'param' => 'id', 'help' => pht( 'The ID of the SSH key which authenticated this request. This is '. 'used to allow logs to report when specific keys were used, to make '. 'it easier to manage credentials.'), ), array( 'name' => 'ssh-command', 'param' => 'command', 'help' => pht( 'Provide a command to execute. This makes testing this script '. 'easier. When running normally, the command is read from the '. 'environment (%s), which is populated by sshd.', 'SSH_ORIGINAL_COMMAND'), ), )); try { $remote_address = null; $ssh_client = getenv('SSH_CLIENT'); if ($ssh_client) { // This has the format " ". Grab the IP. $remote_address = head(explode(' ', $ssh_client)); $ssh_log->setData( array( 'r' => $remote_address, )); } $key_id = $args->getArg('phabricator-ssh-key'); if ($key_id) { $ssh_log->setData( array( 'k' => $key_id, )); } $user_name = $args->getArg('phabricator-ssh-user'); $device_name = $args->getArg('phabricator-ssh-device'); $user = null; $device = null; $is_cluster_request = false; if ($user_name && $device_name) { throw new Exception( pht( 'The %s and %s flags are mutually exclusive. You can not '. 'authenticate as both a user ("%s") and a device ("%s"). '. 'Specify one or the other, but not both.', '--phabricator-ssh-user', '--phabricator-ssh-device', $user_name, $device_name)); } else if (strlen($user_name)) { $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($user_name)) ->executeOne(); if (!$user) { throw new Exception( pht( 'Invalid username ("%s"). There is no user with this username.', $user_name)); } id(new PhabricatorAuthSessionEngine()) ->willServeRequestForUser($user); } else if (strlen($device_name)) { if (!$remote_address) { throw new Exception( pht( 'Unable to identify remote address from the %s environment '. 'variable. Device authentication is accepted only from trusted '. 'sources.', 'SSH_CLIENT')); } if (!PhabricatorEnv::isClusterAddress($remote_address)) { throw new Exception( pht( 'This request originates from outside of the Phabricator cluster '. 'address range. Requests signed with a trusted device key must '. 'originate from trusted hosts.')); } $device = id(new AlmanacDeviceQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withNames(array($device_name)) ->executeOne(); if (!$device) { throw new Exception( pht( 'Invalid device name ("%s"). There is no device with this name.', $device_name)); } // We're authenticated as a device, but we're going to read the user out of // the command below. $is_cluster_request = true; } else { throw new Exception( pht( 'This script must be invoked with either the %s or %s flag.', '--phabricator-ssh-user', '--phabricator-ssh-device')); } if ($args->getArg('ssh-command')) { $original_command = $args->getArg('ssh-command'); } else { $original_command = getenv('SSH_ORIGINAL_COMMAND'); } $original_argv = id(new PhutilShellLexer()) ->splitArguments($original_command); if ($device) { // If we're authenticating as a device, the first argument may be a // "@username" argument to act as a particular user. $first_argument = head($original_argv); if (preg_match('/^@/', $first_argument)) { $act_as_name = array_shift($original_argv); $act_as_name = substr($act_as_name, 1); $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($act_as_name)) ->executeOne(); if (!$user) { throw new Exception( pht( 'Device request identifies an acting user with an invalid '. 'username ("%s"). There is no user with this username.', $act_as_name)); } } else { $user = PhabricatorUser::getOmnipotentUser(); } } if ($user->isOmnipotent()) { $user_name = 'device/'.$device->getName(); } else { $user_name = $user->getUsername(); } $ssh_log->setData( array( 'u' => $user_name, 'P' => $user->getPHID(), )); if (!$device) { if (!$user->canEstablishSSHSessions()) { throw new Exception( pht( 'Your account ("%s") does not have permission to establish SSH '. 'sessions. Visit the web interface for more information.', $user_name)); } } $workflows = id(new PhutilClassMapQuery()) ->setAncestorClass('PhabricatorSSHWorkflow') ->setUniqueMethod('getName') ->execute(); if (!$original_argv) { throw new Exception( pht( "Welcome to Phabricator.\n\n". "You are logged in as %s.\n\n". "You haven't specified a command to run. This means you're requesting ". "an interactive shell, but Phabricator does not provide an ". "interactive shell over SSH.\n\n". "Usually, you should run a command like `%s` or `%s` ". "rather than connecting directly with SSH.\n\n". "Supported commands are: %s.", $user_name, 'git clone', 'hg push', implode(', ', array_keys($workflows)))); } $log_argv = implode(' ', $original_argv); $log_argv = id(new PhutilUTF8StringTruncator()) ->setMaximumCodepoints(128) ->truncateString($log_argv); $ssh_log->setData( array( 'C' => $original_argv[0], 'U' => $log_argv, )); $command = head($original_argv); $parseable_argv = $original_argv; array_unshift($parseable_argv, 'phabricator-ssh-exec'); $parsed_args = new PhutilArgumentParser($parseable_argv); if (empty($workflows[$command])) { throw new Exception(pht('Invalid command.')); } $workflow = $parsed_args->parseWorkflows($workflows); $workflow->setSSHUser($user); $workflow->setOriginalArguments($original_argv); $workflow->setIsClusterRequest($is_cluster_request); $workflow->setRequestIdentifier($request_identifier); $sock_stdin = fopen('php://stdin', 'r'); if (!$sock_stdin) { throw new Exception(pht('Unable to open stdin.')); } $sock_stdout = fopen('php://stdout', 'w'); if (!$sock_stdout) { throw new Exception(pht('Unable to open stdout.')); } $sock_stderr = fopen('php://stderr', 'w'); if (!$sock_stderr) { throw new Exception(pht('Unable to open stderr.')); } $socket_channel = new PhutilSocketChannel( $sock_stdin, $sock_stdout); $error_channel = new PhutilSocketChannel(null, $sock_stderr); $metrics_channel = new PhutilMetricsChannel($socket_channel); $workflow->setIOChannel($metrics_channel); $workflow->setErrorChannel($error_channel); $rethrow = null; try { $err = $workflow->execute($parsed_args); $metrics_channel->flush(); $error_channel->flush(); } catch (Exception $ex) { $rethrow = $ex; } // Always write this if we got as far as building a metrics channel. $ssh_log->setData( array( 'i' => $metrics_channel->getBytesRead(), 'o' => $metrics_channel->getBytesWritten(), )); if ($rethrow) { throw $rethrow; } } catch (Exception $ex) { fwrite(STDERR, "phabricator-ssh-exec: ".$ex->getMessage()."\n"); $err = 1; } $ssh_log->setData( array( 'c' => $err, - 'T' => (int)(1000000 * (microtime(true) - $ssh_start_time)), + 'T' => phutil_microseconds_since($ssh_start_time), )); exit($err); diff --git a/src/applications/conduit/controller/PhabricatorConduitAPIController.php b/src/applications/conduit/controller/PhabricatorConduitAPIController.php index aea3fc0908..f217adf154 100644 --- a/src/applications/conduit/controller/PhabricatorConduitAPIController.php +++ b/src/applications/conduit/controller/PhabricatorConduitAPIController.php @@ -1,716 +1,714 @@ getURIData('method'); $time_start = microtime(true); $api_request = null; $method_implementation = null; $log = new PhabricatorConduitMethodCallLog(); $log->setMethod($method); $metadata = array(); $multimeter = MultimeterControl::getInstance(); if ($multimeter) { $multimeter->setEventContext('api.'.$method); } try { list($metadata, $params, $strictly_typed) = $this->decodeConduitParams( $request, $method); $call = new ConduitCall($method, $params, $strictly_typed); $method_implementation = $call->getMethodImplementation(); $result = null; // TODO: The relationship between ConduitAPIRequest and ConduitCall is a // little odd here and could probably be improved. Specifically, the // APIRequest is a sub-object of the Call, which does not parallel the // role of AphrontRequest (which is an indepenent object). // In particular, the setUser() and getUser() existing independently on // the Call and APIRequest is very awkward. $api_request = $call->getAPIRequest(); $allow_unguarded_writes = false; $auth_error = null; $conduit_username = '-'; if ($call->shouldRequireAuthentication()) { $auth_error = $this->authenticateUser($api_request, $metadata, $method); // If we've explicitly authenticated the user here and either done // CSRF validation or are using a non-web authentication mechanism. $allow_unguarded_writes = true; if ($auth_error === null) { $conduit_user = $api_request->getUser(); if ($conduit_user && $conduit_user->getPHID()) { $conduit_username = $conduit_user->getUsername(); } $call->setUser($api_request->getUser()); } } $access_log = PhabricatorAccessLog::getLog(); if ($access_log) { $access_log->setData( array( 'u' => $conduit_username, 'm' => $method, )); } if ($call->shouldAllowUnguardedWrites()) { $allow_unguarded_writes = true; } if ($auth_error === null) { if ($allow_unguarded_writes) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); } try { $result = $call->execute(); $error_code = null; $error_info = null; } catch (ConduitException $ex) { $result = null; $error_code = $ex->getMessage(); if ($ex->getErrorDescription()) { $error_info = $ex->getErrorDescription(); } else { $error_info = $call->getErrorDescription($error_code); } } if ($allow_unguarded_writes) { unset($unguarded); } } else { list($error_code, $error_info) = $auth_error; } } catch (Exception $ex) { if (!($ex instanceof ConduitMethodNotFoundException)) { phlog($ex); } $result = null; $error_code = ($ex instanceof ConduitException ? 'ERR-CONDUIT-CALL' : 'ERR-CONDUIT-CORE'); $error_info = $ex->getMessage(); } - $time_end = microtime(true); - $log ->setCallerPHID( isset($conduit_user) ? $conduit_user->getPHID() : null) ->setError((string)$error_code) - ->setDuration(1000000 * ($time_end - $time_start)); + ->setDuration(phutil_microseconds_since($time_start)); if (!PhabricatorEnv::isReadOnly()) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $log->save(); unset($unguarded); } $response = id(new ConduitAPIResponse()) ->setResult($result) ->setErrorCode($error_code) ->setErrorInfo($error_info); switch ($request->getStr('output')) { case 'human': return $this->buildHumanReadableResponse( $method, $api_request, $response->toDictionary(), $method_implementation); case 'json': default: return id(new AphrontJSONResponse()) ->setAddJSONShield(false) ->setContent($response->toDictionary()); } } /** * Authenticate the client making the request to a Phabricator user account. * * @param ConduitAPIRequest Request being executed. * @param dict Request metadata. * @return null|pair Null to indicate successful authentication, or * an error code and error message pair. */ private function authenticateUser( ConduitAPIRequest $api_request, array $metadata, $method) { $request = $this->getRequest(); if ($request->getUser()->getPHID()) { $request->validateCSRF(); return $this->validateAuthenticatedUser( $api_request, $request->getUser()); } $auth_type = idx($metadata, 'auth.type'); if ($auth_type === ConduitClient::AUTH_ASYMMETRIC) { $host = idx($metadata, 'auth.host'); if (!$host) { return array( 'ERR-INVALID-AUTH', pht( 'Request is missing required "%s" parameter.', 'auth.host'), ); } // TODO: Validate that we are the host! $raw_key = idx($metadata, 'auth.key'); $public_key = PhabricatorAuthSSHPublicKey::newFromRawKey($raw_key); $ssl_public_key = $public_key->toPKCS8(); // First, verify the signature. try { $protocol_data = $metadata; ConduitClient::verifySignature( $method, $api_request->getAllParameters(), $protocol_data, $ssl_public_key); } catch (Exception $ex) { return array( 'ERR-INVALID-AUTH', pht( 'Signature verification failure. %s', $ex->getMessage()), ); } // If the signature is valid, find the user or device which is // associated with this public key. $stored_key = id(new PhabricatorAuthSSHKeyQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withKeys(array($public_key)) ->withIsActive(true) ->executeOne(); if (!$stored_key) { $key_summary = id(new PhutilUTF8StringTruncator()) ->setMaximumBytes(64) ->truncateString($raw_key); return array( 'ERR-INVALID-AUTH', pht( 'No user or device is associated with the public key "%s".', $key_summary), ); } $object = $stored_key->getObject(); if ($object instanceof PhabricatorUser) { $user = $object; } else { if (!$stored_key->getIsTrusted()) { return array( 'ERR-INVALID-AUTH', pht( 'The key which signed this request is not trusted. Only '. 'trusted keys can be used to sign API calls.'), ); } if (!PhabricatorEnv::isClusterRemoteAddress()) { return array( 'ERR-INVALID-AUTH', pht( 'This request originates from outside of the Phabricator '. 'cluster address range. Requests signed with trusted '. 'device keys must originate from within the cluster.'), ); } $user = PhabricatorUser::getOmnipotentUser(); // Flag this as an intracluster request. $api_request->setIsClusterRequest(true); } return $this->validateAuthenticatedUser( $api_request, $user); } else if ($auth_type === null) { // No specified authentication type, continue with other authentication // methods below. } else { return array( 'ERR-INVALID-AUTH', pht( 'Provided "%s" ("%s") is not recognized.', 'auth.type', $auth_type), ); } $token_string = idx($metadata, 'token'); if (strlen($token_string)) { if (strlen($token_string) != 32) { return array( 'ERR-INVALID-AUTH', pht( 'API token "%s" has the wrong length. API tokens should be '. '32 characters long.', $token_string), ); } $type = head(explode('-', $token_string)); $valid_types = PhabricatorConduitToken::getAllTokenTypes(); $valid_types = array_fuse($valid_types); if (empty($valid_types[$type])) { return array( 'ERR-INVALID-AUTH', pht( 'API token "%s" has the wrong format. API tokens should be '. '32 characters long and begin with one of these prefixes: %s.', $token_string, implode(', ', $valid_types)), ); } $token = id(new PhabricatorConduitTokenQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTokens(array($token_string)) ->withExpired(false) ->executeOne(); if (!$token) { $token = id(new PhabricatorConduitTokenQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withTokens(array($token_string)) ->withExpired(true) ->executeOne(); if ($token) { return array( 'ERR-INVALID-AUTH', pht( 'API token "%s" was previously valid, but has expired.', $token_string), ); } else { return array( 'ERR-INVALID-AUTH', pht( 'API token "%s" is not valid.', $token_string), ); } } // If this is a "cli-" token, it expires shortly after it is generated // by default. Once it is actually used, we extend its lifetime and make // it permanent. This allows stray tokens to get cleaned up automatically // if they aren't being used. if ($token->getTokenType() == PhabricatorConduitToken::TYPE_COMMANDLINE) { if ($token->getExpires()) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $token->setExpires(null); $token->save(); unset($unguarded); } } // If this is a "clr-" token, Phabricator must be configured in cluster // mode and the remote address must be a cluster node. if ($token->getTokenType() == PhabricatorConduitToken::TYPE_CLUSTER) { if (!PhabricatorEnv::isClusterRemoteAddress()) { return array( 'ERR-INVALID-AUTH', pht( 'This request originates from outside of the Phabricator '. 'cluster address range. Requests signed with cluster API '. 'tokens must originate from within the cluster.'), ); } // Flag this as an intracluster request. $api_request->setIsClusterRequest(true); } $user = $token->getObject(); if (!($user instanceof PhabricatorUser)) { return array( 'ERR-INVALID-AUTH', pht('API token is not associated with a valid user.'), ); } return $this->validateAuthenticatedUser( $api_request, $user); } $access_token = idx($metadata, 'access_token'); if ($access_token) { $token = id(new PhabricatorOAuthServerAccessToken()) ->loadOneWhere('token = %s', $access_token); if (!$token) { return array( 'ERR-INVALID-AUTH', pht('Access token does not exist.'), ); } $oauth_server = new PhabricatorOAuthServer(); $authorization = $oauth_server->authorizeToken($token); if (!$authorization) { return array( 'ERR-INVALID-AUTH', pht('Access token is invalid or expired.'), ); } $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($token->getUserPHID())) ->executeOne(); if (!$user) { return array( 'ERR-INVALID-AUTH', pht('Access token is for invalid user.'), ); } $ok = $this->authorizeOAuthMethodAccess($authorization, $method); if (!$ok) { return array( 'ERR-OAUTH-ACCESS', pht('You do not have authorization to call this method.'), ); } $api_request->setOAuthToken($token); return $this->validateAuthenticatedUser( $api_request, $user); } // For intracluster requests, use a public user if no authentication // information is provided. We could do this safely for any request, // but making the API fully public means there's no way to disable badly // behaved clients. if (PhabricatorEnv::isClusterRemoteAddress()) { if (PhabricatorEnv::getEnvConfig('policy.allow-public')) { $api_request->setIsClusterRequest(true); $user = new PhabricatorUser(); return $this->validateAuthenticatedUser( $api_request, $user); } } // Handle sessionless auth. // TODO: This is super messy. // TODO: Remove this in favor of token-based auth. if (isset($metadata['authUser'])) { $user = id(new PhabricatorUser())->loadOneWhere( 'userName = %s', $metadata['authUser']); if (!$user) { return array( 'ERR-INVALID-AUTH', pht('Authentication is invalid.'), ); } $token = idx($metadata, 'authToken'); $signature = idx($metadata, 'authSignature'); $certificate = $user->getConduitCertificate(); $hash = sha1($token.$certificate); if (!phutil_hashes_are_identical($hash, $signature)) { return array( 'ERR-INVALID-AUTH', pht('Authentication is invalid.'), ); } return $this->validateAuthenticatedUser( $api_request, $user); } // Handle session-based auth. // TODO: Remove this in favor of token-based auth. $session_key = idx($metadata, 'sessionKey'); if (!$session_key) { return array( 'ERR-INVALID-SESSION', pht('Session key is not present.'), ); } $user = id(new PhabricatorAuthSessionEngine()) ->loadUserForSession(PhabricatorAuthSession::TYPE_CONDUIT, $session_key); if (!$user) { return array( 'ERR-INVALID-SESSION', pht('Session key is invalid.'), ); } return $this->validateAuthenticatedUser( $api_request, $user); } private function validateAuthenticatedUser( ConduitAPIRequest $request, PhabricatorUser $user) { if (!$user->canEstablishAPISessions()) { return array( 'ERR-INVALID-AUTH', pht('User account is not permitted to use the API.'), ); } $request->setUser($user); id(new PhabricatorAuthSessionEngine()) ->willServeRequestForUser($user); return null; } private function buildHumanReadableResponse( $method, ConduitAPIRequest $request = null, $result = null, ConduitAPIMethod $method_implementation = null) { $param_rows = array(); $param_rows[] = array('Method', $this->renderAPIValue($method)); if ($request) { foreach ($request->getAllParameters() as $key => $value) { $param_rows[] = array( $key, $this->renderAPIValue($value), ); } } $param_table = new AphrontTableView($param_rows); $param_table->setColumnClasses( array( 'header', 'wide', )); $result_rows = array(); foreach ($result as $key => $value) { $result_rows[] = array( $key, $this->renderAPIValue($value), ); } $result_table = new AphrontTableView($result_rows); $result_table->setColumnClasses( array( 'header', 'wide', )); $param_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Method Parameters')) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setTable($param_table); $result_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Method Result')) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setTable($result_table); $method_uri = $this->getApplicationURI('method/'.$method.'/'); $crumbs = $this->buildApplicationCrumbs() ->addTextCrumb($method, $method_uri) ->addTextCrumb(pht('Call')) ->setBorder(true); $example_panel = null; if ($request && $method_implementation) { $params = $request->getAllParameters(); $example_panel = $this->renderExampleBox( $method_implementation, $params); } $title = pht('Method Call Result'); $header = id(new PHUIHeaderView()) ->setHeader($title) ->setHeaderIcon('fa-exchange'); $view = id(new PHUITwoColumnView()) ->setHeader($header) ->setFooter(array( $param_panel, $result_panel, $example_panel, )); $title = pht('Method Call Result'); return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) ->appendChild($view); } private function renderAPIValue($value) { $json = new PhutilJSON(); if (is_array($value)) { $value = $json->encodeFormatted($value); } $value = phutil_tag( 'pre', array('style' => 'white-space: pre-wrap;'), $value); return $value; } private function decodeConduitParams( AphrontRequest $request, $method) { // Look for parameters from the Conduit API Console, which are encoded // as HTTP POST parameters in an array, e.g.: // // params[name]=value¶ms[name2]=value2 // // The fields are individually JSON encoded, since we require users to // enter JSON so that we avoid type ambiguity. $params = $request->getArr('params', null); if ($params !== null) { foreach ($params as $key => $value) { if ($value == '') { // Interpret empty string null (e.g., the user didn't type anything // into the box). $value = 'null'; } $decoded_value = json_decode($value, true); if ($decoded_value === null && strtolower($value) != 'null') { // When json_decode() fails, it returns null. This almost certainly // indicates that a user was using the web UI and didn't put quotes // around a string value. We can either do what we think they meant // (treat it as a string) or fail. For now, err on the side of // caution and fail. In the future, if we make the Conduit API // actually do type checking, it might be reasonable to treat it as // a string if the parameter type is string. throw new Exception( pht( "The value for parameter '%s' is not valid JSON. All ". "parameters must be encoded as JSON values, including strings ". "(which means you need to surround them in double quotes). ". "Check your syntax. Value was: %s.", $key, $value)); } $params[$key] = $decoded_value; } $metadata = idx($params, '__conduit__', array()); unset($params['__conduit__']); return array($metadata, $params, true); } // Otherwise, look for a single parameter called 'params' which has the // entire param dictionary JSON encoded. $params_json = $request->getStr('params'); if (strlen($params_json)) { $params = null; try { $params = phutil_json_decode($params_json); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht( "Invalid parameter information was passed to method '%s'.", $method), $ex); } $metadata = idx($params, '__conduit__', array()); unset($params['__conduit__']); return array($metadata, $params, true); } // If we do not have `params`, assume this is a simple HTTP request with // HTTP key-value pairs. $params = array(); $metadata = array(); foreach ($request->getPassthroughRequestData() as $key => $value) { $meta_key = ConduitAPIMethod::getParameterMetadataKey($key); if ($meta_key !== null) { $metadata[$meta_key] = $value; } else { $params[$key] = $value; } } return array($metadata, $params, false); } private function authorizeOAuthMethodAccess( PhabricatorOAuthClientAuthorization $authorization, $method_name) { $method = ConduitAPIMethod::getConduitMethod($method_name); if (!$method) { return false; } $required_scope = $method->getRequiredScope(); switch ($required_scope) { case ConduitAPIMethod::SCOPE_ALWAYS: return true; case ConduitAPIMethod::SCOPE_NEVER: return false; } $authorization_scope = $authorization->getScope(); if (!empty($authorization_scope[$required_scope])) { return true; } return false; } } diff --git a/src/applications/conduit/ssh/ConduitSSHWorkflow.php b/src/applications/conduit/ssh/ConduitSSHWorkflow.php index 603a479ea0..0b4bd3cdef 100644 --- a/src/applications/conduit/ssh/ConduitSSHWorkflow.php +++ b/src/applications/conduit/ssh/ConduitSSHWorkflow.php @@ -1,87 +1,85 @@ setName('conduit'); $this->setArguments( array( array( 'name' => 'method', 'wildcard' => true, ), )); } public function execute(PhutilArgumentParser $args) { $time_start = microtime(true); $methodv = $args->getArg('method'); if (!$methodv) { throw new Exception(pht('No Conduit method provided.')); } else if (count($methodv) > 1) { throw new Exception(pht('Too many Conduit methods provided.')); } $method = head($methodv); $json = $this->readAllInput(); $raw_params = null; try { $raw_params = phutil_json_decode($json); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht('Invalid JSON input.'), $ex); } $params = idx($raw_params, 'params', '[]'); $params = phutil_json_decode($params); $metadata = idx($params, '__conduit__', array()); unset($params['__conduit__']); $call = null; $error_code = null; $error_info = null; try { $call = new ConduitCall($method, $params); $call->setUser($this->getSSHUser()); $result = $call->execute(); } catch (ConduitException $ex) { $result = null; $error_code = $ex->getMessage(); if ($ex->getErrorDescription()) { $error_info = $ex->getErrorDescription(); } else if ($call) { $error_info = $call->getErrorDescription($error_code); } } $response = id(new ConduitAPIResponse()) ->setResult($result) ->setErrorCode($error_code) ->setErrorInfo($error_info); $json_out = json_encode($response->toDictionary()); $json_out = $json_out."\n"; $this->getIOChannel()->write($json_out); // NOTE: Flush here so we can get an accurate result for the duration, // if the response is large and the receiver is slow to read it. $this->getIOChannel()->flush(); - $time_end = microtime(true); - $connection_id = idx($metadata, 'connectionID'); $log = id(new PhabricatorConduitMethodCallLog()) ->setCallerPHID($this->getSSHUser()->getPHID()) ->setConnectionID($connection_id) ->setMethod($method) ->setError((string)$error_code) - ->setDuration(1000000 * ($time_end - $time_start)) + ->setDuration(phutil_microseconds_since($time_start)) ->save(); } } diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index d22635e859..236e289ec0 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -1,1377 +1,1376 @@ remoteProtocol = $remote_protocol; return $this; } public function getRemoteProtocol() { return $this->remoteProtocol; } public function setRemoteAddress($remote_address) { $this->remoteAddress = $remote_address; return $this; } public function getRemoteAddress() { return $this->remoteAddress; } public function setRequestIdentifier($request_identifier) { $this->requestIdentifier = $request_identifier; return $this; } public function getRequestIdentifier() { return $this->requestIdentifier; } public function setStartTime($start_time) { $this->startTime = $start_time; return $this; } public function getStartTime() { return $this->startTime; } public function setSubversionTransactionInfo($transaction, $repository) { $this->subversionTransaction = $transaction; $this->subversionRepository = $repository; return $this; } public function setStdin($stdin) { $this->stdin = $stdin; return $this; } public function getStdin() { return $this->stdin; } public function setOriginalArgv(array $original_argv) { $this->originalArgv = $original_argv; return $this; } public function getOriginalArgv() { return $this->originalArgv; } public function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setMercurialHook($mercurial_hook) { $this->mercurialHook = $mercurial_hook; return $this; } public function getMercurialHook() { return $this->mercurialHook; } /* -( Hook Execution )----------------------------------------------------- */ public function execute() { $ref_updates = $this->findRefUpdates(); $all_updates = $ref_updates; $caught = null; try { try { $this->rejectDangerousChanges($ref_updates); } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting dangerous changes, flag everything that we've // seen as rejected so it's clear that none of it was accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_DANGEROUS; throw $ex; } $content_updates = $this->findContentUpdates($ref_updates); $all_updates = array_merge($ref_updates, $content_updates); // If this is an "initial import" (a sizable push to a previously empty // repository) we'll allow enormous changes and disable Herald rules. // These rulesets can consume a large amount of time and memory and are // generally not relevant when importing repository history. $is_initial_import = $this->isInitialImport($all_updates); if (!$is_initial_import) { $this->applyHeraldRefRules($ref_updates); } try { if (!$is_initial_import) { $this->rejectEnormousChanges($content_updates); } } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting enormous changes, flag everything. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ENORMOUS; throw $ex; } if (!$is_initial_import) { $this->applyHeraldContentRules($content_updates); } // Run custom scripts in `hook.d/` directories. $this->applyCustomHooks($all_updates); // If we make it this far, we're accepting these changes. Mark all the // logs as accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ACCEPT; } catch (Exception $ex) { // We'll throw this again in a minute, but we want to save all the logs // first. $caught = $ex; } // Save all the logs no matter what the outcome was. $event = $this->newPushEvent(); $event->setRejectCode($this->rejectCode); $event->setRejectDetails($this->rejectDetails); $event->openTransaction(); $event->save(); foreach ($all_updates as $update) { $update->setPushEventPHID($event->getPHID()); $update->save(); } $event->saveTransaction(); if ($caught) { throw $caught; } // If this went through cleanly and was an import, set the importing flag // on the repository. It will be cleared once we fully process everything. if ($is_initial_import) { $repository = $this->getRepository(); $repository->markImporting(); } if ($this->emailPHIDs) { // If Herald rules triggered email to users, queue a worker to send the // mail. We do this out-of-process so that we block pushes as briefly // as possible. // (We do need to pull some commit info here because the commit objects // may not exist yet when this worker runs, which could be immediately.) PhabricatorWorker::scheduleTask( 'PhabricatorRepositoryPushMailWorker', array( 'eventPHID' => $event->getPHID(), 'emailPHIDs' => array_values($this->emailPHIDs), 'info' => $this->loadCommitInfoForWorker($all_updates), ), array( 'priority' => PhabricatorWorker::PRIORITY_ALERTS, )); } return 0; } private function findRefUpdates() { $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionRefUpdates(); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } private function rejectDangerousChanges(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $repository = $this->getRepository(); if ($repository->shouldAllowDangerousChanges()) { return; } $flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; foreach ($ref_updates as $ref_update) { if (!$ref_update->hasChangeFlags($flag_dangerous)) { // This is not a dangerous change. continue; } // We either have a branch deletion or a non fast-forward branch update. // Format a message and reject the push. $message = pht( "DANGEROUS CHANGE: %s\n". "Dangerous change protection is enabled for this repository.\n". "Edit the repository configuration before making dangerous changes.", $ref_update->getDangerousChangeDescription()); throw new DiffusionCommitHookRejectException($message); } } private function findContentUpdates(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionContentUpdates($ref_updates); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } /* -( Herald )------------------------------------------------------------- */ private function applyHeraldRefRules(array $ref_updates) { $this->applyHeraldRules( $ref_updates, new HeraldPreCommitRefAdapter()); } private function applyHeraldContentRules(array $content_updates) { $this->applyHeraldRules( $content_updates, new HeraldPreCommitContentAdapter()); } private function applyHeraldRules( array $updates, HeraldAdapter $adapter_template) { if (!$updates) { return; } $viewer = $this->getViewer(); $adapter_template ->setHookEngine($this) ->setActingAsPHID($viewer->getPHID()); $engine = new HeraldEngine(); $rules = null; $blocking_effect = null; $blocked_update = null; $blocking_xscript = null; foreach ($updates as $update) { $adapter = id(clone $adapter_template) ->setPushLog($update); if ($rules === null) { $rules = $engine->loadRulesForAdapter($adapter); } $effects = $engine->applyRules($rules, $adapter); $engine->applyEffects($effects, $adapter, $rules); $xscript = $engine->getTranscript(); // Store any PHIDs we want to send email to for later. foreach ($adapter->getEmailPHIDs() as $email_phid) { $this->emailPHIDs[$email_phid] = $email_phid; } $block_action = DiffusionBlockHeraldAction::ACTIONCONST; if ($blocking_effect === null) { foreach ($effects as $effect) { if ($effect->getAction() == $block_action) { $blocking_effect = $effect; $blocked_update = $update; $blocking_xscript = $xscript; break; } } } } if ($blocking_effect) { $rule = $blocking_effect->getRule(); $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_HERALD; $this->rejectDetails = $rule->getPHID(); $message = $blocking_effect->getTarget(); if (!strlen($message)) { $message = pht('(None.)'); } $blocked_ref_name = coalesce( $blocked_update->getRefName(), $blocked_update->getRefNewShort()); $blocked_name = $blocked_update->getRefType().'/'.$blocked_ref_name; throw new DiffusionCommitHookRejectException( pht( "This push was rejected by Herald push rule %s.\n". " Change: %s\n". " Rule: %s\n". " Reason: %s\n". "Transcript: %s", $rule->getMonogram(), $blocked_name, $rule->getName(), $message, PhabricatorEnv::getProductionURI( '/herald/transcript/'.$blocking_xscript->getID().'/'))); } } public function loadViewerProjectPHIDsForHerald() { // This just caches the viewer's projects so we don't need to load them // over and over again when applying Herald rules. if ($this->heraldViewerProjects === null) { $this->heraldViewerProjects = id(new PhabricatorProjectQuery()) ->setViewer($this->getViewer()) ->withMemberPHIDs(array($this->getViewer()->getPHID())) ->execute(); } return mpull($this->heraldViewerProjects, 'getPHID'); } /* -( Git )---------------------------------------------------------------- */ private function findGitRefUpdates() { $ref_updates = array(); // First, parse stdin, which lists all the ref changes. The input looks // like this: // // $stdin = $this->getStdin(); $lines = phutil_split_lines($stdin, $retain_endings = false); foreach ($lines as $line) { $parts = explode(' ', $line, 3); if (count($parts) != 3) { throw new Exception(pht('Expected "old new ref", got "%s".', $line)); } $ref_old = $parts[0]; $ref_new = $parts[1]; $ref_raw = $parts[2]; if (preg_match('(^refs/heads/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; $ref_raw = substr($ref_raw, strlen('refs/heads/')); } else if (preg_match('(^refs/tags/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG; $ref_raw = substr($ref_raw, strlen('refs/tags/')); } else { throw new Exception( pht( "Unable to identify the reftype of '%s'. Rejecting push.", $ref_raw)); } $ref_update = $this->newPushLog() ->setRefType($ref_type) ->setRefName($ref_raw) ->setRefOld($ref_old) ->setRefNew($ref_new); $ref_updates[] = $ref_update; } $this->findGitMergeBases($ref_updates); $this->findGitChangeFlags($ref_updates); return $ref_updates; } private function findGitMergeBases(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $futures = array(); foreach ($ref_updates as $key => $ref_update) { // If the old hash is "00000...", the ref is being created (either a new // branch, or a new tag). If the new hash is "00000...", the ref is being // deleted. If both are nonempty, the ref is being updated. For updates, // we'll figure out the `merge-base` of the old and new objects here. This // lets us reject non-FF changes cheaply; later, we'll figure out exactly // which commits are new. $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); if (($ref_old === self::EMPTY_HASH) || ($ref_new === self::EMPTY_HASH)) { continue; } $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'merge-base %s %s', $ref_old, $ref_new); } $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { // If 'old' and 'new' have no common ancestors (for example, a force push // which completely rewrites a ref), `git merge-base` will exit with // an error and no output. It would be nice to find a positive test // for this instead, but I couldn't immediately come up with one. See // T4224. Assume this means there are no ancestors. list($err, $stdout) = $future->resolve(); if ($err) { $merge_base = null; } else { $merge_base = rtrim($stdout, "\n"); } $ref_update = $ref_updates[$key]; $ref_update->setMergeBase($merge_base); } return $ref_updates; } private function findGitChangeFlags(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); foreach ($ref_updates as $key => $ref_update) { $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); $ref_type = $ref_update->getRefType(); $ref_flags = 0; $dangerous = null; if (($ref_old === self::EMPTY_HASH) && ($ref_new === self::EMPTY_HASH)) { // This happens if you try to delete a tag or branch which does not // exist by pushing directly to the ref. Git will warn about it but // allow it. Just call it a delete, without flagging it as dangerous. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else if ($ref_old === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($ref_new === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push deletes the branch '%s'.", $ref_update->getRefName()); } } else { $merge_base = $ref_update->getMergeBase(); if ($merge_base == $ref_old) { // This is a fast-forward update to an existing branch. // These are safe. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; // For now, we don't consider deleting or moving tags to be a // "dangerous" update. It's way harder to get wrong and should be easy // to recover from once we have better logging. Only add the dangerous // flag if this ref is a branch. if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push updates the branch '%s' ". "from '%s' to '%s', but this is not a fast-forward. Pushes ". "which rewrite published branch history are dangerous.", $ref_update->getRefName(), $ref_update->getRefOldShort(), $ref_update->getRefNewShort()); } } } $ref_update->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } } return $ref_updates; } private function findGitContentUpdates(array $ref_updates) { $flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; $futures = array(); foreach ($ref_updates as $key => $ref_update) { if ($ref_update->hasChangeFlags($flag_delete)) { // Deleting a branch or tag can never create any new commits. continue; } // NOTE: This piece of magic finds all new commits, by walking backward // from the new value to the value of *any* existing ref in the // repository. Particularly, this will cover the cases of a new branch, a // completely moved tag, etc. $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'log --format=%s %s --not --all', '%H', $ref_update->getRefNew()); } $content_updates = array(); $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { list($stdout) = $future->resolvex(); if (!strlen(trim($stdout))) { // This change doesn't have any new commits. One common case of this // is creating a new tag which points at an existing commit. continue; } $commits = phutil_split_lines($stdout, $retain_newlines = false); // If we're looking at a branch, mark all of the new commits as on that // branch. It's only possible for these commits to be on updated branches, // since any other branch heads are necessarily behind them. $branch_name = null; $ref_update = $ref_updates[$key]; $type_branch = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; if ($ref_update->getRefType() == $type_branch) { $branch_name = $ref_update->getRefName(); } foreach ($commits as $commit) { if ($branch_name) { $this->gitCommits[$commit][] = $branch_name; } $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } } return $content_updates; } /* -( Custom )------------------------------------------------------------- */ private function applyCustomHooks(array $updates) { $args = $this->getOriginalArgv(); $stdin = $this->getStdin(); $console = PhutilConsole::getConsole(); $env = array( self::ENV_REPOSITORY => $this->getRepository()->getPHID(), self::ENV_USER => $this->getViewer()->getUsername(), self::ENV_REQUEST => $this->getRequestIdentifier(), self::ENV_REMOTE_PROTOCOL => $this->getRemoteProtocol(), self::ENV_REMOTE_ADDRESS => $this->getRemoteAddress(), ); $repository = $this->getRepository(); $env += $repository->getPassthroughEnvironmentalVariables(); $directories = $repository->getHookDirectories(); foreach ($directories as $directory) { $hooks = $this->getExecutablesInDirectory($directory); sort($hooks); foreach ($hooks as $hook) { // NOTE: We're explicitly running the hooks in sequential order to // make this more predictable. $future = id(new ExecFuture('%s %Ls', $hook, $args)) ->setEnv($env, $wipe_process_env = false) ->write($stdin); list($err, $stdout, $stderr) = $future->resolve(); if (!$err) { // This hook ran OK, but echo its output in case there was something // informative. $console->writeOut('%s', $stdout); $console->writeErr('%s', $stderr); continue; } $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_EXTERNAL; $this->rejectDetails = basename($hook); throw new DiffusionCommitHookRejectException( pht( "This push was rejected by custom hook script '%s':\n\n%s%s", basename($hook), $stdout, $stderr)); } } } private function getExecutablesInDirectory($directory) { $executables = array(); if (!Filesystem::pathExists($directory)) { return $executables; } foreach (Filesystem::listDirectory($directory) as $path) { $full_path = $directory.DIRECTORY_SEPARATOR.$path; if (!is_executable($full_path)) { // Don't include non-executable files. continue; } if (basename($full_path) == 'README') { // Don't include README, even if it is marked as executable. It almost // certainly got caught in the crossfire of a sweeping `chmod`, since // users do this with some frequency. continue; } $executables[] = $full_path; } return $executables; } /* -( Mercurial )---------------------------------------------------------- */ private function findMercurialRefUpdates() { $hook = $this->getMercurialHook(); switch ($hook) { case 'pretxnchangegroup': return $this->findMercurialChangegroupRefUpdates(); case 'prepushkey': return $this->findMercurialPushKeyRefUpdates(); default: throw new Exception(pht('Unrecognized hook "%s"!', $hook)); } } private function findMercurialChangegroupRefUpdates() { $hg_node = getenv('HG_NODE'); if (!$hg_node) { throw new Exception( pht( 'Expected %s in environment!', 'HG_NODE')); } // NOTE: We need to make sure this is passed to subprocesses, or they won't // be able to see new commits. Mercurial uses this as a marker to determine // whether the pending changes are visible or not. $_ENV['HG_PENDING'] = getenv('HG_PENDING'); $repository = $this->getRepository(); $futures = array(); foreach (array('old', 'new') as $key) { $futures[$key] = $repository->getLocalCommandFuture( 'heads --template %s', '{node}\1{branch}\2'); } // Wipe HG_PENDING out of the old environment so we see the pre-commit // state of the repository. $futures['old']->updateEnv('HG_PENDING', null); $futures['commits'] = $repository->getLocalCommandFuture( 'log --rev %s --template %s', hgsprintf('%s:%s', $hg_node, 'tip'), '{node}\1{branch}\2'); // Resolve all of the futures now. We don't need the 'commits' future yet, // but it simplifies the logic to just get it out of the way. foreach (new FutureIterator($futures) as $future) { $future->resolve(); } list($commit_raw) = $futures['commits']->resolvex(); $commit_map = $this->parseMercurialCommits($commit_raw); $this->mercurialCommits = $commit_map; // NOTE: `hg heads` exits with an error code and no output if the repository // has no heads. Most commonly this happens on a new repository. We know // we can run `hg` successfully since the `hg log` above didn't error, so // just ignore the error code. list($err, $old_raw) = $futures['old']->resolve(); $old_refs = $this->parseMercurialHeads($old_raw); list($err, $new_raw) = $futures['new']->resolve(); $new_refs = $this->parseMercurialHeads($new_raw); $all_refs = array_keys($old_refs + $new_refs); $ref_updates = array(); foreach ($all_refs as $ref) { $old_heads = idx($old_refs, $ref, array()); $new_heads = idx($new_refs, $ref, array()); sort($old_heads); sort($new_heads); if (!$old_heads && !$new_heads) { // This should never be possible, as it makes no sense. Explode. throw new Exception( pht( 'Mercurial repository has no new or old heads for branch "%s" '. 'after push. This makes no sense; rejecting change.', $ref)); } if ($old_heads === $new_heads) { // No changes to this branch, so skip it. continue; } $stray_heads = array(); $head_map = array(); if ($old_heads && !$new_heads) { // This is a branch deletion with "--close-branch". foreach ($old_heads as $old_head) { $head_map[$old_head] = array(self::EMPTY_HASH); } } else if (count($old_heads) > 1) { // HORRIBLE: In Mercurial, branches can have multiple heads. If the // old branch had multiple heads, we need to figure out which new // heads descend from which old heads, so we can tell whether you're // actively creating new heads (dangerous) or just working in a // repository that's already full of garbage (strongly discouraged but // not as inherently dangerous). These cases should be very uncommon. // NOTE: We're only looking for heads on the same branch. The old // tip of the branch may be the branchpoint for other branches, but that // is OK. $dfutures = array(); foreach ($old_heads as $old_head) { $dfutures[$old_head] = $repository->getLocalCommandFuture( 'log --branch %s --rev %s --template %s', $ref, hgsprintf('(descendants(%s) and head())', $old_head), '{node}\1'); } foreach (new FutureIterator($dfutures) as $future_head => $dfuture) { list($stdout) = $dfuture->resolvex(); $descendant_heads = array_filter(explode("\1", $stdout)); if ($descendant_heads) { // This old head has at least one descendant in the push. $head_map[$future_head] = $descendant_heads; } else { // This old head has no descendants, so it is being deleted. $head_map[$future_head] = array(self::EMPTY_HASH); } } // Now, find all the new stray heads this push creates, if any. These // are new heads which do not descend from the old heads. $seen = array_fuse(array_mergev($head_map)); foreach ($new_heads as $new_head) { if ($new_head === self::EMPTY_HASH) { // If a branch head is being deleted, don't insert it as an add. continue; } if (empty($seen[$new_head])) { $head_map[self::EMPTY_HASH][] = $new_head; } } } else if ($old_heads) { $head_map[head($old_heads)] = $new_heads; } else { $head_map[self::EMPTY_HASH] = $new_heads; } foreach ($head_map as $old_head => $child_heads) { foreach ($child_heads as $new_head) { if ($new_head === $old_head) { continue; } $ref_flags = 0; $dangerous = null; if ($old_head == self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } $deletes_existing_head = ($new_head == self::EMPTY_HASH); $splits_existing_head = (count($child_heads) > 1); $creates_duplicate_head = ($old_head == self::EMPTY_HASH) && (count($head_map) > 1); if ($splits_existing_head || $creates_duplicate_head) { $readable_child_heads = array(); foreach ($child_heads as $child_head) { $readable_child_heads[] = substr($child_head, 0, 12); } $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; if ($splits_existing_head) { // We're splitting an existing head into two or more heads. // This is dangerous, and a super bad idea. Note that we're only // raising this if you're actively splitting a branch head. If a // head split in the past, we don't consider appends to it // to be dangerous. $dangerous = pht( "The change you're attempting to push splits the head of ". "branch '%s' into multiple heads: %s. This is inadvisable ". "and dangerous.", $ref, implode(', ', $readable_child_heads)); } else { // We're adding a second (or more) head to a branch. The new // head is not a descendant of any old head. $dangerous = pht( "The change you're attempting to push creates new, divergent ". "heads for the branch '%s': %s. This is inadvisable and ". "dangerous.", $ref, implode(', ', $readable_child_heads)); } } if ($deletes_existing_head) { // TODO: Somewhere in here we should be setting CHANGEFLAG_REWRITE // if we are also creating at least one other head to replace // this one. // NOTE: In Git, this is a dangerous change, but it is not dangerous // in Mercurial. Mercurial branches are version controlled, and // Mercurial does not prompt you for any special flags when pushing // a `--close-branch` commit by default. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH) ->setRefName($ref) ->setRefOld($old_head) ->setRefNew($new_head) ->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } $ref_updates[] = $ref_update; } } } return $ref_updates; } private function findMercurialPushKeyRefUpdates() { $key_namespace = getenv('HG_NAMESPACE'); if ($key_namespace === 'phases') { // Mercurial changes commit phases as part of normal push operations. We // just ignore these, as they don't seem to represent anything // interesting. return array(); } $key_name = getenv('HG_KEY'); $key_old = getenv('HG_OLD'); if (!strlen($key_old)) { $key_old = null; } $key_new = getenv('HG_NEW'); if (!strlen($key_new)) { $key_new = null; } if ($key_namespace !== 'bookmarks') { throw new Exception( pht( "Unknown Mercurial key namespace '%s', with key '%s' (%s -> %s). ". "Rejecting push.", $key_namespace, $key_name, coalesce($key_old, pht('null')), coalesce($key_new, pht('null')))); } if ($key_old === $key_new) { // We get a callback when the bookmark doesn't change. Just ignore this, // as it's a no-op. return array(); } $ref_flags = 0; $merge_base = null; if ($key_old === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($key_new === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else { list($merge_base_raw) = $this->getRepository()->execxLocalCommand( 'log --template %s --rev %s', '{node}', hgsprintf('ancestor(%s, %s)', $key_old, $key_new)); if (strlen(trim($merge_base_raw))) { $merge_base = trim($merge_base_raw); } if ($merge_base && ($merge_base === $key_old)) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; } } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BOOKMARK) ->setRefName($key_name) ->setRefOld(coalesce($key_old, self::EMPTY_HASH)) ->setRefNew(coalesce($key_new, self::EMPTY_HASH)) ->setChangeFlags($ref_flags); return array($ref_update); } private function findMercurialContentUpdates(array $ref_updates) { $content_updates = array(); foreach ($this->mercurialCommits as $commit => $branches) { $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } return $content_updates; } private function parseMercurialCommits($raw) { $commits_lines = explode("\2", $raw); $commits_lines = array_filter($commits_lines); $commit_map = array(); foreach ($commits_lines as $commit_line) { list($node, $branch) = explode("\1", $commit_line); $commit_map[$node] = array($branch); } return $commit_map; } private function parseMercurialHeads($raw) { $heads_map = $this->parseMercurialCommits($raw); $heads = array(); foreach ($heads_map as $commit => $branches) { foreach ($branches as $branch) { $heads[$branch][] = $commit; } } return $heads; } /* -( Subversion )--------------------------------------------------------- */ private function findSubversionRefUpdates() { // Subversion doesn't have any kind of mutable ref metadata. return array(); } private function findSubversionContentUpdates(array $ref_updates) { list($youngest) = execx( 'svnlook youngest %s', $this->subversionRepository); $ref_new = (int)$youngest + 1; $ref_flags = 0; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; $ref_content = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($ref_new) ->setChangeFlags($ref_flags); return array($ref_content); } /* -( Internals )---------------------------------------------------------- */ private function newPushLog() { // NOTE: We generate PHIDs up front so the Herald transcripts can pick them // up. $phid = id(new PhabricatorRepositoryPushLog())->generatePHID(); $device = AlmanacKeys::getLiveDevice(); if ($device) { $device_phid = $device->getPHID(); } else { $device_phid = null; } return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer()) ->setPHID($phid) ->setDevicePHID($device_phid) ->setRepositoryPHID($this->getRepository()->getPHID()) ->attachRepository($this->getRepository()) ->setEpoch(PhabricatorTime::getNow()); } private function newPushEvent() { $viewer = $this->getViewer(); $hook_start = $this->getStartTime(); - $hook_end = microtime(true); $event = PhabricatorRepositoryPushEvent::initializeNewEvent($viewer) ->setRepositoryPHID($this->getRepository()->getPHID()) ->setRemoteAddress($this->getRemoteAddress()) ->setRemoteProtocol($this->getRemoteProtocol()) ->setEpoch(PhabricatorTime::getNow()) - ->setHookWait((int)(1000000 * ($hook_end - $hook_start))); + ->setHookWait(phutil_microseconds_since($hook_start)); $identifier = $this->getRequestIdentifier(); if (strlen($identifier)) { $event->setRequestIdentifier($identifier); } return $event; } private function rejectEnormousChanges(array $content_updates) { $repository = $this->getRepository(); if ($repository->shouldAllowEnormousChanges()) { return; } // See T13142. Don't cache more than 64MB of changesets. For normal small // pushes, caching everything here can let us hit the cache from Herald if // we need to run content rules, which speeds things up a bit. For large // pushes, we may not be able to hold everything in memory. $cache_limit = 1024 * 1024 * 64; foreach ($content_updates as $update) { $identifier = $update->getRefNew(); try { $info = $this->loadChangesetsForCommit($identifier); list($changesets, $size) = $info; if ($this->changesetsSize + $size <= $cache_limit) { $this->changesets[$identifier] = $changesets; $this->changesetsSize += $size; } } catch (Exception $ex) { $this->changesets[$identifier] = $ex; $message = pht( 'ENORMOUS CHANGE'. "\n". 'Enormous change protection is enabled for this repository, but '. 'you are pushing an enormous change ("%s"). Edit the repository '. 'configuration before making enormous changes.'. "\n\n". "Content Exception: %s", $identifier, $ex->getMessage()); throw new DiffusionCommitHookRejectException($message); } } } private function loadChangesetsForCommit($identifier) { $byte_limit = HeraldCommitAdapter::getEnormousByteLimit(); $time_limit = HeraldCommitAdapter::getEnormousTimeLimit(); $vcs = $this->getRepository()->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // For git and hg, we can use normal commands. $drequest = DiffusionRequest::newFromDictionary( array( 'repository' => $this->getRepository(), 'user' => $this->getViewer(), 'commit' => $identifier, )); $raw_diff = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest) ->setTimeout($time_limit) ->setByteLimit($byte_limit) ->setLinesOfContext(0) ->executeInline(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // TODO: This diff has 3 lines of context, which produces slightly // incorrect "added file content" and "removed file content" results. // This may also choke on binaries, but "svnlook diff" does not support // the "--diff-cmd" flag. // For subversion, we need to use `svnlook`. $future = new ExecFuture( 'svnlook diff -t %s %s', $this->subversionTransaction, $this->subversionRepository); $future->setTimeout($time_limit); $future->setStdoutSizeLimit($byte_limit); $future->setStderrSizeLimit($byte_limit); list($raw_diff) = $future->resolvex(); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } if (strlen($raw_diff) >= $byte_limit) { throw new Exception( pht( 'The raw text of this change ("%s") is enormous (larger than %s '. 'bytes).', $identifier, new PhutilNumber($byte_limit))); } if (!strlen($raw_diff)) { // If the commit is actually empty, just return no changesets. return array(array(), 0); } $parser = new ArcanistDiffParser(); $changes = $parser->parseDiff($raw_diff); $diff = DifferentialDiff::newEphemeralFromRawChanges( $changes); $changesets = $diff->getChangesets(); $size = strlen($raw_diff); return array($changesets, $size); } public function getChangesetsForCommit($identifier) { if (isset($this->changesets[$identifier])) { $cached = $this->changesets[$identifier]; if ($cached instanceof Exception) { throw $cached; } return $cached; } $info = $this->loadChangesetsForCommit($identifier); list($changesets, $size) = $info; return $changesets; } public function loadCommitRefForCommit($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return id(new DiffusionLowLevelCommitQuery()) ->setRepository($repository) ->withIdentifier($identifier) ->execute(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // For subversion, we need to use `svnlook`. list($message) = execx( 'svnlook log -t %s %s', $this->subversionTransaction, $this->subversionRepository); return id(new DiffusionCommitRef()) ->setMessage($message); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } } public function loadBranches($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return idx($this->gitCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // NOTE: This will be "the branch the commit was made to", not // "a list of all branch heads which descend from the commit". // This is consistent with Mercurial, but possibly confusing. return idx($this->mercurialCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // Subversion doesn't have branches. return array(); } } private function loadCommitInfoForWorker(array $all_updates) { $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; $map = array(); foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $map[$update->getRefNew()] = array(); } foreach ($map as $identifier => $info) { $ref = $this->loadCommitRefForCommit($identifier); $map[$identifier] += array( 'summary' => $ref->getSummary(), 'branches' => $this->loadBranches($identifier), ); } return $map; } private function isInitialImport(array $all_updates) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // There is no meaningful way to import history into Subversion by // pushing. return false; default: break; } // Now, apply a heuristic to guess whether this is a normal commit or // an initial import. We guess something is an initial import if: // // - the repository is currently empty; and // - it pushes more than 7 commits at once. // // The number "7" is chosen arbitrarily as seeming reasonable. We could // also look at author data (do the commits come from multiple different // authors?) and commit date data (is the oldest commit more than 48 hours // old), but we don't have immediate access to those and this simple // heuristic might be good enough. $commit_count = 0; $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $commit_count++; } if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) { // If this pushes a very small number of commits, assume it's an // initial commit or stack of a few initial commits. return false; } $any_commits = id(new DiffusionCommitQuery()) ->setViewer($this->getViewer()) ->withRepository($repository) ->setLimit(1) ->execute(); if ($any_commits) { // If the repository already has commits, this isn't an import. return false; } return true; } } diff --git a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php index cac3351ef1..1de58f9240 100644 --- a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php +++ b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php @@ -1,885 +1,882 @@ repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setLog(DiffusionRepositoryClusterEngineLogInterface $log) { $this->logger = $log; return $this; } /* -( Cluster Synchronization )-------------------------------------------- */ /** * Synchronize repository version information after creating a repository. * * This initializes working copy versions for all currently bound devices to * 0, so that we don't get stuck making an ambiguous choice about which * devices are leaders when we later synchronize before a read. * * @task sync */ public function synchronizeWorkingCopyAfterCreation() { if (!$this->shouldEnableSynchronization(false)) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $bindings = $service->getActiveBindings(); foreach ($bindings as $binding) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $binding->getDevicePHID(), 0); } return $this; } /** * @task sync */ public function synchronizeWorkingCopyAfterHostingChange() { if (!$this->shouldEnableSynchronization(false)) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); // After converting a hosted repository to observed, or vice versa, we // need to reset version numbers because the clocks for observed and hosted // repositories run on different units. // We identify all the cluster leaders and reset their version to 0. // We identify all the cluster followers and demote them. // This allows the cluster to start over again at version 0 but keep the // same leaders. if ($versions) { $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); foreach ($versions as $version) { $device_phid = $version->getDevicePHID(); if ($version->getRepositoryVersion() == $max_version) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, 0); } else { PhabricatorRepositoryWorkingCopyVersion::demoteDevice( $repository_phid, $device_phid); } } } return $this; } /** * @task sync */ public function synchronizeWorkingCopyBeforeRead() { if (!$this->shouldEnableSynchronization(true)) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); $read_lock = PhabricatorRepositoryWorkingCopyVersion::getReadLock( $repository_phid, $device_phid); $lock_wait = phutil_units('2 minutes in seconds'); $this->logLine( pht( 'Acquiring read lock for repository "%s" on device "%s"...', $repository->getDisplayName(), $device->getName())); try { $start = PhabricatorTime::getNow(); $read_lock->lock($lock_wait); $waited = (PhabricatorTime::getNow() - $start); if ($waited) { $this->logLine( pht( 'Acquired read lock after %s second(s).', new PhutilNumber($waited))); } else { $this->logLine( pht( 'Acquired read lock immediately.')); } } catch (PhutilLockException $ex) { throw new PhutilProxyException( pht( 'Failed to acquire read lock after waiting %s second(s). You '. 'may be able to retry later. (%s)', new PhutilNumber($lock_wait), $ex->getHint()), $ex); } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); $this_version = idx($versions, $device_phid); if ($this_version) { $this_version = (int)$this_version->getRepositoryVersion(); } else { $this_version = -1; } if ($versions) { // This is the normal case, where we have some version information and // can identify which nodes are leaders. If the current node is not a // leader, we want to fetch from a leader and then update our version. $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); if ($max_version > $this_version) { if ($repository->isHosted()) { $fetchable = array(); foreach ($versions as $version) { if ($version->getRepositoryVersion() == $max_version) { $fetchable[] = $version->getDevicePHID(); } } $this->synchronizeWorkingCopyFromDevices( $fetchable, $this_version, $max_version); } else { $this->synchronizeWorkingCopyFromRemote(); } PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, $max_version); } else { $this->logLine( pht( 'Device "%s" is already a cluster leader and does not need '. 'to be synchronized.', $device->getName())); } $result_version = $max_version; } else { // If no version records exist yet, we need to be careful, because we // can not tell which nodes are leaders. // There might be several nodes with arbitrary existing data, and we have // no way to tell which one has the "right" data. If we pick wrong, we // might erase some or all of the data in the repository. // Since this is dangerous, we refuse to guess unless there is only one // device. If we're the only device in the group, we obviously must be // a leader. $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $bindings = $service->getActiveBindings(); $device_map = array(); foreach ($bindings as $binding) { $device_map[$binding->getDevicePHID()] = true; } if (count($device_map) > 1) { throw new Exception( pht( 'Repository "%s" exists on more than one device, but no device '. 'has any repository version information. Phabricator can not '. 'guess which copy of the existing data is authoritative. Promote '. 'a device or see "Ambiguous Leaders" in the documentation.', $repository->getDisplayName())); } if (empty($device_map[$device->getPHID()])) { throw new Exception( pht( 'Repository "%s" is being synchronized on device "%s", but '. 'this device is not bound to the corresponding cluster '. 'service ("%s").', $repository->getDisplayName(), $device->getName(), $service->getName())); } // The current device is the only device in service, so it must be a // leader. We can safely have any future nodes which come online read // from it. PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, 0); $result_version = 0; } $read_lock->unlock(); return $result_version; } /** * @task sync */ public function synchronizeWorkingCopyBeforeWrite() { if (!$this->shouldEnableSynchronization(true)) { return; } $repository = $this->getRepository(); $viewer = $this->getViewer(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); $table = new PhabricatorRepositoryWorkingCopyVersion(); $locked_connection = $table->establishConnection('w'); $write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock( $repository_phid); $write_lock->useSpecificConnection($locked_connection); $this->logLine( pht( 'Acquiring write lock for repository "%s"...', $repository->getDisplayName())); $lock_wait = phutil_units('2 minutes in seconds'); try { $write_wait_start = microtime(true); $start = PhabricatorTime::getNow(); $step_wait = 1; while (true) { try { $write_lock->lock((int)floor($step_wait)); $write_wait_end = microtime(true); break; } catch (PhutilLockException $ex) { $waited = (PhabricatorTime::getNow() - $start); if ($waited > $lock_wait) { throw $ex; } $this->logActiveWriter($viewer, $repository); } // Wait a little longer before the next message we print. $step_wait = $step_wait + 0.5; $step_wait = min($step_wait, 3); } $waited = (PhabricatorTime::getNow() - $start); if ($waited) { $this->logLine( pht( 'Acquired write lock after %s second(s).', new PhutilNumber($waited))); } else { $this->logLine( pht( 'Acquired write lock immediately.')); } } catch (PhutilLockException $ex) { throw new PhutilProxyException( pht( 'Failed to acquire write lock after waiting %s second(s). You '. 'may be able to retry later. (%s)', new PhutilNumber($lock_wait), $ex->getHint()), $ex); } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); foreach ($versions as $version) { if (!$version->getIsWriting()) { continue; } throw new Exception( pht( 'An previous write to this repository was interrupted; refusing '. 'new writes. This issue requires operator intervention to resolve, '. 'see "Write Interruptions" in the "Cluster: Repositories" in the '. 'documentation for instructions.')); } $read_wait_start = microtime(true); try { $max_version = $this->synchronizeWorkingCopyBeforeRead(); } catch (Exception $ex) { $write_lock->unlock(); throw $ex; } $read_wait_end = microtime(true); $pid = getmypid(); $hash = Filesystem::readRandomCharacters(12); $this->clusterWriteOwner = "{$pid}.{$hash}"; PhabricatorRepositoryWorkingCopyVersion::willWrite( $locked_connection, $repository_phid, $device_phid, array( 'userPHID' => $viewer->getPHID(), 'epoch' => PhabricatorTime::getNow(), 'devicePHID' => $device_phid, ), $this->clusterWriteOwner); $this->clusterWriteVersion = $max_version; $this->clusterWriteLock = $write_lock; $write_wait = ($write_wait_end - $write_wait_start); $read_wait = ($read_wait_end - $read_wait_start); $log = $this->logger; if ($log) { $log->writeClusterEngineLogProperty('readWait', $read_wait); $log->writeClusterEngineLogProperty('writeWait', $write_wait); } } public function synchronizeWorkingCopyAfterDiscovery($new_version) { if (!$this->shouldEnableSynchronization(true)) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); if ($repository->isHosted()) { return; } $viewer = $this->getViewer(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); // NOTE: We are not holding a lock here because this method is only called // from PhabricatorRepositoryDiscoveryEngine, which already holds a device // lock. Even if we do race here and record an older version, the // consequences are mild: we only do extra work to correct it later. $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); $this_version = idx($versions, $device_phid); if ($this_version) { $this_version = (int)$this_version->getRepositoryVersion(); } else { $this_version = -1; } if ($new_version > $this_version) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, $new_version); } } /** * @task sync */ public function synchronizeWorkingCopyAfterWrite() { if (!$this->shouldEnableSynchronization(true)) { return; } if (!$this->clusterWriteLock) { throw new Exception( pht( 'Trying to synchronize after write, but not holding a write '. 'lock!')); } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); // It is possible that we've lost the global lock while receiving the push. // For example, the master database may have been restarted between the // time we acquired the global lock and now, when the push has finished. // We wrote a durable lock while we were holding the the global lock, // essentially upgrading our lock. We can still safely release this upgraded // lock even if we're no longer holding the global lock. // If we fail to release the lock, the repository will be frozen until // an operator can figure out what happened, so we try pretty hard to // reconnect to the database and release the lock. $now = PhabricatorTime::getNow(); $duration = phutil_units('5 minutes in seconds'); $try_until = $now + $duration; $did_release = false; $already_failed = false; while (PhabricatorTime::getNow() <= $try_until) { try { // NOTE: This means we're still bumping the version when pushes fail. We // could select only un-rejected events instead to bump a little less // often. $new_log = id(new PhabricatorRepositoryPushEventQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withRepositoryPHIDs(array($repository_phid)) ->setLimit(1) ->executeOne(); $old_version = $this->clusterWriteVersion; if ($new_log) { $new_version = $new_log->getID(); } else { $new_version = $old_version; } PhabricatorRepositoryWorkingCopyVersion::didWrite( $repository_phid, $device_phid, $this->clusterWriteVersion, $new_version, $this->clusterWriteOwner); $did_release = true; break; } catch (AphrontConnectionQueryException $ex) { $connection_exception = $ex; } catch (AphrontConnectionLostQueryException $ex) { $connection_exception = $ex; } if (!$already_failed) { $already_failed = true; $this->logLine( pht('CRITICAL. Failed to release cluster write lock!')); $this->logLine( pht( 'The connection to the master database was lost while receiving '. 'the write.')); $this->logLine( pht( 'This process will spend %s more second(s) attempting to '. 'recover, then give up.', new PhutilNumber($duration))); } sleep(1); } if ($did_release) { if ($already_failed) { $this->logLine( pht('RECOVERED. Link to master database was restored.')); } $this->logLine(pht('Released cluster write lock.')); } else { throw new Exception( pht( 'Failed to reconnect to master database and release held write '. 'lock ("%s") on device "%s" for repository "%s" after trying '. 'for %s seconds(s). This repository will be frozen.', $this->clusterWriteOwner, $device->getName(), $this->getDisplayName(), new PhutilNumber($duration))); } // We can continue even if we've lost this lock, everything is still // consistent. try { $this->clusterWriteLock->unlock(); } catch (Exception $ex) { // Ignore. } $this->clusterWriteLock = null; $this->clusterWriteOwner = null; } /* -( Internals )---------------------------------------------------------- */ /** * @task internal */ private function shouldEnableSynchronization($require_device) { $repository = $this->getRepository(); $service_phid = $repository->getAlmanacServicePHID(); if (!$service_phid) { return false; } if (!$repository->supportsSynchronization()) { return false; } if ($require_device) { $device = AlmanacKeys::getLiveDevice(); if (!$device) { return false; } } return true; } /** * @task internal */ private function synchronizeWorkingCopyFromRemote() { $repository = $this->getRepository(); $device = AlmanacKeys::getLiveDevice(); $local_path = $repository->getLocalPath(); $fetch_uri = $repository->getRemoteURIEnvelope(); if ($repository->isGit()) { $this->requireWorkingCopy(); $argv = array( 'fetch --prune -- %P %s', $fetch_uri, '+refs/*:refs/*', ); } else { throw new Exception(pht('Remote sync only supported for git!')); } $future = DiffusionCommandEngine::newCommandEngine($repository) ->setArgv($argv) ->setSudoAsDaemon(true) ->setCredentialPHID($repository->getCredentialPHID()) ->setURI($repository->getRemoteURIObject()) ->newFuture(); $future->setCWD($local_path); try { $future->resolvex(); } catch (Exception $ex) { $this->logLine( pht( 'Synchronization of "%s" from remote failed: %s', $device->getName(), $ex->getMessage())); throw $ex; } } /** * @task internal */ private function synchronizeWorkingCopyFromDevices( array $device_phids, $local_version, $remote_version) { $repository = $this->getRepository(); $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $device_map = array_fuse($device_phids); $bindings = $service->getActiveBindings(); $fetchable = array(); foreach ($bindings as $binding) { // We can't fetch from nodes which don't have the newest version. $device_phid = $binding->getDevicePHID(); if (empty($device_map[$device_phid])) { continue; } // TODO: For now, only fetch over SSH. We could support fetching over // HTTP eventually. if ($binding->getAlmanacPropertyValue('protocol') != 'ssh') { continue; } $fetchable[] = $binding; } if (!$fetchable) { throw new Exception( pht( 'Leader lost: no up-to-date nodes in repository cluster are '. 'fetchable.')); } // If we can synchronize from multiple sources, choose one at random. shuffle($fetchable); $caught = null; foreach ($fetchable as $binding) { try { $this->synchronizeWorkingCopyFromBinding( $binding, $local_version, $remote_version); $caught = null; break; } catch (Exception $ex) { $caught = $ex; } } if ($caught) { throw $caught; } } /** * @task internal */ private function synchronizeWorkingCopyFromBinding( AlmanacBinding $binding, $local_version, $remote_version) { $repository = $this->getRepository(); $device = AlmanacKeys::getLiveDevice(); $this->logLine( pht( 'Synchronizing this device ("%s") from cluster leader ("%s").', $device->getName(), $binding->getDevice()->getName())); $fetch_uri = $repository->getClusterRepositoryURIFromBinding($binding); $local_path = $repository->getLocalPath(); if ($repository->isGit()) { $this->requireWorkingCopy(); $argv = array( 'fetch --prune -- %s %s', $fetch_uri, '+refs/*:refs/*', ); } else { throw new Exception(pht('Binding sync only supported for git!')); } $future = DiffusionCommandEngine::newCommandEngine($repository) ->setArgv($argv) ->setConnectAsDevice(true) ->setSudoAsDaemon(true) ->setURI($fetch_uri) ->newFuture(); $future->setCWD($local_path); $log = PhabricatorRepositorySyncEvent::initializeNewEvent() ->setRepositoryPHID($repository->getPHID()) ->setEpoch(PhabricatorTime::getNow()) ->setDevicePHID($device->getPHID()) ->setFromDevicePHID($binding->getDevice()->getPHID()) ->setDeviceVersion($local_version) ->setFromDeviceVersion($remote_version); $sync_start = microtime(true); try { $future->resolvex(); } catch (Exception $ex) { - $sync_end = microtime(true); - $log->setSyncWait((int)(1000000 * ($sync_end - $sync_start))); + $log->setSyncWait(phutil_microseconds_since($sync_start)); if ($ex instanceof CommandException) { if ($future->getWasKilledByTimeout()) { $result_type = PhabricatorRepositorySyncEvent::RESULT_TIMEOUT; } else { $result_type = PhabricatorRepositorySyncEvent::RESULT_ERROR; } $log ->setResultCode($ex->getError()) ->setResultType($result_type) ->setProperty('stdout', $ex->getStdout()) ->setProperty('stderr', $ex->getStderr()); } else { $log ->setResultCode(1) ->setResultType(PhabricatorRepositorySyncEvent::RESULT_EXCEPTION) ->setProperty('message', $ex->getMessage()); } $log->save(); $this->logLine( pht( 'Synchronization of "%s" from leader "%s" failed: %s', $device->getName(), $binding->getDevice()->getName(), $ex->getMessage())); throw $ex; } - $sync_end = microtime(true); - $log - ->setSyncWait((int)(1000000 * ($sync_end - $sync_start))) + ->setSyncWait(phutil_microseconds_since($sync_start)) ->setResultCode(0) ->setResultType(PhabricatorRepositorySyncEvent::RESULT_SYNC) ->save(); } /** * @task internal */ private function logLine($message) { return $this->logText("# {$message}\n"); } /** * @task internal */ private function logText($message) { $log = $this->logger; if ($log) { $log->writeClusterEngineLogMessage($message); } return $this; } private function requireWorkingCopy() { $repository = $this->getRepository(); $local_path = $repository->getLocalPath(); if (!Filesystem::pathExists($local_path)) { $device = AlmanacKeys::getLiveDevice(); throw new Exception( pht( 'Repository "%s" does not have a working copy on this device '. 'yet, so it can not be synchronized. Wait for the daemons to '. 'construct one or run `bin/repository update %s` on this host '. '("%s") to build it explicitly.', $repository->getDisplayName(), $repository->getMonogram(), $device->getName())); } } private function logActiveWriter( PhabricatorUser $viewer, PhabricatorRepository $repository) { $writer = PhabricatorRepositoryWorkingCopyVersion::loadWriter( $repository->getPHID()); if (!$writer) { $this->logLine(pht('Waiting on another user to finish writing...')); return; } $user_phid = $writer->getWriteProperty('userPHID'); $device_phid = $writer->getWriteProperty('devicePHID'); $epoch = $writer->getWriteProperty('epoch'); $phids = array($user_phid, $device_phid); $handles = $viewer->loadHandles($phids); $duration = (PhabricatorTime::getNow() - $epoch) + 1; $this->logLine( pht( 'Waiting for %s to finish writing (on device "%s" for %ss)...', $handles[$user_phid]->getName(), $handles[$device_phid]->getName(), new PhutilNumber($duration))); } } diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php index e1a6ef1500..ed1eeaea63 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php @@ -1,229 +1,228 @@ self::IDS_COUNTER, self::CONFIG_TIMESTAMPS => false, self::CONFIG_KEY_SCHEMA => array( 'dataID' => array( 'columns' => array('dataID'), 'unique' => true, ), 'taskClass' => array( 'columns' => array('taskClass'), ), 'leaseExpires' => array( 'columns' => array('leaseExpires'), ), 'leaseOwner' => array( 'columns' => array('leaseOwner(16)'), ), 'key_failuretime' => array( 'columns' => array('failureTime'), ), 'leaseOwner_2' => array( 'columns' => array('leaseOwner', 'priority', 'id'), ), ) + $parent[self::CONFIG_KEY_SCHEMA], ); $config[self::CONFIG_COLUMN_SCHEMA] = array( // T6203/NULLABILITY // This isn't nullable in the archive table, so at a minimum these // should be the same. 'dataID' => 'uint32?', ) + $parent[self::CONFIG_COLUMN_SCHEMA]; return $config + $parent; } public function setServerTime($server_time) { $this->serverTime = $server_time; $this->localTime = time(); return $this; } public function setLeaseDuration($lease_duration) { $this->checkLease(); $server_lease_expires = $this->serverTime + $lease_duration; $this->setLeaseExpires($server_lease_expires); // NOTE: This is primarily to allow unit tests to set negative lease // durations so they don't have to wait around for leases to expire. We // check that the lease is valid above. return $this->forceSaveWithoutLease(); } public function save() { $this->checkLease(); return $this->forceSaveWithoutLease(); } public function forceSaveWithoutLease() { $is_new = !$this->getID(); if ($is_new) { $this->failureCount = 0; } if ($is_new && ($this->getData() !== null)) { $data = new PhabricatorWorkerTaskData(); $data->setData($this->getData()); $data->save(); $this->setDataID($data->getID()); } return parent::save(); } protected function checkLease() { $owner = $this->leaseOwner; if (!$owner) { return; } if ($owner == PhabricatorWorker::YIELD_OWNER) { return; } $current_server_time = $this->serverTime + (time() - $this->localTime); if ($current_server_time >= $this->leaseExpires) { throw new Exception( pht( 'Trying to update Task %d (%s) after lease expiration!', $this->getID(), $this->getTaskClass())); } } public function delete() { throw new Exception( pht( 'Active tasks can not be deleted directly. '. 'Use %s to move tasks to the archive.', 'archiveTask()')); } public function archiveTask($result, $duration) { if ($this->getID() === null) { throw new Exception( pht("Attempting to archive a task which hasn't been saved!")); } $this->checkLease(); $archive = id(new PhabricatorWorkerArchiveTask()) ->setID($this->getID()) ->setTaskClass($this->getTaskClass()) ->setLeaseOwner($this->getLeaseOwner()) ->setLeaseExpires($this->getLeaseExpires()) ->setFailureCount($this->getFailureCount()) ->setDataID($this->getDataID()) ->setPriority($this->getPriority()) ->setObjectPHID($this->getObjectPHID()) ->setResult($result) ->setDuration($duration); // NOTE: This deletes the active task (this object)! $archive->save(); return $archive; } public function executeTask() { // We do this outside of the try .. catch because we don't have permission // to release the lease otherwise. $this->checkLease(); $did_succeed = false; $worker = null; try { $worker = $this->getWorkerInstance(); $worker->setCurrentWorkerTask($this); $maximum_failures = $worker->getMaximumRetryCount(); if ($maximum_failures !== null) { if ($this->getFailureCount() > $maximum_failures) { throw new PhabricatorWorkerPermanentFailureException( pht( 'Task %d has exceeded the maximum number of failures (%d).', $this->getID(), $maximum_failures)); } } $lease = $worker->getRequiredLeaseTime(); if ($lease !== null) { $this->setLeaseDuration($lease); } $t_start = microtime(true); $worker->executeTask(); - $t_end = microtime(true); - $duration = (int)(1000000 * ($t_end - $t_start)); + $duration = phutil_microseconds_since($t_start); $result = $this->archiveTask( PhabricatorWorkerArchiveTask::RESULT_SUCCESS, $duration); $did_succeed = true; } catch (PhabricatorWorkerPermanentFailureException $ex) { $result = $this->archiveTask( PhabricatorWorkerArchiveTask::RESULT_FAILURE, 0); $result->setExecutionException($ex); } catch (PhabricatorWorkerYieldException $ex) { $this->setExecutionException($ex); $this->setLeaseOwner(PhabricatorWorker::YIELD_OWNER); $retry = $ex->getDuration(); $retry = max($retry, 5); // NOTE: As a side effect, this saves the object. $this->setLeaseDuration($retry); $result = $this; } catch (Exception $ex) { $this->setExecutionException($ex); $this->setFailureCount($this->getFailureCount() + 1); $this->setFailureTime(time()); $retry = null; if ($worker) { $retry = $worker->getWaitBeforeRetry($this); } $retry = coalesce( $retry, PhabricatorWorkerLeaseQuery::getDefaultWaitBeforeRetry()); // NOTE: As a side effect, this saves the object. $this->setLeaseDuration($retry); $result = $this; } // NOTE: If this throws, we don't want it to cause the task to fail again, // so execute it out here and just let the exception escape. if ($did_succeed) { // Default the new task priority to our own priority. $defaults = array( 'priority' => (int)$this->getPriority(), ); $worker->flushTaskQueue($defaults); } return $result; } } diff --git a/support/startup/PhabricatorStartup.php b/support/startup/PhabricatorStartup.php index 212b057376..1bfb74d886 100644 --- a/support/startup/PhabricatorStartup.php +++ b/support/startup/PhabricatorStartup.php @@ -1,826 +1,828 @@ setEncoding($encoding); } $input = ''; do { $bytes = $stream->readData(); if ($bytes === null) { break; } $input .= $bytes; } while (true); self::$rawInput = $input; } return self::$rawInput; } /* -( Startup Hooks )------------------------------------------------------ */ /** * @param float Request start time, from `microtime(true)`. * @task hook */ public static function didStartup($start_time) { self::$startTime = $start_time; self::$phases = array(); self::$accessLog = null; static $registered; if (!$registered) { // NOTE: This protects us against multiple calls to didStartup() in the // same request, but also against repeated requests to the same // interpreter state, which we may implement in the future. register_shutdown_function(array(__CLASS__, 'didShutdown')); $registered = true; } self::setupPHP(); self::verifyPHP(); // If we've made it this far, the environment isn't completely broken so // we can switch over to relying on our own exception recovery mechanisms. ini_set('display_errors', 0); self::connectRateLimits(); self::normalizeInput(); self::verifyRewriteRules(); self::detectPostMaxSizeTriggered(); self::beginOutputCapture(); } /** * @task hook */ public static function didShutdown() { // Disconnect any active rate limits before we shut down. If we don't do // this, requests which exit early will lock a slot in any active // connection limits, and won't count for rate limits. self::disconnectRateLimits(array()); $event = error_get_last(); if (!$event) { return; } switch ($event['type']) { case E_ERROR: case E_PARSE: case E_COMPILE_ERROR: break; default: return; } $msg = ">>> UNRECOVERABLE FATAL ERROR <<<\n\n"; if ($event) { // Even though we should be emitting this as text-plain, escape things // just to be sure since we can't really be sure what the program state // is when we get here. $msg .= htmlspecialchars( $event['message']."\n\n".$event['file'].':'.$event['line'], ENT_QUOTES, 'UTF-8'); } // flip dem tables $msg .= "\n\n\n"; $msg .= "\xe2\x94\xbb\xe2\x94\x81\xe2\x94\xbb\x20\xef\xb8\xb5\x20\xc2\xaf". "\x5c\x5f\x28\xe3\x83\x84\x29\x5f\x2f\xc2\xaf\x20\xef\xb8\xb5\x20". "\xe2\x94\xbb\xe2\x94\x81\xe2\x94\xbb"; self::didFatal($msg); } public static function loadCoreLibraries() { $phabricator_root = dirname(dirname(dirname(__FILE__))); $libraries_root = dirname($phabricator_root); $root = null; if (!empty($_SERVER['PHUTIL_LIBRARY_ROOT'])) { $root = $_SERVER['PHUTIL_LIBRARY_ROOT']; } ini_set( 'include_path', $libraries_root.PATH_SEPARATOR.ini_get('include_path')); @include_once $root.'libphutil/src/__phutil_library_init__.php'; if (!@constant('__LIBPHUTIL__')) { self::didFatal( "Unable to load libphutil. Put libphutil/ next to phabricator/, or ". "update your PHP 'include_path' to include the parent directory of ". "libphutil/."); } phutil_load_library('arcanist/src'); // Load Phabricator itself using the absolute path, so we never end up doing // anything surprising (loading index.php and libraries from different // directories). phutil_load_library($phabricator_root.'/src'); } /* -( Output Capture )----------------------------------------------------- */ public static function beginOutputCapture() { if (self::$capturingOutput) { self::didFatal('Already capturing output!'); } self::$capturingOutput = true; ob_start(); } public static function endOutputCapture() { if (!self::$capturingOutput) { return null; } self::$capturingOutput = false; return ob_get_clean(); } /* -( Debug Time Limit )--------------------------------------------------- */ /** * Set a time limit (in seconds) for the current script. After time expires, * the script fatals. * * This works like `max_execution_time`, but prints out a useful stack trace * when the time limit expires. This is primarily intended to make it easier * to debug pages which hang by allowing extraction of a stack trace: set a * short debug limit, then use the trace to figure out what's happening. * * The limit is implemented with a tick function, so enabling it implies * some accounting overhead. * * @param int Time limit in seconds. * @return void */ public static function setDebugTimeLimit($limit) { self::$debugTimeLimit = $limit; static $initialized; if (!$initialized) { declare(ticks=1); register_tick_function(array(__CLASS__, 'onDebugTick')); } } /** * Callback tick function used by @{method:setDebugTimeLimit}. * * Fatals with a useful stack trace after the time limit expires. * * @return void */ public static function onDebugTick() { $limit = self::$debugTimeLimit; if (!$limit) { return; } $elapsed = (microtime(true) - self::getStartTime()); if ($elapsed > $limit) { $frames = array(); foreach (debug_backtrace() as $frame) { $file = isset($frame['file']) ? $frame['file'] : '-'; $file = basename($file); $line = isset($frame['line']) ? $frame['line'] : '-'; $class = isset($frame['class']) ? $frame['class'].'->' : null; $func = isset($frame['function']) ? $frame['function'].'()' : '?'; $frames[] = "{$file}:{$line} {$class}{$func}"; } self::didFatal( "Request aborted by debug time limit after {$limit} seconds.\n\n". "STACK TRACE\n". implode("\n", $frames)); } } /* -( In Case of Apocalypse )---------------------------------------------- */ /** * Fatal the request completely in response to an exception, sending a plain * text message to the client. Calls @{method:didFatal} internally. * * @param string Brief description of the exception context, like * `"Rendering Exception"`. * @param Exception The exception itself. * @param bool True if it's okay to show the exception's stack trace * to the user. The trace will always be logged. * @return exit This method **does not return**. * * @task apocalypse */ public static function didEncounterFatalException( $note, Exception $ex, $show_trace) { $message = '['.$note.'/'.get_class($ex).'] '.$ex->getMessage(); $full_message = $message; $full_message .= "\n\n"; $full_message .= $ex->getTraceAsString(); if ($show_trace) { $message = $full_message; } self::didFatal($message, $full_message); } /** * Fatal the request completely, sending a plain text message to the client. * * @param string Plain text message to send to the client. * @param string Plain text message to send to the error log. If not * provided, the client message is used. You can pass a more * detailed message here (e.g., with stack traces) to avoid * showing it to users. * @return exit This method **does not return**. * * @task apocalypse */ public static function didFatal($message, $log_message = null) { if ($log_message === null) { $log_message = $message; } self::endOutputCapture(); $access_log = self::$accessLog; if ($access_log) { // We may end up here before the access log is initialized, e.g. from // verifyPHP(). $access_log->setData( array( 'c' => 500, )); $access_log->write(); } header( 'Content-Type: text/plain; charset=utf-8', $replace = true, $http_error = 500); error_log($log_message); echo $message."\n"; exit(1); } /* -( Validation )--------------------------------------------------------- */ /** * @task validation */ private static function setupPHP() { error_reporting(E_ALL | E_STRICT); self::$oldMemoryLimit = ini_get('memory_limit'); ini_set('memory_limit', -1); // If we have libxml, disable the incredibly dangerous entity loader. if (function_exists('libxml_disable_entity_loader')) { libxml_disable_entity_loader(true); } // See T13060. If the locale for this process (the parent process) is not // a UTF-8 locale we can encounter problems when launching subprocesses // which receive UTF-8 parameters in their command line argument list. @setlocale(LC_ALL, 'en_US.UTF-8'); } /** * @task validation */ public static function getOldMemoryLimit() { return self::$oldMemoryLimit; } /** * @task validation */ private static function normalizeInput() { // Replace superglobals with unfiltered versions, disrespect php.ini (we // filter ourselves). // NOTE: We don't filter INPUT_SERVER because we don't want to overwrite // changes made in "preamble.php". // NOTE: WE don't filter INPUT_POST because we may be constructing it // lazily if "enable_post_data_reading" is disabled. $filter = array( INPUT_GET, INPUT_ENV, INPUT_COOKIE, ); foreach ($filter as $type) { $filtered = filter_input_array($type, FILTER_UNSAFE_RAW); if (!is_array($filtered)) { continue; } switch ($type) { case INPUT_GET: $_GET = array_merge($_GET, $filtered); break; case INPUT_COOKIE: $_COOKIE = array_merge($_COOKIE, $filtered); break; case INPUT_ENV; $env = array_merge($_ENV, $filtered); $_ENV = self::filterEnvSuperglobal($env); break; } } self::rebuildRequest(); } /** * @task validation */ public static function rebuildRequest() { // Rebuild $_REQUEST, respecting order declared in ".ini" files. $order = ini_get('request_order'); if (!$order) { $order = ini_get('variables_order'); } if (!$order) { // $_REQUEST will be empty, so leave it alone. return; } $_REQUEST = array(); for ($ii = 0; $ii < strlen($order); $ii++) { switch ($order[$ii]) { case 'G': $_REQUEST = array_merge($_REQUEST, $_GET); break; case 'P': $_REQUEST = array_merge($_REQUEST, $_POST); break; case 'C': $_REQUEST = array_merge($_REQUEST, $_COOKIE); break; default: // $_ENV and $_SERVER never go into $_REQUEST. break; } } } /** * Adjust `$_ENV` before execution. * * Adjustments here primarily impact the environment as seen by subprocesses. * The environment is forwarded explicitly by @{class:ExecFuture}. * * @param map Input `$_ENV`. * @return map Suitable `$_ENV`. * @task validation */ private static function filterEnvSuperglobal(array $env) { // In some configurations, we may get "argc" and "argv" set in $_ENV. // These are not real environmental variables, and "argv" may have an array // value which can not be forwarded to subprocesses. Remove these from the // environment if they are present. unset($env['argc']); unset($env['argv']); return $env; } /** * @task validation */ private static function verifyPHP() { $required_version = '5.2.3'; if (version_compare(PHP_VERSION, $required_version) < 0) { self::didFatal( "You are running PHP version '".PHP_VERSION."', which is older than ". "the minimum version, '{$required_version}'. Update to at least ". "'{$required_version}'."); } if (get_magic_quotes_gpc()) { self::didFatal( "Your server is configured with PHP 'magic_quotes_gpc' enabled. This ". "feature is 'highly discouraged' by PHP's developers and you must ". "disable it to run Phabricator. Consult the PHP manual for ". "instructions."); } if (extension_loaded('apc')) { $apc_version = phpversion('apc'); $known_bad = array( '3.1.14' => true, '3.1.15' => true, '3.1.15-dev' => true, ); if (isset($known_bad[$apc_version])) { self::didFatal( "You have APC {$apc_version} installed. This version of APC is ". "known to be bad, and does not work with Phabricator (it will ". "cause Phabricator to fatal unrecoverably with nonsense errors). ". "Downgrade to version 3.1.13."); } } if (isset($_SERVER['HTTP_PROXY'])) { self::didFatal( 'This HTTP request included a "Proxy:" header, poisoning the '. 'environment (CVE-2016-5385 / httpoxy). Declining to process this '. 'request. For details, see: https://phurl.io/u/httpoxy'); } } /** * @task validation */ private static function verifyRewriteRules() { if (isset($_REQUEST['__path__']) && strlen($_REQUEST['__path__'])) { return; } if (php_sapi_name() == 'cli-server') { // Compatibility with PHP 5.4+ built-in web server. $url = parse_url($_SERVER['REQUEST_URI']); $_REQUEST['__path__'] = $url['path']; return; } if (!isset($_REQUEST['__path__'])) { self::didFatal( "Request parameter '__path__' is not set. Your rewrite rules ". "are not configured correctly."); } if (!strlen($_REQUEST['__path__'])) { self::didFatal( "Request parameter '__path__' is set, but empty. Your rewrite rules ". "are not configured correctly. The '__path__' should always ". "begin with a '/'."); } } /** * Detect if this request has had its POST data stripped by exceeding the * 'post_max_size' PHP configuration limit. * * PHP has a setting called 'post_max_size'. If a POST request arrives with * a body larger than the limit, PHP doesn't generate $_POST but processes * the request anyway, and provides no formal way to detect that this * happened. * * We can still read the entire body out of `php://input`. However according * to the documentation the stream isn't available for "multipart/form-data" * (on nginx + php-fpm it appears that it is available, though, at least) so * any attempt to generate $_POST would be fragile. * * @task validation */ private static function detectPostMaxSizeTriggered() { // If this wasn't a POST, we're fine. if ($_SERVER['REQUEST_METHOD'] != 'POST') { return; } // If "enable_post_data_reading" is off, we won't have $_POST and this // condition is effectively impossible. if (!ini_get('enable_post_data_reading')) { return; } // If there's POST data, clearly we're in good shape. if ($_POST) { return; } // For HTML5 drag-and-drop file uploads, Safari submits the data as // "application/x-www-form-urlencoded". For most files this generates // something in POST because most files decode to some nonempty (albeit // meaningless) value. However, some files (particularly small images) // don't decode to anything. If we know this is a drag-and-drop upload, // we can skip this check. if (isset($_REQUEST['__upload__'])) { return; } // PHP generates $_POST only for two content types. This routing happens // in `main/php_content_types.c` in PHP. Normally, all forms use one of // these content types, but some requests may not -- for example, Firefox // submits files sent over HTML5 XMLHTTPRequest APIs with the Content-Type // of the file itself. If we don't have a recognized content type, we // don't need $_POST. // // NOTE: We use strncmp() because the actual content type may be something // like "multipart/form-data; boundary=...". // // NOTE: Chrome sometimes omits this header, see some discussion in T1762 // and http://code.google.com/p/chromium/issues/detail?id=6800 $content_type = isset($_SERVER['CONTENT_TYPE']) ? $_SERVER['CONTENT_TYPE'] : ''; $parsed_types = array( 'application/x-www-form-urlencoded', 'multipart/form-data', ); $is_parsed_type = false; foreach ($parsed_types as $parsed_type) { if (strncmp($content_type, $parsed_type, strlen($parsed_type)) === 0) { $is_parsed_type = true; break; } } if (!$is_parsed_type) { return; } // Check for 'Content-Length'. If there's no data, we don't expect $_POST // to exist. $length = (int)$_SERVER['CONTENT_LENGTH']; if (!$length) { return; } // Time to fatal: we know this was a POST with data that should have been // populated into $_POST, but it wasn't. $config = ini_get('post_max_size'); self::didFatal( "As received by the server, this request had a nonzero content length ". "but no POST data.\n\n". "Normally, this indicates that it exceeds the 'post_max_size' setting ". "in the PHP configuration on the server. Increase the 'post_max_size' ". "setting or reduce the size of the request.\n\n". "Request size according to 'Content-Length' was '{$length}', ". "'post_max_size' is set to '{$config}'."); } /* -( Rate Limiting )------------------------------------------------------ */ /** * Add a new client limits. * * @param PhabricatorClientLimit New limit. * @return PhabricatorClientLimit The limit. */ public static function addRateLimit(PhabricatorClientLimit $limit) { self::$limits[] = $limit; return $limit; } /** * Apply configured rate limits. * * If any limit is exceeded, this method terminates the request. * * @return void * @task ratelimit */ private static function connectRateLimits() { $limits = self::$limits; $reason = null; $connected = array(); foreach ($limits as $limit) { $reason = $limit->didConnect(); $connected[] = $limit; if ($reason !== null) { break; } } // If we're killing the request here, disconnect any limits that we // connected to try to keep the accounting straight. if ($reason !== null) { foreach ($connected as $limit) { $limit->didDisconnect(array()); } self::didRateLimit($reason); } } /** * Tear down rate limiting and allow limits to score the request. * * @param map Additional, freeform request state. * @return void * @task ratelimit */ public static function disconnectRateLimits(array $request_state) { $limits = self::$limits; // Remove all limits before disconnecting them so this works properly if // it runs twice. (We run this automatically as a shutdown handler.) self::$limits = array(); foreach ($limits as $limit) { $limit->didDisconnect($request_state); } } /** * Emit an HTTP 429 "Too Many Requests" response (indicating that the user * has exceeded application rate limits) and exit. * * @return exit This method **does not return**. * @task ratelimit */ private static function didRateLimit($reason) { header( 'Content-Type: text/plain; charset=utf-8', $replace = true, $http_error = 429); echo $reason; exit(1); } /* -( Startup Timers )----------------------------------------------------- */ /** * Record the beginning of a new startup phase. * * For phases which occur before @{class:PhabricatorStartup} loads, save the * time and record it with @{method:recordStartupPhase} after the class is * available. * * @param string Phase name. * @task phases */ public static function beginStartupPhase($phase) { self::recordStartupPhase($phase, microtime(true)); } /** * Record the start time of a previously executed startup phase. * * For startup phases which occur after @{class:PhabricatorStartup} loads, * use @{method:beginStartupPhase} instead. This method can be used to * record a time before the class loads, then hand it over once the class * becomes available. * * @param string Phase name. * @param float Phase start time, from `microtime(true)`. * @task phases */ public static function recordStartupPhase($phase, $time) { self::$phases[$phase] = $time; } /** * Get information about startup phase timings. * * Sometimes, performance problems can occur before we start the profiler. * Since the profiler can't examine these phases, it isn't useful in * understanding their performance costs. * * Instead, the startup process marks when it enters various phases using * @{method:beginStartupPhase}. A later call to this method can retrieve this * information, which can be examined to gain greater insight into where * time was spent. The output is still crude, but better than nothing. * * @task phases */ public static function getPhases() { return self::$phases; } }