diff --git a/resources/sql/autopatches/20160202.ipv6.1.sql b/resources/sql/autopatches/20160202.ipv6.1.sql new file mode 100644 index 0000000000..d6a3ee5ccc --- /dev/null +++ b/resources/sql/autopatches/20160202.ipv6.1.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_repository.repository_pullevent + CHANGE remoteAddress remoteAddress VARBINARY(64); + +ALTER TABLE {$NAMESPACE}_repository.repository_pushevent + CHANGE remoteAddress remoteAddress VARBINARY(64); diff --git a/resources/sql/autopatches/20160202.ipv6.2.php b/resources/sql/autopatches/20160202.ipv6.2.php new file mode 100644 index 0000000000..50def09444 --- /dev/null +++ b/resources/sql/autopatches/20160202.ipv6.2.php @@ -0,0 +1,39 @@ +establishConnection('w'); + +$log_types = array($pull, $push); +foreach ($log_types as $log) { + foreach (new LiskMigrationIterator($log) as $row) { + $addr = $row->getRemoteAddress(); + + $addr = (string)$addr; + if (!strlen($addr)) { + continue; + } + + if (!ctype_digit($addr)) { + continue; + } + + if (!(int)$addr) { + continue; + } + + $ip = long2ip($addr); + if (!is_string($ip) || !strlen($ip)) { + continue; + } + + $id = $row->getID(); + queryfx( + $conn_w, + 'UPDATE %T SET remoteAddress = %s WHERE id = %d', + $log->getTableName(), + $ip, + $id); + } +} diff --git a/src/aphront/AphrontRequest.php b/src/aphront/AphrontRequest.php index e88a5a5e36..f8b01eb94c 100644 --- a/src/aphront/AphrontRequest.php +++ b/src/aphront/AphrontRequest.php @@ -1,796 +1,800 @@ host = $host; $this->path = $path; } public function setURIMap(array $uri_data) { $this->uriData = $uri_data; return $this; } public function getURIMap() { return $this->uriData; } public function getURIData($key, $default = null) { return idx($this->uriData, $key, $default); } public function setApplicationConfiguration( $application_configuration) { $this->applicationConfiguration = $application_configuration; return $this; } public function getApplicationConfiguration() { return $this->applicationConfiguration; } public function setPath($path) { $this->path = $path; return $this; } public function getPath() { return $this->path; } public function getHost() { // The "Host" header may include a port number, or may be a malicious // header in the form "realdomain.com:ignored@evil.com". Invoke the full // parser to extract the real domain correctly. See here for coverage of // a similar issue in Django: // // https://www.djangoproject.com/weblog/2012/oct/17/security/ $uri = new PhutilURI('http://'.$this->host); return $uri->getDomain(); } public function setSite(AphrontSite $site) { $this->site = $site; return $this; } public function getSite() { return $this->site; } public function setController(AphrontController $controller) { $this->controller = $controller; return $this; } public function getController() { return $this->controller; } /* -( Accessing Request Data )--------------------------------------------- */ /** * @task data */ public function setRequestData(array $request_data) { $this->requestData = $request_data; return $this; } /** * @task data */ public function getRequestData() { return $this->requestData; } /** * @task data */ public function getInt($name, $default = null) { if (isset($this->requestData[$name])) { // Converting from array to int is "undefined". Don't rely on whatever // PHP decides to do. if (is_array($this->requestData[$name])) { return $default; } return (int)$this->requestData[$name]; } else { return $default; } } /** * @task data */ public function getBool($name, $default = null) { if (isset($this->requestData[$name])) { if ($this->requestData[$name] === 'true') { return true; } else if ($this->requestData[$name] === 'false') { return false; } else { return (bool)$this->requestData[$name]; } } else { return $default; } } /** * @task data */ public function getStr($name, $default = null) { if (isset($this->requestData[$name])) { $str = (string)$this->requestData[$name]; // Normalize newline craziness. $str = str_replace( array("\r\n", "\r"), array("\n", "\n"), $str); return $str; } else { return $default; } } /** * @task data */ public function getArr($name, $default = array()) { if (isset($this->requestData[$name]) && is_array($this->requestData[$name])) { return $this->requestData[$name]; } else { return $default; } } /** * @task data */ public function getStrList($name, $default = array()) { if (!isset($this->requestData[$name])) { return $default; } $list = $this->getStr($name); $list = preg_split('/[\s,]+/', $list, $limit = -1, PREG_SPLIT_NO_EMPTY); return $list; } /** * @task data */ public function getExists($name) { return array_key_exists($name, $this->requestData); } public function getFileExists($name) { return isset($_FILES[$name]) && (idx($_FILES[$name], 'error') !== UPLOAD_ERR_NO_FILE); } public function isHTTPGet() { return ($_SERVER['REQUEST_METHOD'] == 'GET'); } public function isHTTPPost() { return ($_SERVER['REQUEST_METHOD'] == 'POST'); } public function isAjax() { return $this->getExists(self::TYPE_AJAX) && !$this->isQuicksand(); } public function isWorkflow() { return $this->getExists(self::TYPE_WORKFLOW) && !$this->isQuicksand(); } public function isQuicksand() { return $this->getExists(self::TYPE_QUICKSAND); } public function isConduit() { return $this->getExists(self::TYPE_CONDUIT); } public static function getCSRFTokenName() { return '__csrf__'; } public static function getCSRFHeaderName() { return 'X-Phabricator-Csrf'; } public static function getViaHeaderName() { return 'X-Phabricator-Via'; } public function validateCSRF() { $token_name = self::getCSRFTokenName(); $token = $this->getStr($token_name); // No token in the request, check the HTTP header which is added for Ajax // requests. if (empty($token)) { $token = self::getHTTPHeader(self::getCSRFHeaderName()); } $valid = $this->getUser()->validateCSRFToken($token); if (!$valid) { // Add some diagnostic details so we can figure out if some CSRF issues // are JS problems or people accessing Ajax URIs directly with their // browsers. $more_info = array(); if ($this->isAjax()) { $more_info[] = pht('This was an Ajax request.'); } else { $more_info[] = pht('This was a Web request.'); } if ($token) { $more_info[] = pht('This request had an invalid CSRF token.'); } else { $more_info[] = pht('This request had no CSRF token.'); } // Give a more detailed explanation of how to avoid the exception // in developer mode. if (PhabricatorEnv::getEnvConfig('phabricator.developer-mode')) { // TODO: Clean this up, see T1921. $more_info[] = pht( "To avoid this error, use %s to construct forms. If you are already ". "using %s, make sure the form 'action' uses a relative URI (i.e., ". "begins with a '%s'). Forms using absolute URIs do not include CSRF ". "tokens, to prevent leaking tokens to external sites.\n\n". "If this page performs writes which do not require CSRF protection ". "(usually, filling caches or logging), you can use %s to ". "temporarily bypass CSRF protection while writing. You should use ". "this only for writes which can not be protected with normal CSRF ". "mechanisms.\n\n". "Some UI elements (like %s) also have methods which will allow you ". "to render links as forms (like %s).", 'phabricator_form()', 'phabricator_form()', '/', 'AphrontWriteGuard::beginScopedUnguardedWrites()', 'PhabricatorActionListView', 'setRenderAsForm(true)'); } // This should only be able to happen if you load a form, pull your // internet for 6 hours, and then reconnect and immediately submit, // but give the user some indication of what happened since the workflow // is incredibly confusing otherwise. throw new AphrontCSRFException( pht( 'You are trying to save some data to Phabricator, but the request '. 'your browser made included an incorrect token. Reload the page '. 'and try again. You may need to clear your cookies.')."\n\n". implode("\n", $more_info)); } return true; } public function isFormPost() { $post = $this->getExists(self::TYPE_FORM) && !$this->getExists(self::TYPE_HISEC) && $this->isHTTPPost(); if (!$post) { return false; } return $this->validateCSRF(); } public function isFormOrHisecPost() { $post = $this->getExists(self::TYPE_FORM) && $this->isHTTPPost(); if (!$post) { return false; } return $this->validateCSRF(); } public function setCookiePrefix($prefix) { $this->cookiePrefix = $prefix; return $this; } private function getPrefixedCookieName($name) { if (strlen($this->cookiePrefix)) { return $this->cookiePrefix.'_'.$name; } else { return $name; } } public function getCookie($name, $default = null) { $name = $this->getPrefixedCookieName($name); $value = idx($_COOKIE, $name, $default); // Internally, PHP deletes cookies by setting them to the value 'deleted' // with an expiration date in the past. // At least in Safari, the browser may send this cookie anyway in some // circumstances. After logging out, the 302'd GET to /login/ consistently // includes deleted cookies on my local install. If a cookie value is // literally 'deleted', pretend it does not exist. if ($value === 'deleted') { return null; } return $value; } public function clearCookie($name) { $this->setCookieWithExpiration($name, '', time() - (60 * 60 * 24 * 30)); unset($_COOKIE[$name]); } /** * Get the domain which cookies should be set on for this request, or null * if the request does not correspond to a valid cookie domain. * * @return PhutilURI|null Domain URI, or null if no valid domain exists. * * @task cookie */ private function getCookieDomainURI() { if (PhabricatorEnv::getEnvConfig('security.require-https') && !$this->isHTTPS()) { return null; } $host = $this->getHost(); // If there's no base domain configured, just use whatever the request // domain is. This makes setup easier, and we'll tell administrators to // configure a base domain during the setup process. $base_uri = PhabricatorEnv::getEnvConfig('phabricator.base-uri'); if (!strlen($base_uri)) { return new PhutilURI('http://'.$host.'/'); } $alternates = PhabricatorEnv::getEnvConfig('phabricator.allowed-uris'); $allowed_uris = array_merge( array($base_uri), $alternates); foreach ($allowed_uris as $allowed_uri) { $uri = new PhutilURI($allowed_uri); if ($uri->getDomain() == $host) { return $uri; } } return null; } /** * Determine if security policy rules will allow cookies to be set when * responding to the request. * * @return bool True if setCookie() will succeed. If this method returns * false, setCookie() will throw. * * @task cookie */ public function canSetCookies() { return (bool)$this->getCookieDomainURI(); } /** * Set a cookie which does not expire for a long time. * * To set a temporary cookie, see @{method:setTemporaryCookie}. * * @param string Cookie name. * @param string Cookie value. * @return this * @task cookie */ public function setCookie($name, $value) { $far_future = time() + (60 * 60 * 24 * 365 * 5); return $this->setCookieWithExpiration($name, $value, $far_future); } /** * Set a cookie which expires soon. * * To set a durable cookie, see @{method:setCookie}. * * @param string Cookie name. * @param string Cookie value. * @return this * @task cookie */ public function setTemporaryCookie($name, $value) { return $this->setCookieWithExpiration($name, $value, 0); } /** * Set a cookie with a given expiration policy. * * @param string Cookie name. * @param string Cookie value. * @param int Epoch timestamp for cookie expiration. * @return this * @task cookie */ private function setCookieWithExpiration( $name, $value, $expire) { $is_secure = false; $base_domain_uri = $this->getCookieDomainURI(); if (!$base_domain_uri) { $configured_as = PhabricatorEnv::getEnvConfig('phabricator.base-uri'); $accessed_as = $this->getHost(); throw new Exception( pht( 'This Phabricator install is configured as "%s", but you are '. 'using the domain name "%s" to access a page which is trying to '. 'set a cookie. Acccess Phabricator on the configured primary '. 'domain or a configured alternate domain. Phabricator will not '. 'set cookies on other domains for security reasons.', $configured_as, $accessed_as)); } $base_domain = $base_domain_uri->getDomain(); $is_secure = ($base_domain_uri->getProtocol() == 'https'); $name = $this->getPrefixedCookieName($name); if (php_sapi_name() == 'cli') { // Do nothing, to avoid triggering "Cannot modify header information" // warnings. // TODO: This is effectively a test for whether we're running in a unit // test or not. Move this actual call to HTTPSink? } else { setcookie( $name, $value, $expire, $path = '/', $base_domain, $is_secure, $http_only = true); } $_COOKIE[$name] = $value; return $this; } public function setUser($user) { $this->user = $user; return $this; } public function getUser() { return $this->user; } public function getViewer() { return $this->user; } public function getRequestURI() { $get = $_GET; unset($get['__path__']); $path = phutil_escape_uri($this->getPath()); return id(new PhutilURI($path))->setQueryParams($get); } public function isDialogFormPost() { return $this->isFormPost() && $this->getStr('__dialog__'); } - public function getRemoteAddr() { - return $_SERVER['REMOTE_ADDR']; + public function getRemoteAddress() { + $address = $_SERVER['REMOTE_ADDR']; + if (!strlen($address)) { + return null; + } + return substr($address, 0, 64); } public function isHTTPS() { if (empty($_SERVER['HTTPS'])) { return false; } if (!strcasecmp($_SERVER['HTTPS'], 'off')) { return false; } return true; } public function isContinueRequest() { return $this->isFormPost() && $this->getStr('__continue__'); } public function isPreviewRequest() { return $this->isFormPost() && $this->getStr('__preview__'); } /** * Get application request parameters in a flattened form suitable for * inclusion in an HTTP request, excluding parameters with special meanings. * This is primarily useful if you want to ask the user for more input and * then resubmit their request. * * @return dict Original request parameters. */ public function getPassthroughRequestParameters($include_quicksand = false) { return self::flattenData( $this->getPassthroughRequestData($include_quicksand)); } /** * Get request data other than "magic" parameters. * * @return dict Request data, with magic filtered out. */ public function getPassthroughRequestData($include_quicksand = false) { $data = $this->getRequestData(); // Remove magic parameters like __dialog__ and __ajax__. foreach ($data as $key => $value) { if ($include_quicksand && $key == self::TYPE_QUICKSAND) { continue; } if (!strncmp($key, '__', 2)) { unset($data[$key]); } } return $data; } /** * Flatten an array of key-value pairs (possibly including arrays as values) * into a list of key-value pairs suitable for submitting via HTTP request * (with arrays flattened). * * @param dict Data to flatten. * @return dict Flat data suitable for inclusion in an HTTP * request. */ public static function flattenData(array $data) { $result = array(); foreach ($data as $key => $value) { if (is_array($value)) { foreach (self::flattenData($value) as $fkey => $fvalue) { $fkey = '['.preg_replace('/(?=\[)|$/', ']', $fkey, $limit = 1); $result[$key.$fkey] = $fvalue; } } else { $result[$key] = (string)$value; } } ksort($result); return $result; } /** * Read the value of an HTTP header from `$_SERVER`, or a similar datasource. * * This function accepts a canonical header name, like `"Accept-Encoding"`, * and looks up the appropriate value in `$_SERVER` (in this case, * `"HTTP_ACCEPT_ENCODING"`). * * @param string Canonical header name, like `"Accept-Encoding"`. * @param wild Default value to return if header is not present. * @param array? Read this instead of `$_SERVER`. * @return string|wild Header value if present, or `$default` if not. */ public static function getHTTPHeader($name, $default = null, $data = null) { // PHP mangles HTTP headers by uppercasing them and replacing hyphens with // underscores, then prepending 'HTTP_'. $php_index = strtoupper($name); $php_index = str_replace('-', '_', $php_index); $try_names = array(); $try_names[] = 'HTTP_'.$php_index; if ($php_index == 'CONTENT_TYPE' || $php_index == 'CONTENT_LENGTH') { // These headers may be available under alternate names. See // http://www.php.net/manual/en/reserved.variables.server.php#110763 $try_names[] = $php_index; } if ($data === null) { $data = $_SERVER; } foreach ($try_names as $try_name) { if (array_key_exists($try_name, $data)) { return $data[$try_name]; } } return $default; } /* -( Working With a Phabricator Cluster )--------------------------------- */ /** * Is this a proxied request originating from within the Phabricator cluster? * * IMPORTANT: This means the request is dangerous! * * These requests are **more dangerous** than normal requests (they can not * be safely proxied, because proxying them may cause a loop). Cluster * requests are not guaranteed to come from a trusted source, and should * never be treated as safer than normal requests. They are strictly less * safe. */ public function isProxiedClusterRequest() { return (bool)self::getHTTPHeader('X-Phabricator-Cluster'); } /** * Build a new @{class:HTTPSFuture} which proxies this request to another * node in the cluster. * * IMPORTANT: This is very dangerous! * * The future forwards authentication information present in the request. * Proxied requests must only be sent to trusted hosts. (We attempt to * enforce this.) * * This is not a general-purpose proxying method; it is a specialized * method with niche applications and severe security implications. * * @param string URI identifying the host we are proxying the request to. * @return HTTPSFuture New proxy future. * * @phutil-external-symbol class PhabricatorStartup */ public function newClusterProxyFuture($uri) { $uri = new PhutilURI($uri); $domain = $uri->getDomain(); $ip = gethostbyname($domain); if (!$ip) { throw new Exception( pht( 'Unable to resolve domain "%s"!', $domain)); } if (!PhabricatorEnv::isClusterAddress($ip)) { throw new Exception( pht( 'Refusing to proxy a request to IP address ("%s") which is not '. 'in the cluster address block (this address was derived by '. 'resolving the domain "%s").', $ip, $domain)); } $uri->setPath($this->getPath()); $uri->setQueryParams(self::flattenData($_GET)); $input = PhabricatorStartup::getRawInput(); $future = id(new HTTPSFuture($uri)) ->addHeader('Host', self::getHost()) ->addHeader('X-Phabricator-Cluster', true) ->setMethod($_SERVER['REQUEST_METHOD']) ->write($input); if (isset($_SERVER['PHP_AUTH_USER'])) { $future->setHTTPBasicAuthCredentials( $_SERVER['PHP_AUTH_USER'], new PhutilOpaqueEnvelope(idx($_SERVER, 'PHP_AUTH_PW', ''))); } $headers = array(); $seen = array(); // NOTE: apache_request_headers() might provide a nicer way to do this, // but isn't available under FCGI until PHP 5.4.0. foreach ($_SERVER as $key => $value) { if (preg_match('/^HTTP_/', $key)) { // Unmangle the header as best we can. $key = str_replace('_', ' ', $key); $key = strtolower($key); $key = ucwords($key); $key = str_replace(' ', '-', $key); $headers[] = array($key, $value); $seen[$key] = true; } } // In some situations, this may not be mapped into the HTTP_X constants. // CONTENT_LENGTH is similarly affected, but we trust cURL to take care // of that if it matters, since we're handing off a request body. if (empty($seen['Content-Type'])) { if (isset($_SERVER['CONTENT_TYPE'])) { $headers[] = array('Content-Type', $_SERVER['CONTENT_TYPE']); } } foreach ($headers as $header) { list($key, $value) = $header; switch ($key) { case 'Host': case 'Authorization': // Don't forward these headers, we've already handled them elsewhere. unset($headers[$key]); break; default: break; } } foreach ($headers as $header) { list($key, $value) = $header; $future->addHeader($key, $value); } return $future; } } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index 5b48fcbd03..740402524a 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -1,368 +1,369 @@ '; public function setUTF8SortingCollation($utf8_sorting_collation) { $this->utf8SortingCollation = $utf8_sorting_collation; return $this; } public function getUTF8SortingCollation() { return $this->utf8SortingCollation; } public function setUTF8BinaryCollation($utf8_binary_collation) { $this->utf8BinaryCollation = $utf8_binary_collation; return $this; } public function getUTF8BinaryCollation() { return $this->utf8BinaryCollation; } public function setUTF8Charset($utf8_charset) { $this->utf8Charset = $utf8_charset; return $this; } public function getUTF8Charset() { return $this->utf8Charset; } public function setServer(PhabricatorConfigServerSchema $server) { $this->server = $server; return $this; } public function getServer() { return $this->server; } abstract public function buildSchemata(); protected function buildLiskObjectSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), $object->getTableName(), $object->getSchemaColumns(), $object->getSchemaKeys()); } protected function buildRawSchema( $database_name, $table_name, array $columns, array $keys) { $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); foreach ($columns as $name => $type) { if ($type === null) { continue; } $details = $this->getDetailsForDataType($type); list($column_type, $charset, $collation, $nullable, $auto) = $details; $column = $this->newColumn($name) ->setDataType($type) ->setColumnType($column_type) ->setCharacterSet($charset) ->setCollation($collation) ->setNullable($nullable) ->setAutoIncrement($auto); $table->addColumn($column); } foreach ($keys as $key_name => $key_spec) { if ($key_spec === null) { // This is a subclass removing a key which Lisk expects. continue; } $key = $this->newKey($key_name) ->setColumnNames(idx($key_spec, 'columns', array())); $key->setUnique((bool)idx($key_spec, 'unique')); $key->setIndexType(idx($key_spec, 'type', 'BTREE')); $table->addKey($key); } $database->addTable($table); } protected function buildEdgeSchemata(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGE, array( 'src' => 'phid', 'type' => 'uint32', 'dst' => 'phid', 'dateCreated' => 'epoch', 'seq' => 'uint32', 'dataID' => 'id?', ), array( 'PRIMARY' => array( 'columns' => array('src', 'type', 'dst'), 'unique' => true, ), 'src' => array( 'columns' => array('src', 'type', 'dateCreated', 'seq'), ), 'key_dst' => array( 'columns' => array('dst', 'type', 'src'), 'unique' => true, ), )); $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGEDATA, array( 'id' => 'auto', 'data' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), )); } protected function getDatabase($name) { $server = $this->getServer(); $database = $server->getDatabase($this->getNamespacedDatabase($name)); if (!$database) { $database = $this->newDatabase($name); $server->addDatabase($database); } return $database; } protected function newDatabase($name) { return id(new PhabricatorConfigDatabaseSchema()) ->setName($this->getNamespacedDatabase($name)) ->setCharacterSet($this->getUTF8Charset()) ->setCollation($this->getUTF8BinaryCollation()); } protected function getNamespacedDatabase($name) { $namespace = PhabricatorLiskDAO::getStorageNamespace(); return $namespace.'_'.$name; } protected function newTable($name) { return id(new PhabricatorConfigTableSchema()) ->setName($name) ->setCollation($this->getUTF8BinaryCollation()); } protected function newColumn($name) { return id(new PhabricatorConfigColumnSchema()) ->setName($name); } protected function newKey($name) { return id(new PhabricatorConfigKeySchema()) ->setName($name); } private function getDetailsForDataType($data_type) { $column_type = null; $charset = null; $collation = null; $auto = false; // If the type ends with "?", make the column nullable. $nullable = false; if (preg_match('/\?$/', $data_type)) { $nullable = true; $data_type = substr($data_type, 0, -1); } // NOTE: MySQL allows fragments like "VARCHAR(32) CHARACTER SET binary", // but just interprets that to mean "VARBINARY(32)". The fragment is // totally disallowed in a MODIFY statement vs a CREATE TABLE statement. $is_binary = ($this->getUTF8Charset() == 'binary'); $matches = null; $pattern = '/^(fulltext|sort|text|char)(\d+)?\z/'; if (preg_match($pattern, $data_type, $matches)) { // Limit the permitted column lengths under the theory that it would // be nice to eventually reduce this to a small set of standard lengths. static $valid_types = array( 'text255' => true, 'text160' => true, 'text128' => true, 'text80' => true, 'text64' => true, 'text40' => true, 'text32' => true, 'text20' => true, 'text16' => true, 'text12' => true, 'text8' => true, 'text4' => true, 'text' => true, 'char3' => true, 'sort255' => true, 'sort128' => true, 'sort64' => true, 'sort32' => true, 'sort' => true, 'fulltext' => true, ); if (empty($valid_types[$data_type])) { throw new Exception(pht('Unknown column type "%s"!', $data_type)); } $type = $matches[1]; $size = idx($matches, 2); switch ($type) { case 'text': if ($is_binary) { if ($size) { $column_type = 'varbinary('.$size.')'; } else { $column_type = 'longblob'; } } else { if ($size) { $column_type = 'varchar('.$size.')'; } else { $column_type = 'longtext'; } } break; case 'sort': if ($size) { $column_type = 'varchar('.$size.')'; } else { $column_type = 'longtext'; } break; case 'fulltext'; // MySQL (at least, under MyISAM) refuses to create a FULLTEXT index // on a LONGBLOB column. We'd also lose case insensitivity in search. // Force this column to utf8 collation. This will truncate results // with 4-byte UTF characters in their text, but work reasonably in // the majority of cases. $column_type = 'longtext'; break; case 'char': $column_type = 'char('.$size.')'; break; } switch ($type) { case 'text': case 'char': if ($is_binary) { // We leave collation and character set unspecified in order to // generate valid SQL. } else { $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); } break; case 'sort': case 'fulltext': if ($is_binary) { $charset = 'utf8'; } else { $charset = $this->getUTF8Charset(); } $collation = $this->getUTF8SortingCollation(); break; } } else { switch ($data_type) { case 'auto': $column_type = 'int(10) unsigned'; $auto = true; break; case 'auto64': $column_type = 'bigint(20) unsigned'; $auto = true; break; case 'id': case 'epoch': case 'uint32': $column_type = 'int(10) unsigned'; break; case 'sint32': $column_type = 'int(10)'; break; case 'id64': case 'uint64': $column_type = 'bigint(20) unsigned'; break; case 'sint64': $column_type = 'bigint(20)'; break; case 'phid': case 'policy'; case 'hashpath64': + case 'ipaddress': $column_type = 'varbinary(64)'; break; case 'bytes64': $column_type = 'binary(64)'; break; case 'bytes40': $column_type = 'binary(40)'; break; case 'bytes32': $column_type = 'binary(32)'; break; case 'bytes20': $column_type = 'binary(20)'; break; case 'bytes12': $column_type = 'binary(12)'; break; case 'bytes4': $column_type = 'binary(4)'; break; case 'bytes': $column_type = 'longblob'; break; case 'bool': $column_type = 'tinyint(1)'; break; case 'double': $column_type = 'double'; break; case 'date': $column_type = 'date'; break; default: $column_type = self::DATATYPE_UNKNOWN; $charset = self::DATATYPE_UNKNOWN; $collation = self::DATATYPE_UNKNOWN; break; } } return array($column_type, $charset, $collation, $nullable, $auto); } } diff --git a/src/applications/diffusion/controller/DiffusionServeController.php b/src/applications/diffusion/controller/DiffusionServeController.php index 13290b9f41..8f3eb364f6 100644 --- a/src/applications/diffusion/controller/DiffusionServeController.php +++ b/src/applications/diffusion/controller/DiffusionServeController.php @@ -1,732 +1,731 @@ serviceViewer = $viewer; return $this; } public function getServiceViewer() { return $this->serviceViewer; } public function setServiceRepository(PhabricatorRepository $repository) { $this->serviceRepository = $repository; return $this; } public function getServiceRepository() { return $this->serviceRepository; } public function isVCSRequest(AphrontRequest $request) { $identifier = $this->getRepositoryIdentifierFromRequest($request); if ($identifier === null) { return null; } $content_type = $request->getHTTPHeader('Content-Type'); $user_agent = idx($_SERVER, 'HTTP_USER_AGENT'); $vcs = null; if ($request->getExists('service')) { $service = $request->getStr('service'); // We get this initially for `info/refs`. // Git also gives us a User-Agent like "git/1.8.2.3". $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; } else if (strncmp($user_agent, 'git/', 4) === 0) { $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; } else if ($content_type == 'application/x-git-upload-pack-request') { // We get this for `git-upload-pack`. $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; } else if ($content_type == 'application/x-git-receive-pack-request') { // We get this for `git-receive-pack`. $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT; } else if ($request->getExists('cmd')) { // Mercurial also sends an Accept header like // "application/mercurial-0.1", and a User-Agent like // "mercurial/proto-1.0". $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL; } else { // Subversion also sends an initial OPTIONS request (vs GET/POST), and // has a User-Agent like "SVN/1.8.3 (x86_64-apple-darwin11.4.2) // serf/1.3.2". $dav = $request->getHTTPHeader('DAV'); $dav = new PhutilURI($dav); if ($dav->getDomain() === 'subversion.tigris.org') { $vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_SVN; } } return $vcs; } public function handleRequest(AphrontRequest $request) { $service_exception = null; $response = null; try { $response = $this->serveRequest($request); } catch (Exception $ex) { $service_exception = $ex; } try { - $remote_addr = $request->getRemoteAddr(); - $remote_addr = ip2long($remote_addr); + $remote_addr = $request->getRemoteAddress(); $pull_event = id(new PhabricatorRepositoryPullEvent()) ->setEpoch(PhabricatorTime::getNow()) ->setRemoteAddress($remote_addr) ->setRemoteProtocol('http'); if ($response) { $pull_event ->setResultType('wild') ->setResultCode($response->getHTTPResponseCode()); if ($response instanceof PhabricatorVCSResponse) { $pull_event->setProperties( array( 'response.message' => $response->getMessage(), )); } } else { $pull_event ->setResultType('exception') ->setResultCode(500) ->setProperties( array( 'exception.class' => $ex->getClass(), 'exception.message' => $ex->getMessage(), )); } $viewer = $this->getServiceViewer(); if ($viewer) { $pull_event->setPullerPHID($viewer->getPHID()); } $repository = $this->getServiceRepository(); if ($repository) { $pull_event->setRepositoryPHID($repository->getPHID()); } $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $pull_event->save(); unset($unguarded); } catch (Exception $ex) { if ($service_exception) { throw $service_exception; } throw $ex; } if ($service_exception) { throw $service_exception; } return $response; } private function serveRequest(AphrontRequest $request) { $identifier = $this->getRepositoryIdentifierFromRequest($request); // If authentication credentials have been provided, try to find a user // that actually matches those credentials. if (isset($_SERVER['PHP_AUTH_USER']) && isset($_SERVER['PHP_AUTH_PW'])) { $username = $_SERVER['PHP_AUTH_USER']; $password = new PhutilOpaqueEnvelope($_SERVER['PHP_AUTH_PW']); $viewer = $this->authenticateHTTPRepositoryUser($username, $password); if (!$viewer) { return new PhabricatorVCSResponse( 403, pht('Invalid credentials.')); } } else { // User hasn't provided credentials, which means we count them as // being "not logged in". $viewer = new PhabricatorUser(); } $this->setServiceViewer($viewer); $allow_public = PhabricatorEnv::getEnvConfig('policy.allow-public'); $allow_auth = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth'); if (!$allow_public) { if (!$viewer->isLoggedIn()) { if ($allow_auth) { return new PhabricatorVCSResponse( 401, pht('You must log in to access repositories.')); } else { return new PhabricatorVCSResponse( 403, pht('Public and authenticated HTTP access are both forbidden.')); } } } try { $repository = id(new PhabricatorRepositoryQuery()) ->setViewer($viewer) ->withIdentifiers(array($identifier)) ->executeOne(); if (!$repository) { return new PhabricatorVCSResponse( 404, pht('No such repository exists.')); } } catch (PhabricatorPolicyException $ex) { if ($viewer->isLoggedIn()) { return new PhabricatorVCSResponse( 403, pht('You do not have permission to access this repository.')); } else { if ($allow_auth) { return new PhabricatorVCSResponse( 401, pht('You must log in to access this repository.')); } else { return new PhabricatorVCSResponse( 403, pht( 'This repository requires authentication, which is forbidden '. 'over HTTP.')); } } } $this->setServiceRepository($repository); if (!$repository->isTracked()) { return new PhabricatorVCSResponse( 403, pht('This repository is inactive.')); } $is_push = !$this->isReadOnlyRequest($repository); switch ($repository->getServeOverHTTP()) { case PhabricatorRepository::SERVE_READONLY: if ($is_push) { return new PhabricatorVCSResponse( 403, pht('This repository is read-only over HTTP.')); } break; case PhabricatorRepository::SERVE_READWRITE: if ($is_push) { $can_push = PhabricatorPolicyFilter::hasCapability( $viewer, $repository, DiffusionPushCapability::CAPABILITY); if (!$can_push) { if ($viewer->isLoggedIn()) { return new PhabricatorVCSResponse( 403, pht('You do not have permission to push to this repository.')); } else { if ($allow_auth) { return new PhabricatorVCSResponse( 401, pht('You must log in to push to this repository.')); } else { return new PhabricatorVCSResponse( 403, pht( 'Pushing to this repository requires authentication, '. 'which is forbidden over HTTP.')); } } } } break; case PhabricatorRepository::SERVE_OFF: default: return new PhabricatorVCSResponse( 403, pht('This repository is not available over HTTP.')); } $vcs_type = $repository->getVersionControlSystem(); $req_type = $this->isVCSRequest($request); if ($vcs_type != $req_type) { switch ($req_type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $result = new PhabricatorVCSResponse( 500, pht('This is not a Git repository.')); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $result = new PhabricatorVCSResponse( 500, pht('This is not a Mercurial repository.')); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $result = new PhabricatorVCSResponse( 500, pht('This is not a Subversion repository.')); break; default: $result = new PhabricatorVCSResponse( 500, pht('Unknown request type.')); break; } } else { switch ($vcs_type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $result = $this->serveVCSRequest($repository, $viewer); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $result = new PhabricatorVCSResponse( 500, pht( 'Phabricator does not support HTTP access to Subversion '. 'repositories.')); break; default: $result = new PhabricatorVCSResponse( 500, pht('Unknown version control system.')); break; } } $code = $result->getHTTPResponseCode(); if ($is_push && ($code == 200)) { $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $repository->writeStatusMessage( PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE, PhabricatorRepositoryStatusMessage::CODE_OKAY); unset($unguarded); } return $result; } private function serveVCSRequest( PhabricatorRepository $repository, PhabricatorUser $viewer) { // If this repository is hosted on a service, we need to proxy the request // to a host which can serve it. $is_cluster_request = $this->getRequest()->isProxiedClusterRequest(); $uri = $repository->getAlmanacServiceURI( $viewer, $is_cluster_request, array( 'http', 'https', )); if ($uri) { $future = $this->getRequest()->newClusterProxyFuture($uri); return id(new AphrontHTTPProxyResponse()) ->setHTTPFuture($future); } // Otherwise, we're going to handle the request locally. $vcs_type = $repository->getVersionControlSystem(); switch ($vcs_type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $result = $this->serveGitRequest($repository, $viewer); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $result = $this->serveMercurialRequest($repository, $viewer); break; } return $result; } private function isReadOnlyRequest( PhabricatorRepository $repository) { $request = $this->getRequest(); $method = $_SERVER['REQUEST_METHOD']; // TODO: This implementation is safe by default, but very incomplete. switch ($repository->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $service = $request->getStr('service'); $path = $this->getRequestDirectoryPath($repository); // NOTE: Service names are the reverse of what you might expect, as they // are from the point of view of the server. The main read service is // "git-upload-pack", and the main write service is "git-receive-pack". if ($method == 'GET' && $path == '/info/refs' && $service == 'git-upload-pack') { return true; } if ($path == '/git-upload-pack') { return true; } break; case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: $cmd = $request->getStr('cmd'); if ($cmd == 'batch') { $cmds = idx($this->getMercurialArguments(), 'cmds'); return DiffusionMercurialWireProtocol::isReadOnlyBatchCommand($cmds); } return DiffusionMercurialWireProtocol::isReadOnlyCommand($cmd); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: break; } return false; } /** * @phutil-external-symbol class PhabricatorStartup */ private function serveGitRequest( PhabricatorRepository $repository, PhabricatorUser $viewer) { $request = $this->getRequest(); $request_path = $this->getRequestDirectoryPath($repository); $repository_root = $repository->getLocalPath(); // Rebuild the query string to strip `__magic__` parameters and prevent // issues where we might interpret inputs like "service=read&service=write" // differently than the server does and pass it an unsafe command. // NOTE: This does not use getPassthroughRequestParameters() because // that code is HTTP-method agnostic and will encode POST data. $query_data = $_GET; foreach ($query_data as $key => $value) { if (!strncmp($key, '__', 2)) { unset($query_data[$key]); } } $query_string = http_build_query($query_data, '', '&'); // We're about to wipe out PATH with the rest of the environment, so // resolve the binary first. $bin = Filesystem::resolveBinary('git-http-backend'); if (!$bin) { throw new Exception( pht( 'Unable to find `%s` in %s!', 'git-http-backend', '$PATH')); } $env = array( 'REQUEST_METHOD' => $_SERVER['REQUEST_METHOD'], 'QUERY_STRING' => $query_string, 'CONTENT_TYPE' => $request->getHTTPHeader('Content-Type'), 'HTTP_CONTENT_ENCODING' => $request->getHTTPHeader('Content-Encoding'), 'REMOTE_ADDR' => $_SERVER['REMOTE_ADDR'], 'GIT_PROJECT_ROOT' => $repository_root, 'GIT_HTTP_EXPORT_ALL' => '1', 'PATH_INFO' => $request_path, 'REMOTE_USER' => $viewer->getUsername(), // TODO: Set these correctly. // GIT_COMMITTER_NAME // GIT_COMMITTER_EMAIL ) + $this->getCommonEnvironment($viewer); $input = PhabricatorStartup::getRawInput(); $command = csprintf('%s', $bin); $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); list($err, $stdout, $stderr) = id(new ExecFuture('%C', $command)) ->setEnv($env, true) ->write($input) ->resolve(); if ($err) { if ($this->isValidGitShallowCloneResponse($stdout, $stderr)) { // Ignore the error if the response passes this special check for // validity. $err = 0; } } if ($err) { return new PhabricatorVCSResponse( 500, pht( 'Error %d: %s', $err, phutil_utf8ize($stderr))); } return id(new DiffusionGitResponse())->setGitData($stdout); } private function getRequestDirectoryPath(PhabricatorRepository $repository) { $request = $this->getRequest(); $request_path = $request->getRequestURI()->getPath(); $base_path = preg_replace('@^/diffusion/[A-Z]+@', '', $request_path); // For Git repositories, strip an optional directory component if it // isn't the name of a known Git resource. This allows users to clone // repositories as "/diffusion/X/anything.git", for example. if ($repository->isGit()) { $known = array( 'info', 'git-upload-pack', 'git-receive-pack', ); foreach ($known as $key => $path) { $known[$key] = preg_quote($path, '@'); } $known = implode('|', $known); if (preg_match('@^/([^/]+)/('.$known.')(/|$)@', $base_path)) { $base_path = preg_replace('@^/([^/]+)@', '', $base_path); } } return $base_path; } private function authenticateHTTPRepositoryUser( $username, PhutilOpaqueEnvelope $password) { if (!PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth')) { // No HTTP auth permitted. return null; } if (!strlen($username)) { // No username. return null; } if (!strlen($password->openEnvelope())) { // No password. return null; } $user = id(new PhabricatorPeopleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withUsernames(array($username)) ->executeOne(); if (!$user) { // Username doesn't match anything. return null; } if (!$user->isUserActivated()) { // User is not activated. return null; } $password_entry = id(new PhabricatorRepositoryVCSPassword()) ->loadOneWhere('userPHID = %s', $user->getPHID()); if (!$password_entry) { // User doesn't have a password set. return null; } if (!$password_entry->comparePassword($password, $user)) { // Password doesn't match. return null; } // If the user's password is stored using a less-than-optimal hash, upgrade // them to the strongest available hash. $hash_envelope = new PhutilOpaqueEnvelope( $password_entry->getPasswordHash()); if (PhabricatorPasswordHasher::canUpgradeHash($hash_envelope)) { $password_entry->setPassword($password, $user); $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $password_entry->save(); unset($unguarded); } return $user; } private function serveMercurialRequest( PhabricatorRepository $repository, PhabricatorUser $viewer) { $request = $this->getRequest(); $bin = Filesystem::resolveBinary('hg'); if (!$bin) { throw new Exception( pht( 'Unable to find `%s` in %s!', 'hg', '$PATH')); } $env = $this->getCommonEnvironment($viewer); $input = PhabricatorStartup::getRawInput(); $cmd = $request->getStr('cmd'); $args = $this->getMercurialArguments(); $args = $this->formatMercurialArguments($cmd, $args); if (strlen($input)) { $input = strlen($input)."\n".$input."0\n"; } $command = csprintf('%s serve --stdio', $bin); $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command); list($err, $stdout, $stderr) = id(new ExecFuture('%C', $command)) ->setEnv($env, true) ->setCWD($repository->getLocalPath()) ->write("{$cmd}\n{$args}{$input}") ->resolve(); if ($err) { return new PhabricatorVCSResponse( 500, pht('Error %d: %s', $err, $stderr)); } if ($cmd == 'getbundle' || $cmd == 'changegroup' || $cmd == 'changegroupsubset') { // We're not completely sure that "changegroup" and "changegroupsubset" // actually work, they're for very old Mercurial. $body = gzcompress($stdout); } else if ($cmd == 'unbundle') { // This includes diagnostic information and anything echoed by commit // hooks. We ignore `stdout` since it just has protocol garbage, and // substitute `stderr`. $body = strlen($stderr)."\n".$stderr; } else { list($length, $body) = explode("\n", $stdout, 2); if ($cmd == 'capabilities') { $body = DiffusionMercurialWireProtocol::filterBundle2Capability($body); } } return id(new DiffusionMercurialResponse())->setContent($body); } private function getMercurialArguments() { // Mercurial sends arguments in HTTP headers. "Why?", you might wonder, // "Why would you do this?". $args_raw = array(); for ($ii = 1;; $ii++) { $header = 'HTTP_X_HGARG_'.$ii; if (!array_key_exists($header, $_SERVER)) { break; } $args_raw[] = $_SERVER[$header]; } $args_raw = implode('', $args_raw); return id(new PhutilQueryStringParser()) ->parseQueryString($args_raw); } private function formatMercurialArguments($command, array $arguments) { $spec = DiffusionMercurialWireProtocol::getCommandArgs($command); $out = array(); // Mercurial takes normal arguments like this: // // name // value $has_star = false; foreach ($spec as $arg_key) { if ($arg_key == '*') { $has_star = true; continue; } if (isset($arguments[$arg_key])) { $value = $arguments[$arg_key]; $size = strlen($value); $out[] = "{$arg_key} {$size}\n{$value}"; unset($arguments[$arg_key]); } } if ($has_star) { // Mercurial takes arguments for variable argument lists roughly like // this: // // * // argname1 // argvalue1 // argname2 // argvalue2 $count = count($arguments); $out[] = "* {$count}\n"; foreach ($arguments as $key => $value) { if (in_array($key, $spec)) { // We already added this argument above, so skip it. continue; } $size = strlen($value); $out[] = "{$key} {$size}\n{$value}"; } } return implode('', $out); } private function isValidGitShallowCloneResponse($stdout, $stderr) { // If you execute `git clone --depth N ...`, git sends a request which // `git-http-backend` responds to by emitting valid output and then exiting // with a failure code and an error message. If we ignore this error, // everything works. // This is a pretty funky fix: it would be nice to more precisely detect // that a request is a `--depth N` clone request, but we don't have any code // to decode protocol frames yet. Instead, look for reasonable evidence // in the error and output that we're looking at a `--depth` clone. // For evidence this isn't completely crazy, see: // https://github.com/schacon/grack/pull/7 $stdout_regexp = '(^Content-Type: application/x-git-upload-pack-result)m'; $stderr_regexp = '(The remote end hung up unexpectedly)'; $has_pack = preg_match($stdout_regexp, $stdout); $is_hangup = preg_match($stderr_regexp, $stderr); return $has_pack && $is_hangup; } private function getCommonEnvironment(PhabricatorUser $viewer) { - $remote_addr = $this->getRequest()->getRemoteAddr(); + $remote_address = $this->getRequest()->getRemoteAddress(); return array( DiffusionCommitHookEngine::ENV_USER => $viewer->getUsername(), - DiffusionCommitHookEngine::ENV_REMOTE_ADDRESS => $remote_addr, + DiffusionCommitHookEngine::ENV_REMOTE_ADDRESS => $remote_address, DiffusionCommitHookEngine::ENV_REMOTE_PROTOCOL => 'http', ); } } diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php index a723591212..741b21bd19 100644 --- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php +++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php @@ -1,1268 +1,1259 @@ remoteProtocol = $remote_protocol; return $this; } public function getRemoteProtocol() { return $this->remoteProtocol; } public function setRemoteAddress($remote_address) { $this->remoteAddress = $remote_address; return $this; } public function getRemoteAddress() { return $this->remoteAddress; } - private function getRemoteAddressForLog() { - // If whatever we have here isn't a valid IPv4 address, just store `null`. - // Older versions of PHP return `-1` on failure instead of `false`. - $remote_address = $this->getRemoteAddress(); - $remote_address = max(0, ip2long($remote_address)); - $remote_address = nonempty($remote_address, null); - return $remote_address; - } - public function setSubversionTransactionInfo($transaction, $repository) { $this->subversionTransaction = $transaction; $this->subversionRepository = $repository; return $this; } public function setStdin($stdin) { $this->stdin = $stdin; return $this; } public function getStdin() { return $this->stdin; } public function setOriginalArgv(array $original_argv) { $this->originalArgv = $original_argv; return $this; } public function getOriginalArgv() { return $this->originalArgv; } public function setRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setMercurialHook($mercurial_hook) { $this->mercurialHook = $mercurial_hook; return $this; } public function getMercurialHook() { return $this->mercurialHook; } /* -( Hook Execution )----------------------------------------------------- */ public function execute() { $ref_updates = $this->findRefUpdates(); $all_updates = $ref_updates; $caught = null; try { try { $this->rejectDangerousChanges($ref_updates); } catch (DiffusionCommitHookRejectException $ex) { // If we're rejecting dangerous changes, flag everything that we've // seen as rejected so it's clear that none of it was accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_DANGEROUS; throw $ex; } $this->applyHeraldRefRules($ref_updates, $all_updates); $content_updates = $this->findContentUpdates($ref_updates); $all_updates = array_merge($all_updates, $content_updates); $this->applyHeraldContentRules($content_updates, $all_updates); // Run custom scripts in `hook.d/` directories. $this->applyCustomHooks($all_updates); // If we make it this far, we're accepting these changes. Mark all the // logs as accepted. $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ACCEPT; } catch (Exception $ex) { // We'll throw this again in a minute, but we want to save all the logs // first. $caught = $ex; } // Save all the logs no matter what the outcome was. $event = $this->newPushEvent(); $event->setRejectCode($this->rejectCode); $event->setRejectDetails($this->rejectDetails); $event->openTransaction(); $event->save(); foreach ($all_updates as $update) { $update->setPushEventPHID($event->getPHID()); $update->save(); } $event->saveTransaction(); if ($caught) { throw $caught; } // If this went through cleanly, detect pushes which are actually imports // of an existing repository rather than an addition of new commits. If // this push is importing a bunch of stuff, set the importing flag on // the repository. It will be cleared once we fully process everything. if ($this->isInitialImport($all_updates)) { $repository = $this->getRepository(); $repository->openTransaction(); $repository->beginReadLocking(); $repository = $repository->reload(); $repository->setDetail('importing', true); $repository->save(); $repository->endReadLocking(); $repository->saveTransaction(); } if ($this->emailPHIDs) { // If Herald rules triggered email to users, queue a worker to send the // mail. We do this out-of-process so that we block pushes as briefly // as possible. // (We do need to pull some commit info here because the commit objects // may not exist yet when this worker runs, which could be immediately.) PhabricatorWorker::scheduleTask( 'PhabricatorRepositoryPushMailWorker', array( 'eventPHID' => $event->getPHID(), 'emailPHIDs' => array_values($this->emailPHIDs), 'info' => $this->loadCommitInfoForWorker($all_updates), ), array( 'priority' => PhabricatorWorker::PRIORITY_ALERTS, )); } return 0; } private function findRefUpdates() { $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialRefUpdates(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionRefUpdates(); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } private function rejectDangerousChanges(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $repository = $this->getRepository(); if ($repository->shouldAllowDangerousChanges()) { return; } $flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; foreach ($ref_updates as $ref_update) { if (!$ref_update->hasChangeFlags($flag_dangerous)) { // This is not a dangerous change. continue; } // We either have a branch deletion or a non fast-forward branch update. // Format a message and reject the push. $message = pht( "DANGEROUS CHANGE: %s\n". "Dangerous change protection is enabled for this repository.\n". "Edit the repository configuration before making dangerous changes.", $ref_update->getDangerousChangeDescription()); throw new DiffusionCommitHookRejectException($message); } } private function findContentUpdates(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $type = $this->getRepository()->getVersionControlSystem(); switch ($type) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return $this->findGitContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return $this->findMercurialContentUpdates($ref_updates); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: return $this->findSubversionContentUpdates($ref_updates); default: throw new Exception(pht('Unsupported repository type "%s"!', $type)); } } /* -( Herald )------------------------------------------------------------- */ private function applyHeraldRefRules( array $ref_updates, array $all_updates) { $this->applyHeraldRules( $ref_updates, new HeraldPreCommitRefAdapter(), $all_updates); } private function applyHeraldContentRules( array $content_updates, array $all_updates) { $this->applyHeraldRules( $content_updates, new HeraldPreCommitContentAdapter(), $all_updates); } private function applyHeraldRules( array $updates, HeraldAdapter $adapter_template, array $all_updates) { if (!$updates) { return; } $adapter_template->setHookEngine($this); $engine = new HeraldEngine(); $rules = null; $blocking_effect = null; $blocked_update = null; $blocking_xscript = null; foreach ($updates as $update) { $adapter = id(clone $adapter_template) ->setPushLog($update); if ($rules === null) { $rules = $engine->loadRulesForAdapter($adapter); } $effects = $engine->applyRules($rules, $adapter); $engine->applyEffects($effects, $adapter, $rules); $xscript = $engine->getTranscript(); // Store any PHIDs we want to send email to for later. foreach ($adapter->getEmailPHIDs() as $email_phid) { $this->emailPHIDs[$email_phid] = $email_phid; } $block_action = DiffusionBlockHeraldAction::ACTIONCONST; if ($blocking_effect === null) { foreach ($effects as $effect) { if ($effect->getAction() == $block_action) { $blocking_effect = $effect; $blocked_update = $update; $blocking_xscript = $xscript; break; } } } } if ($blocking_effect) { $rule = $blocking_effect->getRule(); $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_HERALD; $this->rejectDetails = $rule->getPHID(); $message = $blocking_effect->getTarget(); if (!strlen($message)) { $message = pht('(None.)'); } $blocked_ref_name = coalesce( $blocked_update->getRefName(), $blocked_update->getRefNewShort()); $blocked_name = $blocked_update->getRefType().'/'.$blocked_ref_name; throw new DiffusionCommitHookRejectException( pht( "This push was rejected by Herald push rule %s.\n". " Change: %s\n". " Rule: %s\n". " Reason: %s\n". "Transcript: %s", $rule->getMonogram(), $blocked_name, $rule->getName(), $message, PhabricatorEnv::getProductionURI( '/herald/transcript/'.$blocking_xscript->getID().'/'))); } } public function loadViewerProjectPHIDsForHerald() { // This just caches the viewer's projects so we don't need to load them // over and over again when applying Herald rules. if ($this->heraldViewerProjects === null) { $this->heraldViewerProjects = id(new PhabricatorProjectQuery()) ->setViewer($this->getViewer()) ->withMemberPHIDs(array($this->getViewer()->getPHID())) ->execute(); } return mpull($this->heraldViewerProjects, 'getPHID'); } /* -( Git )---------------------------------------------------------------- */ private function findGitRefUpdates() { $ref_updates = array(); // First, parse stdin, which lists all the ref changes. The input looks // like this: // // $stdin = $this->getStdin(); $lines = phutil_split_lines($stdin, $retain_endings = false); foreach ($lines as $line) { $parts = explode(' ', $line, 3); if (count($parts) != 3) { throw new Exception(pht('Expected "old new ref", got "%s".', $line)); } $ref_old = $parts[0]; $ref_new = $parts[1]; $ref_raw = $parts[2]; if (preg_match('(^refs/heads/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; $ref_raw = substr($ref_raw, strlen('refs/heads/')); } else if (preg_match('(^refs/tags/)', $ref_raw)) { $ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG; $ref_raw = substr($ref_raw, strlen('refs/tags/')); } else { throw new Exception( pht( "Unable to identify the reftype of '%s'. Rejecting push.", $ref_raw)); } $ref_update = $this->newPushLog() ->setRefType($ref_type) ->setRefName($ref_raw) ->setRefOld($ref_old) ->setRefNew($ref_new); $ref_updates[] = $ref_update; } $this->findGitMergeBases($ref_updates); $this->findGitChangeFlags($ref_updates); return $ref_updates; } private function findGitMergeBases(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); $futures = array(); foreach ($ref_updates as $key => $ref_update) { // If the old hash is "00000...", the ref is being created (either a new // branch, or a new tag). If the new hash is "00000...", the ref is being // deleted. If both are nonempty, the ref is being updated. For updates, // we'll figure out the `merge-base` of the old and new objects here. This // lets us reject non-FF changes cheaply; later, we'll figure out exactly // which commits are new. $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); if (($ref_old === self::EMPTY_HASH) || ($ref_new === self::EMPTY_HASH)) { continue; } $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'merge-base %s %s', $ref_old, $ref_new); } $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { // If 'old' and 'new' have no common ancestors (for example, a force push // which completely rewrites a ref), `git merge-base` will exit with // an error and no output. It would be nice to find a positive test // for this instead, but I couldn't immediately come up with one. See // T4224. Assume this means there are no ancestors. list($err, $stdout) = $future->resolve(); if ($err) { $merge_base = null; } else { $merge_base = rtrim($stdout, "\n"); } $ref_update = $ref_updates[$key]; $ref_update->setMergeBase($merge_base); } return $ref_updates; } private function findGitChangeFlags(array $ref_updates) { assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog'); foreach ($ref_updates as $key => $ref_update) { $ref_old = $ref_update->getRefOld(); $ref_new = $ref_update->getRefNew(); $ref_type = $ref_update->getRefType(); $ref_flags = 0; $dangerous = null; if (($ref_old === self::EMPTY_HASH) && ($ref_new === self::EMPTY_HASH)) { // This happens if you try to delete a tag or branch which does not // exist by pushing directly to the ref. Git will warn about it but // allow it. Just call it a delete, without flagging it as dangerous. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else if ($ref_old === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($ref_new === self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push deletes the branch '%s'.", $ref_update->getRefName()); } } else { $merge_base = $ref_update->getMergeBase(); if ($merge_base == $ref_old) { // This is a fast-forward update to an existing branch. // These are safe. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; // For now, we don't consider deleting or moving tags to be a // "dangerous" update. It's way harder to get wrong and should be easy // to recover from once we have better logging. Only add the dangerous // flag if this ref is a branch. if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; $dangerous = pht( "The change you're attempting to push updates the branch '%s' ". "from '%s' to '%s', but this is not a fast-forward. Pushes ". "which rewrite published branch history are dangerous.", $ref_update->getRefName(), $ref_update->getRefOldShort(), $ref_update->getRefNewShort()); } } } $ref_update->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } } return $ref_updates; } private function findGitContentUpdates(array $ref_updates) { $flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; $futures = array(); foreach ($ref_updates as $key => $ref_update) { if ($ref_update->hasChangeFlags($flag_delete)) { // Deleting a branch or tag can never create any new commits. continue; } // NOTE: This piece of magic finds all new commits, by walking backward // from the new value to the value of *any* existing ref in the // repository. Particularly, this will cover the cases of a new branch, a // completely moved tag, etc. $futures[$key] = $this->getRepository()->getLocalCommandFuture( 'log --format=%s %s --not --all', '%H', $ref_update->getRefNew()); } $content_updates = array(); $futures = id(new FutureIterator($futures)) ->limit(8); foreach ($futures as $key => $future) { list($stdout) = $future->resolvex(); if (!strlen(trim($stdout))) { // This change doesn't have any new commits. One common case of this // is creating a new tag which points at an existing commit. continue; } $commits = phutil_split_lines($stdout, $retain_newlines = false); // If we're looking at a branch, mark all of the new commits as on that // branch. It's only possible for these commits to be on updated branches, // since any other branch heads are necessarily behind them. $branch_name = null; $ref_update = $ref_updates[$key]; $type_branch = PhabricatorRepositoryPushLog::REFTYPE_BRANCH; if ($ref_update->getRefType() == $type_branch) { $branch_name = $ref_update->getRefName(); } foreach ($commits as $commit) { if ($branch_name) { $this->gitCommits[$commit][] = $branch_name; } $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } } return $content_updates; } /* -( Custom )------------------------------------------------------------- */ private function applyCustomHooks(array $updates) { $args = $this->getOriginalArgv(); $stdin = $this->getStdin(); $console = PhutilConsole::getConsole(); $env = array( 'PHABRICATOR_REPOSITORY' => $this->getRepository()->getCallsign(), self::ENV_USER => $this->getViewer()->getUsername(), self::ENV_REMOTE_PROTOCOL => $this->getRemoteProtocol(), self::ENV_REMOTE_ADDRESS => $this->getRemoteAddress(), ); $directories = $this->getRepository()->getHookDirectories(); foreach ($directories as $directory) { $hooks = $this->getExecutablesInDirectory($directory); sort($hooks); foreach ($hooks as $hook) { // NOTE: We're explicitly running the hooks in sequential order to // make this more predictable. $future = id(new ExecFuture('%s %Ls', $hook, $args)) ->setEnv($env, $wipe_process_env = false) ->write($stdin); list($err, $stdout, $stderr) = $future->resolve(); if (!$err) { // This hook ran OK, but echo its output in case there was something // informative. $console->writeOut('%s', $stdout); $console->writeErr('%s', $stderr); continue; } $this->rejectCode = PhabricatorRepositoryPushLog::REJECT_EXTERNAL; $this->rejectDetails = basename($hook); throw new DiffusionCommitHookRejectException( pht( "This push was rejected by custom hook script '%s':\n\n%s%s", basename($hook), $stdout, $stderr)); } } } private function getExecutablesInDirectory($directory) { $executables = array(); if (!Filesystem::pathExists($directory)) { return $executables; } foreach (Filesystem::listDirectory($directory) as $path) { $full_path = $directory.DIRECTORY_SEPARATOR.$path; if (!is_executable($full_path)) { // Don't include non-executable files. continue; } if (basename($full_path) == 'README') { // Don't include README, even if it is marked as executable. It almost // certainly got caught in the crossfire of a sweeping `chmod`, since // users do this with some frequency. continue; } $executables[] = $full_path; } return $executables; } /* -( Mercurial )---------------------------------------------------------- */ private function findMercurialRefUpdates() { $hook = $this->getMercurialHook(); switch ($hook) { case 'pretxnchangegroup': return $this->findMercurialChangegroupRefUpdates(); case 'prepushkey': return $this->findMercurialPushKeyRefUpdates(); default: throw new Exception(pht('Unrecognized hook "%s"!', $hook)); } } private function findMercurialChangegroupRefUpdates() { $hg_node = getenv('HG_NODE'); if (!$hg_node) { throw new Exception( pht( 'Expected %s in environment!', 'HG_NODE')); } // NOTE: We need to make sure this is passed to subprocesses, or they won't // be able to see new commits. Mercurial uses this as a marker to determine // whether the pending changes are visible or not. $_ENV['HG_PENDING'] = getenv('HG_PENDING'); $repository = $this->getRepository(); $futures = array(); foreach (array('old', 'new') as $key) { $futures[$key] = $repository->getLocalCommandFuture( 'heads --template %s', '{node}\1{branch}\2'); } // Wipe HG_PENDING out of the old environment so we see the pre-commit // state of the repository. $futures['old']->updateEnv('HG_PENDING', null); $futures['commits'] = $repository->getLocalCommandFuture( 'log --rev %s --template %s', hgsprintf('%s:%s', $hg_node, 'tip'), '{node}\1{branch}\2'); // Resolve all of the futures now. We don't need the 'commits' future yet, // but it simplifies the logic to just get it out of the way. foreach (new FutureIterator($futures) as $future) { $future->resolve(); } list($commit_raw) = $futures['commits']->resolvex(); $commit_map = $this->parseMercurialCommits($commit_raw); $this->mercurialCommits = $commit_map; // NOTE: `hg heads` exits with an error code and no output if the repository // has no heads. Most commonly this happens on a new repository. We know // we can run `hg` successfully since the `hg log` above didn't error, so // just ignore the error code. list($err, $old_raw) = $futures['old']->resolve(); $old_refs = $this->parseMercurialHeads($old_raw); list($err, $new_raw) = $futures['new']->resolve(); $new_refs = $this->parseMercurialHeads($new_raw); $all_refs = array_keys($old_refs + $new_refs); $ref_updates = array(); foreach ($all_refs as $ref) { $old_heads = idx($old_refs, $ref, array()); $new_heads = idx($new_refs, $ref, array()); sort($old_heads); sort($new_heads); if (!$old_heads && !$new_heads) { // This should never be possible, as it makes no sense. Explode. throw new Exception( pht( 'Mercurial repository has no new or old heads for branch "%s" '. 'after push. This makes no sense; rejecting change.', $ref)); } if ($old_heads === $new_heads) { // No changes to this branch, so skip it. continue; } $stray_heads = array(); if ($old_heads && !$new_heads) { // This is a branch deletion with "--close-branch". $head_map = array(); foreach ($old_heads as $old_head) { $head_map[$old_head] = array(self::EMPTY_HASH); } } else if (count($old_heads) > 1) { // HORRIBLE: In Mercurial, branches can have multiple heads. If the // old branch had multiple heads, we need to figure out which new // heads descend from which old heads, so we can tell whether you're // actively creating new heads (dangerous) or just working in a // repository that's already full of garbage (strongly discouraged but // not as inherently dangerous). These cases should be very uncommon. // NOTE: We're only looking for heads on the same branch. The old // tip of the branch may be the branchpoint for other branches, but that // is OK. $dfutures = array(); foreach ($old_heads as $old_head) { $dfutures[$old_head] = $repository->getLocalCommandFuture( 'log --branch %s --rev %s --template %s', $ref, hgsprintf('(descendants(%s) and head())', $old_head), '{node}\1'); } $head_map = array(); foreach (new FutureIterator($dfutures) as $future_head => $dfuture) { list($stdout) = $dfuture->resolvex(); $descendant_heads = array_filter(explode("\1", $stdout)); if ($descendant_heads) { // This old head has at least one descendant in the push. $head_map[$future_head] = $descendant_heads; } else { // This old head has no descendants, so it is being deleted. $head_map[$future_head] = array(self::EMPTY_HASH); } } // Now, find all the new stray heads this push creates, if any. These // are new heads which do not descend from the old heads. $seen = array_fuse(array_mergev($head_map)); foreach ($new_heads as $new_head) { if ($new_head === self::EMPTY_HASH) { // If a branch head is being deleted, don't insert it as an add. continue; } if (empty($seen[$new_head])) { $head_map[self::EMPTY_HASH][] = $new_head; } } } else if ($old_heads) { $head_map[head($old_heads)] = $new_heads; } else { $head_map[self::EMPTY_HASH] = $new_heads; } foreach ($head_map as $old_head => $child_heads) { foreach ($child_heads as $new_head) { if ($new_head === $old_head) { continue; } $ref_flags = 0; $dangerous = null; if ($old_head == self::EMPTY_HASH) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } $deletes_existing_head = ($new_head == self::EMPTY_HASH); $splits_existing_head = (count($child_heads) > 1); $creates_duplicate_head = ($old_head == self::EMPTY_HASH) && (count($head_map) > 1); if ($splits_existing_head || $creates_duplicate_head) { $readable_child_heads = array(); foreach ($child_heads as $child_head) { $readable_child_heads[] = substr($child_head, 0, 12); } $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS; if ($splits_existing_head) { // We're splitting an existing head into two or more heads. // This is dangerous, and a super bad idea. Note that we're only // raising this if you're actively splitting a branch head. If a // head split in the past, we don't consider appends to it // to be dangerous. $dangerous = pht( "The change you're attempting to push splits the head of ". "branch '%s' into multiple heads: %s. This is inadvisable ". "and dangerous.", $ref, implode(', ', $readable_child_heads)); } else { // We're adding a second (or more) head to a branch. The new // head is not a descendant of any old head. $dangerous = pht( "The change you're attempting to push creates new, divergent ". "heads for the branch '%s': %s. This is inadvisable and ". "dangerous.", $ref, implode(', ', $readable_child_heads)); } } if ($deletes_existing_head) { // TODO: Somewhere in here we should be setting CHANGEFLAG_REWRITE // if we are also creating at least one other head to replace // this one. // NOTE: In Git, this is a dangerous change, but it is not dangerous // in Mercurial. Mercurial branches are version controlled, and // Mercurial does not prompt you for any special flags when pushing // a `--close-branch` commit by default. $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH) ->setRefName($ref) ->setRefOld($old_head) ->setRefNew($new_head) ->setChangeFlags($ref_flags); if ($dangerous !== null) { $ref_update->attachDangerousChangeDescription($dangerous); } $ref_updates[] = $ref_update; } } } return $ref_updates; } private function findMercurialPushKeyRefUpdates() { $key_namespace = getenv('HG_NAMESPACE'); if ($key_namespace === 'phases') { // Mercurial changes commit phases as part of normal push operations. We // just ignore these, as they don't seem to represent anything // interesting. return array(); } $key_name = getenv('HG_KEY'); $key_old = getenv('HG_OLD'); if (!strlen($key_old)) { $key_old = null; } $key_new = getenv('HG_NEW'); if (!strlen($key_new)) { $key_new = null; } if ($key_namespace !== 'bookmarks') { throw new Exception( pht( "Unknown Mercurial key namespace '%s', with key '%s' (%s -> %s). ". "Rejecting push.", $key_namespace, $key_name, coalesce($key_old, pht('null')), coalesce($key_new, pht('null')))); } if ($key_old === $key_new) { // We get a callback when the bookmark doesn't change. Just ignore this, // as it's a no-op. return array(); } $ref_flags = 0; $merge_base = null; if ($key_old === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; } else if ($key_new === null) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE; } else { list($merge_base_raw) = $this->getRepository()->execxLocalCommand( 'log --template %s --rev %s', '{node}', hgsprintf('ancestor(%s, %s)', $key_old, $key_new)); if (strlen(trim($merge_base_raw))) { $merge_base = trim($merge_base_raw); } if ($merge_base && ($merge_base === $key_old)) { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; } else { $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE; } } $ref_update = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BOOKMARK) ->setRefName($key_name) ->setRefOld(coalesce($key_old, self::EMPTY_HASH)) ->setRefNew(coalesce($key_new, self::EMPTY_HASH)) ->setChangeFlags($ref_flags); return array($ref_update); } private function findMercurialContentUpdates(array $ref_updates) { $content_updates = array(); foreach ($this->mercurialCommits as $commit => $branches) { $content_updates[$commit] = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($commit) ->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD); } return $content_updates; } private function parseMercurialCommits($raw) { $commits_lines = explode("\2", $raw); $commits_lines = array_filter($commits_lines); $commit_map = array(); foreach ($commits_lines as $commit_line) { list($node, $branch) = explode("\1", $commit_line); $commit_map[$node] = array($branch); } return $commit_map; } private function parseMercurialHeads($raw) { $heads_map = $this->parseMercurialCommits($raw); $heads = array(); foreach ($heads_map as $commit => $branches) { foreach ($branches as $branch) { $heads[$branch][] = $commit; } } return $heads; } /* -( Subversion )--------------------------------------------------------- */ private function findSubversionRefUpdates() { // Subversion doesn't have any kind of mutable ref metadata. return array(); } private function findSubversionContentUpdates(array $ref_updates) { list($youngest) = execx( 'svnlook youngest %s', $this->subversionRepository); $ref_new = (int)$youngest + 1; $ref_flags = 0; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD; $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND; $ref_content = $this->newPushLog() ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT) ->setRefNew($ref_new) ->setChangeFlags($ref_flags); return array($ref_content); } /* -( Internals )---------------------------------------------------------- */ private function newPushLog() { // NOTE: We generate PHIDs up front so the Herald transcripts can pick them // up. $phid = id(new PhabricatorRepositoryPushLog())->generatePHID(); return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer()) ->setPHID($phid) ->setRepositoryPHID($this->getRepository()->getPHID()) ->attachRepository($this->getRepository()) ->setEpoch(time()); } private function newPushEvent() { $viewer = $this->getViewer(); return PhabricatorRepositoryPushEvent::initializeNewEvent($viewer) ->setRepositoryPHID($this->getRepository()->getPHID()) - ->setRemoteAddress($this->getRemoteAddressForLog()) + ->setRemoteAddress($this->getRemoteAddress()) ->setRemoteProtocol($this->getRemoteProtocol()) ->setEpoch(time()); } public function loadChangesetsForCommit($identifier) { $byte_limit = HeraldCommitAdapter::getEnormousByteLimit(); $time_limit = HeraldCommitAdapter::getEnormousTimeLimit(); $vcs = $this->getRepository()->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // For git and hg, we can use normal commands. $drequest = DiffusionRequest::newFromDictionary( array( 'repository' => $this->getRepository(), 'user' => $this->getViewer(), 'commit' => $identifier, )); $raw_diff = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest) ->setTimeout($time_limit) ->setByteLimit($byte_limit) ->setLinesOfContext(0) ->loadRawDiff(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // TODO: This diff has 3 lines of context, which produces slightly // incorrect "added file content" and "removed file content" results. // This may also choke on binaries, but "svnlook diff" does not support // the "--diff-cmd" flag. // For subversion, we need to use `svnlook`. $future = new ExecFuture( 'svnlook diff -t %s %s', $this->subversionTransaction, $this->subversionRepository); $future->setTimeout($time_limit); $future->setStdoutSizeLimit($byte_limit); $future->setStderrSizeLimit($byte_limit); list($raw_diff) = $future->resolvex(); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } if (strlen($raw_diff) >= $byte_limit) { throw new Exception( pht( 'The raw text of this change is enormous (larger than %d '. 'bytes). Herald can not process it.', $byte_limit)); } if (!strlen($raw_diff)) { // If the commit is actually empty, just return no changesets. return array(); } $parser = new ArcanistDiffParser(); $changes = $parser->parseDiff($raw_diff); $diff = DifferentialDiff::newEphemeralFromRawChanges( $changes); return $diff->getChangesets(); } public function loadCommitRefForCommit($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: return id(new DiffusionLowLevelCommitQuery()) ->setRepository($repository) ->withIdentifier($identifier) ->execute(); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // For subversion, we need to use `svnlook`. list($message) = execx( 'svnlook log -t %s %s', $this->subversionTransaction, $this->subversionRepository); return id(new DiffusionCommitRef()) ->setMessage($message); break; default: throw new Exception(pht("Unknown VCS '%s!'", $vcs)); } } public function loadBranches($identifier) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: return idx($this->gitCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: // NOTE: This will be "the branch the commit was made to", not // "a list of all branch heads which descend from the commit". // This is consistent with Mercurial, but possibly confusing. return idx($this->mercurialCommits, $identifier, array()); case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // Subversion doesn't have branches. return array(); } } private function loadCommitInfoForWorker(array $all_updates) { $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; $map = array(); foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $map[$update->getRefNew()] = array(); } foreach ($map as $identifier => $info) { $ref = $this->loadCommitRefForCommit($identifier); $map[$identifier] += array( 'summary' => $ref->getSummary(), 'branches' => $this->loadBranches($identifier), ); } return $map; } private function isInitialImport(array $all_updates) { $repository = $this->getRepository(); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: // There is no meaningful way to import history into Subversion by // pushing. return false; default: break; } // Now, apply a heuristic to guess whether this is a normal commit or // an initial import. We guess something is an initial import if: // // - the repository is currently empty; and // - it pushes more than 7 commits at once. // // The number "7" is chosen arbitrarily as seeming reasonable. We could // also look at author data (do the commits come from multiple different // authors?) and commit date data (is the oldest commit more than 48 hours // old), but we don't have immediate access to those and this simple // heruistic might be good enough. $commit_count = 0; $type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT; foreach ($all_updates as $update) { if ($update->getRefType() != $type_commit) { continue; } $commit_count++; } if ($commit_count <= 7) { // If this pushes a very small number of commits, assume it's an // initial commit or stack of a few initial commits. return false; } $any_commits = id(new DiffusionCommitQuery()) ->setViewer($this->getViewer()) ->withRepository($repository) ->setLimit(1) ->execute(); if ($any_commits) { // If the repository already has commits, this isn't an import. return false; } return true; } } diff --git a/src/applications/diffusion/view/DiffusionPushLogListView.php b/src/applications/diffusion/view/DiffusionPushLogListView.php index 860320e624..7cc02a49b4 100644 --- a/src/applications/diffusion/view/DiffusionPushLogListView.php +++ b/src/applications/diffusion/view/DiffusionPushLogListView.php @@ -1,131 +1,128 @@ logs = $logs; return $this; } public function setHandles(array $handles) { $this->handles = $handles; return $this; } public function render() { $logs = $this->logs; $viewer = $this->getUser(); $handles = $this->handles; // Figure out which repositories are editable. We only let you see remote // IPs if you have edit capability on a repository. $editable_repos = array(); if ($logs) { $editable_repos = id(new PhabricatorRepositoryQuery()) ->setViewer($viewer) ->requireCapabilities( array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, )) ->withPHIDs(mpull($logs, 'getRepositoryPHID')) ->execute(); $editable_repos = mpull($editable_repos, null, 'getPHID'); } $rows = array(); foreach ($logs as $log) { $repository = $log->getRepository(); // Reveal this if it's valid and the user can edit the repository. - $remote_addr = '-'; + $remote_address = '-'; if (isset($editable_repos[$log->getRepositoryPHID()])) { - $remote_long = $log->getPushEvent()->getRemoteAddress(); - if ($remote_long) { - $remote_addr = long2ip($remote_long); - } + $remote_address = $log->getPushEvent()->getRemoteAddress(); } $event_id = $log->getPushEvent()->getID(); $old_ref_link = null; if ($log->getRefOld() != DiffusionCommitHookEngine::EMPTY_HASH) { $old_ref_link = phutil_tag( 'a', array( 'href' => $repository->getCommitURI($log->getRefOld()), ), $log->getRefOldShort()); } $rows[] = array( phutil_tag( 'a', array( 'href' => '/diffusion/pushlog/view/'.$event_id.'/', ), $event_id), phutil_tag( 'a', array( 'href' => $repository->getURI(), ), $repository->getDisplayName()), $handles[$log->getPusherPHID()]->renderLink(), - $remote_addr, + $remote_address, $log->getPushEvent()->getRemoteProtocol(), $log->getRefType(), $log->getRefName(), $old_ref_link, phutil_tag( 'a', array( 'href' => $repository->getCommitURI($log->getRefNew()), ), $log->getRefNewShort()), // TODO: Make these human-readable. $log->getChangeFlags(), $log->getPushEvent()->getRejectCode(), phabricator_datetime($log->getEpoch(), $viewer), ); } $table = id(new AphrontTableView($rows)) ->setHeaders( array( pht('Push'), pht('Repository'), pht('Pusher'), pht('From'), pht('Via'), pht('Type'), pht('Name'), pht('Old'), pht('New'), pht('Flags'), pht('Code'), pht('Date'), )) ->setColumnClasses( array( '', '', '', '', '', '', 'wide', 'n', 'n', 'date', )); return $table; } } diff --git a/src/applications/people/view/PhabricatorUserLogView.php b/src/applications/people/view/PhabricatorUserLogView.php index 12bcee9d76..c467a9010d 100644 --- a/src/applications/people/view/PhabricatorUserLogView.php +++ b/src/applications/people/view/PhabricatorUserLogView.php @@ -1,95 +1,95 @@ searchBaseURI = $search_base_uri; return $this; } public function setLogs(array $logs) { assert_instances_of($logs, 'PhabricatorUserLog'); $this->logs = $logs; return $this; } public function setHandles(array $handles) { assert_instances_of($handles, 'PhabricatorObjectHandle'); $this->handles = $handles; return $this; } public function render() { $logs = $this->logs; $handles = $this->handles; $viewer = $this->getUser(); $action_map = PhabricatorUserLog::getActionTypeMap(); $base_uri = $this->searchBaseURI; $rows = array(); foreach ($logs as $log) { $ip = $log->getRemoteAddr(); $session = substr($log->getSession(), 0, 6); if ($base_uri) { $ip = phutil_tag( 'a', array( - 'href' => $base_uri.'?ip='.$log->getRemoteAddr().'#R', + 'href' => $base_uri.'?ip='.$ip.'#R', ), $ip); $session = phutil_tag( 'a', array( - 'href' => $base_uri.'?sessions='.$log->getSession().'#R', + 'href' => $base_uri.'?sessions='.$ip.'#R', ), $session); } $action = $log->getAction(); $action_name = idx($action_map, $action, $action); $rows[] = array( phabricator_date($log->getDateCreated(), $viewer), phabricator_time($log->getDateCreated(), $viewer), $action_name, $log->getActorPHID() ? $handles[$log->getActorPHID()]->getName() : null, $handles[$log->getUserPHID()]->getName(), $ip, $session, ); } $table = new AphrontTableView($rows); $table->setHeaders( array( pht('Date'), pht('Time'), pht('Action'), pht('Actor'), pht('User'), pht('IP'), pht('Session'), )); $table->setColumnClasses( array( '', 'right', 'wide', '', '', '', 'n', )); return $table; } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryPullEvent.php b/src/applications/repository/storage/PhabricatorRepositoryPullEvent.php index d17fded9a8..c1227402d7 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPullEvent.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPullEvent.php @@ -1,87 +1,87 @@ setPusherPHID($viewer->getPHID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_TIMESTAMPS => false, self::CONFIG_SERIALIZATION => array( 'properties' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'repositoryPHID' => 'phid?', 'pullerPHID' => 'phid?', - 'remoteAddress' => 'uint32?', + 'remoteAddress' => 'ipaddress?', 'remoteProtocol' => 'text32?', 'resultType' => 'text32', 'resultCode' => 'uint32', ), self::CONFIG_KEY_SCHEMA => array( 'key_repository' => array( 'columns' => array('repositoryPHID'), ), 'key_epoch' => array( 'columns' => array('epoch'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryPullEventPHIDType::TYPECONST); } public function attachRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->assertAttached($this->repository); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getRepository()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getRepository()->hasAutomaticCapability($capability, $viewer); } public function describeAutomaticCapability($capability) { return pht( "A repository's pull events are visible to users who can see the ". "repository."); } } diff --git a/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php b/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php index 2455499b88..2bc751ffca 100644 --- a/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php +++ b/src/applications/repository/storage/PhabricatorRepositoryPushEvent.php @@ -1,92 +1,92 @@ setPusherPHID($viewer->getPHID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_TIMESTAMPS => false, self::CONFIG_COLUMN_SCHEMA => array( - 'remoteAddress' => 'uint32?', + 'remoteAddress' => 'ipaddress?', 'remoteProtocol' => 'text32?', 'rejectCode' => 'uint32', 'rejectDetails' => 'text64?', ), self::CONFIG_KEY_SCHEMA => array( 'key_repository' => array( 'columns' => array('repositoryPHID'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( PhabricatorRepositoryPushEventPHIDType::TYPECONST); } public function attachRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->assertAttached($this->repository); } public function attachLogs(array $logs) { $this->logs = $logs; return $this; } public function getLogs() { return $this->assertAttached($this->logs); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getRepository()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getRepository()->hasAutomaticCapability($capability, $viewer); } public function describeAutomaticCapability($capability) { return pht( "A repository's push events are visible to users who can see the ". "repository."); } }