diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php index 3e02c78077..f86458c26b 100644 --- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php @@ -1,407 +1,407 @@ getPHID(); if (!$phid) { throw new Exception(pht('Document has no PHID!')); } $store = new PhabricatorSearchDocument(); $store->setPHID($doc->getPHID()); $store->setDocumentType($doc->getDocumentType()); $store->setDocumentTitle($doc->getDocumentTitle()); $store->setDocumentCreated($doc->getDocumentCreated()); $store->setDocumentModified($doc->getDocumentModified()); $store->replace(); $conn_w = $store->establishConnection('w'); $stemmer = new PhutilSearchStemmer(); $field_dao = new PhabricatorSearchDocumentField(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $field_dao->getTableName(), $phid); foreach ($doc->getFieldData() as $field) { list($ftype, $corpus, $aux_phid) = $field; $stemmed_corpus = $stemmer->stemCorpus($corpus); queryfx( $conn_w, 'INSERT INTO %T (phid, phidType, field, auxPHID, corpus, stemmedCorpus) '. 'VALUES (%s, %s, %s, %ns, %s, %s)', $field_dao->getTableName(), $phid, $doc->getDocumentType(), $ftype, $aux_phid, $corpus, $stemmed_corpus); } $sql = array(); foreach ($doc->getRelationshipData() as $relationship) { list($rtype, $to_phid, $to_type, $time) = $relationship; $sql[] = qsprintf( $conn_w, '(%s, %s, %s, %s, %d)', $phid, $to_phid, $rtype, $to_type, $time); } $rship_dao = new PhabricatorSearchDocumentRelationship(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $rship_dao->getTableName(), $phid); if ($sql) { queryfx( $conn_w, 'INSERT INTO %T '. '(phid, relatedPHID, relation, relatedType, relatedTime) '. 'VALUES %Q', $rship_dao->getTableName(), implode(', ', $sql)); } } /** * Rebuild the PhabricatorSearchAbstractDocument that was used to index * an object out of the index itself. This is primarily useful for debugging, * as it allows you to inspect the search index representation of a * document. * * @param phid PHID of a document which exists in the search index. * @return null|PhabricatorSearchAbstractDocument Abstract document object * which corresponds to the original abstract document used to * build the document index. */ public function reconstructDocument($phid) { $dao_doc = new PhabricatorSearchDocument(); $dao_field = new PhabricatorSearchDocumentField(); $dao_relationship = new PhabricatorSearchDocumentRelationship(); $t_doc = $dao_doc->getTableName(); $t_field = $dao_field->getTableName(); $t_relationship = $dao_relationship->getTableName(); $doc = queryfx_one( $dao_doc->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_doc, $phid); if (!$doc) { return null; } $fields = queryfx_all( $dao_field->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_field, $phid); $relationships = queryfx_all( $dao_relationship->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_relationship, $phid); $adoc = id(new PhabricatorSearchAbstractDocument()) ->setPHID($phid) ->setDocumentType($doc['documentType']) ->setDocumentTitle($doc['documentTitle']) ->setDocumentCreated($doc['documentCreated']) ->setDocumentModified($doc['documentModified']); foreach ($fields as $field) { $adoc->addField( $field['field'], $field['corpus'], $field['auxPHID']); } foreach ($relationships as $relationship) { $adoc->addRelationship( $relationship['relation'], $relationship['relatedPHID'], $relationship['relatedType'], $relationship['relatedTime']); } return $adoc; } public function executeSearch(PhabricatorSavedQuery $query) { $table = new PhabricatorSearchDocument(); $document_table = $table->getTableName(); $conn = $table->establishConnection('r'); $subquery = $this->newFulltextSubquery($query, $conn); $offset = (int)$query->getParameter('offset', 0); $limit = (int)$query->getParameter('limit', 25); // NOTE: We must JOIN the subquery in order to apply a limit. $results = queryfx_all( $conn, 'SELECT documentPHID, MAX(fieldScore) AS documentScore FROM (%Q) query JOIN %T root ON query.documentPHID = root.phid GROUP BY documentPHID ORDER BY documentScore DESC LIMIT %d, %d', $subquery, $document_table, $offset, $limit); return ipull($results, 'documentPHID'); } private function newFulltextSubquery( PhabricatorSavedQuery $query, AphrontDatabaseConnection $conn) { $field = new PhabricatorSearchDocumentField(); $field_table = $field->getTableName(); $document = new PhabricatorSearchDocument(); $document_table = $document->getTableName(); $select = array(); $select[] = 'document.phid AS documentPHID'; $join = array(); $where = array(); $title_field = PhabricatorSearchDocumentFieldType::FIELD_TITLE; $title_boost = 1024; $raw_query = $query->getParameter('query'); $compiled_query = $this->compileQuery($raw_query); if (strlen($compiled_query)) { $select[] = qsprintf( $conn, 'IF(field.field = %s, %d, 0) + MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE) AS fieldScore', $title_field, $title_boost, $compiled_query); $join[] = qsprintf( $conn, '%T field ON field.phid = document.phid', $field_table); $where[] = qsprintf( $conn, 'MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE)', $compiled_query); if ($query->getParameter('field')) { $where[] = qsprintf( $conn, 'field.field = %s', $field); } } else { $select[] = qsprintf( $conn, - 'document.dateCreated AS fieldScore'); + 'document.documentCreated AS fieldScore'); } $exclude = $query->getParameter('exclude'); if ($exclude) { $where[] = qsprintf( $conn, 'document.phid != %s', $exclude); } $types = $query->getParameter('types'); if ($types) { if (strlen($compiled_query)) { $where[] = qsprintf( $conn, 'field.phidType IN (%Ls)', $types); } $where[] = qsprintf( $conn, 'document.documentType IN (%Ls)', $types); } $join[] = $this->joinRelationship( $conn, $query, 'authorPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR); $statuses = $query->getParameter('statuses', array()); $statuses = array_fuse($statuses); $open_rel = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; $closed_rel = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED; $include_open = !empty($statuses[$open_rel]); $include_closed = !empty($statuses[$closed_rel]); if ($include_open && !$include_closed) { $join[] = $this->joinRelationship( $conn, $query, 'statuses', $open_rel, true); } else if ($include_closed && !$include_open) { $join[] = $this->joinRelationship( $conn, $query, 'statuses', $closed_rel, true); } if ($query->getParameter('withAnyOwner')) { $join[] = $this->joinRelationship( $conn, $query, 'withAnyOwner', PhabricatorSearchRelationship::RELATIONSHIP_OWNER, true); } else if ($query->getParameter('withUnowned')) { $join[] = $this->joinRelationship( $conn, $query, 'withUnowned', PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED, true); } else { $join[] = $this->joinRelationship( $conn, $query, 'ownerPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_OWNER); } $join[] = $this->joinRelationship( $conn, $query, 'subscriberPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER); $join[] = $this->joinRelationship( $conn, $query, 'projectPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_PROJECT); $join[] = $this->joinRelationship( $conn, $query, 'repository', PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY); $select = implode(', ', $select); $join = array_filter($join); foreach ($join as $key => $clause) { $join[$key] = ' JOIN '.$clause; } $join = implode(' ', $join); if ($where) { $where = 'WHERE '.implode(' AND ', $where); } else { $where = ''; } return qsprintf( $conn, 'SELECT %Q FROM %T document %Q %Q LIMIT 1000', $select, $document_table, $join, $where); } protected function joinRelationship( AphrontDatabaseConnection $conn, PhabricatorSavedQuery $query, $field, $type, $is_existence = false) { $sql = qsprintf( $conn, '%T AS %C ON %C.phid = document.phid AND %C.relation = %s', id(new PhabricatorSearchDocumentRelationship())->getTableName(), $field, $field, $field, $type); if (!$is_existence) { $phids = $query->getParameter($field, array()); if (!$phids) { return null; } $sql .= qsprintf( $conn, ' AND %C.relatedPHID in (%Ls)', $field, $phids); } return $sql; } private function compileQuery($raw_query) { $stemmer = new PhutilSearchStemmer(); $compiler = PhabricatorSearchDocument::newQueryCompiler() ->setQuery($raw_query) ->setStemmer($stemmer); $queries = array(); $queries[] = $compiler->compileLiteralQuery(); $queries[] = $compiler->compileStemmedQuery(); return implode(' ', array_filter($queries)); } public function indexExists() { return true; } } diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php index a4f0259060..337d139df8 100644 --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -1,721 +1,721 @@ host = $host; return $this; } public function getHost() { return $this->host; } public function setPort($port) { $this->port = $port; return $this; } public function getPort() { return $this->port; } public function setUser($user) { $this->user = $user; return $this; } public function getUser() { return $this->user; } public function setPass(PhutilOpaqueEnvelope $pass) { $this->pass = $pass; return $this; } public function getPass() { return $this->pass; } public function setIsMaster($is_master) { $this->isMaster = $is_master; return $this; } public function getIsMaster() { return $this->isMaster; } public function setDisabled($disabled) { $this->disabled = $disabled; return $this; } public function getDisabled() { return $this->disabled; } public function setConnectionLatency($connection_latency) { $this->connectionLatency = $connection_latency; return $this; } public function getConnectionLatency() { return $this->connectionLatency; } public function setConnectionStatus($connection_status) { $this->connectionStatus = $connection_status; return $this; } public function getConnectionStatus() { if ($this->connectionStatus === null) { throw new PhutilInvalidStateException('queryAll'); } return $this->connectionStatus; } public function setConnectionMessage($connection_message) { $this->connectionMessage = $connection_message; return $this; } public function getConnectionMessage() { return $this->connectionMessage; } public function setReplicaStatus($replica_status) { $this->replicaStatus = $replica_status; return $this; } public function getReplicaStatus() { return $this->replicaStatus; } public function setReplicaMessage($replica_message) { $this->replicaMessage = $replica_message; return $this; } public function getReplicaMessage() { return $this->replicaMessage; } public function setReplicaDelay($replica_delay) { $this->replicaDelay = $replica_delay; return $this; } public function getReplicaDelay() { return $this->replicaDelay; } public function setIsIndividual($is_individual) { $this->isIndividual = $is_individual; return $this; } public function getIsIndividual() { return $this->isIndividual; } public function setIsDefaultPartition($is_default_partition) { $this->isDefaultPartition = $is_default_partition; return $this; } public function getIsDefaultPartition() { return $this->isDefaultPartition; } public function setUsePersistentConnections($use_persistent_connections) { $this->usePersistentConnections = $use_persistent_connections; return $this; } public function getUsePersistentConnections() { return $this->usePersistentConnections; } public function setApplicationMap(array $application_map) { $this->applicationMap = $application_map; return $this; } public function getApplicationMap() { return $this->applicationMap; } public function getPartitionStateForCommit() { $state = PhabricatorEnv::getEnvConfig('cluster.databases'); foreach ($state as $key => $value) { // Don't store passwords, since we don't care if they differ and // users may find it surprising. unset($state[$key]['pass']); } return phutil_json_encode($state); } public function setMasterRef(PhabricatorDatabaseRef $master_ref) { $this->masterRef = $master_ref; return $this; } public function getMasterRef() { return $this->masterRef; } public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) { $this->replicaRefs[] = $replica_ref; return $this; } public function getReplicaRefs() { return $this->replicaRefs; } public function getRefKey() { $host = $this->getHost(); $port = $this->getPort(); if (strlen($port)) { return "{$host}:{$port}"; } return $host; } public static function getConnectionStatusMap() { return array( self::STATUS_OKAY => array( 'icon' => 'fa-exchange', 'color' => 'green', 'label' => pht('Okay'), ), self::STATUS_FAIL => array( 'icon' => 'fa-times', 'color' => 'red', 'label' => pht('Failed'), ), self::STATUS_AUTH => array( 'icon' => 'fa-key', 'color' => 'red', 'label' => pht('Invalid Credentials'), ), self::STATUS_REPLICATION_CLIENT => array( 'icon' => 'fa-eye-slash', 'color' => 'yellow', 'label' => pht('Missing Permission'), ), ); } public static function getReplicaStatusMap() { return array( self::REPLICATION_OKAY => array( 'icon' => 'fa-download', 'color' => 'green', 'label' => pht('Okay'), ), self::REPLICATION_MASTER_REPLICA => array( 'icon' => 'fa-database', 'color' => 'red', 'label' => pht('Replicating Master'), ), self::REPLICATION_REPLICA_NONE => array( 'icon' => 'fa-download', 'color' => 'red', 'label' => pht('Not A Replica'), ), self::REPLICATION_SLOW => array( 'icon' => 'fa-hourglass', 'color' => 'red', 'label' => pht('Slow Replication'), ), self::REPLICATION_NOT_REPLICATING => array( 'icon' => 'fa-exclamation-triangle', 'color' => 'red', 'label' => pht('Not Replicating'), ), ); } public static function getClusterRefs() { $cache = PhabricatorCaches::getRequestCache(); $refs = $cache->getKey(self::KEY_REFS); if (!$refs) { $refs = self::newRefs(); $cache->setKey(self::KEY_REFS, $refs); } return $refs; } public static function getLiveIndividualRef() { $cache = PhabricatorCaches::getRequestCache(); $ref = $cache->getKey(self::KEY_INDIVIDUAL); if (!$ref) { $ref = self::newIndividualRef(); $cache->setKey(self::KEY_INDIVIDUAL, $ref); } return $ref; } public static function newRefs() { $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); $default_port = nonempty($default_port, 3306); $default_user = PhabricatorEnv::getEnvConfig('mysql.user'); $default_pass = PhabricatorEnv::getEnvConfig('mysql.pass'); $default_pass = new PhutilOpaqueEnvelope($default_pass); $config = PhabricatorEnv::getEnvConfig('cluster.databases'); return id(new PhabricatorDatabaseRefParser()) ->setDefaultPort($default_port) ->setDefaultUser($default_user) ->setDefaultPass($default_pass) ->newRefs($config); } public static function queryAll() { $refs = self::getActiveDatabaseRefs(); return self::queryRefs($refs); } private static function queryRefs(array $refs) { foreach ($refs as $ref) { $conn = $ref->newManagementConnection(); $t_start = microtime(true); $replica_status = false; try { $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS'); $ref->setConnectionStatus(self::STATUS_OKAY); } catch (AphrontAccessDeniedQueryException $ex) { $ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT); $ref->setConnectionMessage( pht( 'No permission to run "SHOW SLAVE STATUS". Grant this user '. '"REPLICATION CLIENT" permission to allow Phabricator to '. 'monitor replica health.')); } catch (AphrontInvalidCredentialsQueryException $ex) { $ref->setConnectionStatus(self::STATUS_AUTH); $ref->setConnectionMessage($ex->getMessage()); } catch (AphrontQueryException $ex) { $ref->setConnectionStatus(self::STATUS_FAIL); $class = get_class($ex); $message = $ex->getMessage(); $ref->setConnectionMessage( pht( '%s: %s', get_class($ex), $ex->getMessage())); } $t_end = microtime(true); $ref->setConnectionLatency($t_end - $t_start); if ($replica_status !== false) { $is_replica = (bool)$replica_status; if ($ref->getIsMaster() && $is_replica) { $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA); $ref->setReplicaMessage( pht( 'This host has a "master" role, but is replicating data from '. 'another host ("%s")!', idx($replica_status, 'Master_Host'))); } else if (!$ref->getIsMaster() && !$is_replica) { $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE); $ref->setReplicaMessage( pht( 'This host has a "replica" role, but is not replicating data '. 'from a master (no output from "SHOW SLAVE STATUS").')); } else { $ref->setReplicaStatus(self::REPLICATION_OKAY); } if ($is_replica) { $latency = idx($replica_status, 'Seconds_Behind_Master'); if (!strlen($latency)) { $ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING); } else { $latency = (int)$latency; $ref->setReplicaDelay($latency); if ($latency > 30) { $ref->setReplicaStatus(self::REPLICATION_SLOW); $ref->setReplicaMessage( pht( 'This replica is lagging far behind the master. Data is at '. 'risk!')); } } } } } return $refs; } public function newManagementConnection() { return $this->newConnection( array( 'retries' => 0, 'timeout' => 2, )); } public function newApplicationConnection($database) { return $this->newConnection( array( 'database' => $database, )); } public function isSevered() { // If we only have an individual database, never sever our connection to // it, at least for now. It's possible that using the same severing rules // might eventually make sense to help alleviate load-related failures, // but we should wait for all the cluster stuff to stabilize first. if ($this->getIsIndividual()) { return false; } if ($this->didFailToConnect) { return true; } $record = $this->getHealthRecord(); $is_healthy = $record->getIsHealthy(); if (!$is_healthy) { return true; } return false; } public function isReachable(AphrontDatabaseConnection $connection) { $record = $this->getHealthRecord(); $should_check = $record->getShouldCheck(); if ($this->isSevered() && !$should_check) { return false; } try { $connection->openConnection(); $reachable = true; } catch (AphrontSchemaQueryException $ex) { // We get one of these if the database we're trying to select does not // exist. In this case, just re-throw the exception. This is expected // during first-time setup, when databases like "config" will not exist // yet. throw $ex; } catch (Exception $ex) { $reachable = false; } if ($should_check) { $record->didHealthCheck($reachable); } if (!$reachable) { $this->didFailToConnect = true; } return $reachable; } public function checkHealth() { $health = $this->getHealthRecord(); $should_check = $health->getShouldCheck(); if ($should_check) { // This does an implicit health update. $connection = $this->newManagementConnection(); $this->isReachable($connection); } return $this; } public function getHealthRecord() { if (!$this->healthRecord) { $this->healthRecord = new PhabricatorDatabaseHealthRecord($this); } return $this->healthRecord; } public static function getActiveDatabaseRefs() { $refs = array(); foreach (self::getMasterDatabaseRefs() as $ref) { $refs[] = $ref; } foreach (self::getReplicaDatabaseRefs() as $ref) { $refs[] = $ref; } return $refs; } public static function getAllMasterDatabaseRefs() { $refs = self::getClusterRefs(); if (!$refs) { return array(self::getLiveIndividualRef()); } $masters = array(); foreach ($refs as $ref) { if ($ref->getIsMaster()) { $masters[] = $ref; } } return $masters; } public static function getMasterDatabaseRefs() { $refs = self::getAllMasterDatabaseRefs(); return self::getEnabledRefs($refs); } public function isApplicationHost($database) { return isset($this->applicationMap[$database]); } public function loadRawMySQLConfigValue($key) { $conn = $this->newManagementConnection(); try { $value = queryfx_one($conn, 'SELECT @@%Q', $key); $value = $value['@@'.$key]; } catch (AphrontQueryException $ex) { $value = null; } return $value; } public static function getMasterDatabaseRefForApplication($application) { $masters = self::getMasterDatabaseRefs(); $application_master = null; $default_master = null; foreach ($masters as $master) { if ($master->isApplicationHost($application)) { $application_master = $master; break; } if ($master->getIsDefaultPartition()) { $default_master = $master; } } if ($application_master) { $masters = array($application_master); } else if ($default_master) { $masters = array($default_master); } else { $masters = array(); } $masters = self::getEnabledRefs($masters); $master = head($masters); return $master; } public static function newIndividualRef() { $default_user = PhabricatorEnv::getEnvConfig('mysql.user'); $default_pass = new PhutilOpaqueEnvelope( PhabricatorEnv::getEnvConfig('mysql.pass')); $default_host = PhabricatorEnv::getEnvConfig('mysql.host'); $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); return id(new self()) ->setUser($default_user) ->setPass($default_pass) ->setHost($default_host) ->setPort($default_port) ->setIsIndividual(true) ->setIsMaster(true) ->setIsDefaultPartition(true) ->setUsePersistentConnections(false); } public static function getAllReplicaDatabaseRefs() { $refs = self::getClusterRefs(); if (!$refs) { return array(); } $replicas = array(); foreach ($refs as $ref) { if ($ref->getIsMaster()) { continue; } $replicas[] = $ref; } return $replicas; } public static function getReplicaDatabaseRefs() { $refs = self::getAllReplicaDatabaseRefs(); return self::getEnabledRefs($refs); } private static function getEnabledRefs(array $refs) { foreach ($refs as $key => $ref) { if ($ref->getDisabled()) { unset($refs[$key]); } } return $refs; } public static function getReplicaDatabaseRefForApplication($application) { $replicas = self::getReplicaDatabaseRefs(); $application_replicas = array(); $default_replicas = array(); foreach ($replicas as $replica) { - $master = $replica->getMaster(); + $master = $replica->getMasterRef(); if ($master->isApplicationHost($application)) { $application_replicas[] = $replica; } if ($master->getIsDefaultPartition()) { $default_replicas[] = $replica; } } if ($application_replicas) { $replicas = $application_replicas; } else { $replicas = $default_replicas; } $replicas = self::getEnabledRefs($replicas); // TODO: We may have multiple replicas to choose from, and could make // more of an effort to pick the "best" one here instead of always // picking the first one. Once we've picked one, we should try to use // the same replica for the rest of the request, though. return head($replicas); } private function newConnection(array $options) { // If we believe the database is unhealthy, don't spend as much time // trying to connect to it, since it's likely to continue to fail and // hammering it can only make the problem worse. $record = $this->getHealthRecord(); if ($record->getIsHealthy()) { $default_retries = 3; $default_timeout = 10; } else { $default_retries = 0; $default_timeout = 2; } $spec = $options + array( 'user' => $this->getUser(), 'pass' => $this->getPass(), 'host' => $this->getHost(), 'port' => $this->getPort(), 'database' => null, 'retries' => $default_retries, 'timeout' => $default_timeout, 'persistent' => $this->getUsePersistentConnections(), ); $is_cli = (php_sapi_name() == 'cli'); $use_persistent = false; if (!empty($spec['persistent']) && !$is_cli) { $use_persistent = true; } unset($spec['persistent']); $connection = self::newRawConnection($spec); // If configured, use persistent connections. See T11672 for details. if ($use_persistent) { $connection->setPersistent($use_persistent); } // Unless this is a script running from the CLI, prevent any query from // running for more than 30 seconds. See T10849 for details. if (!$is_cli) { $connection->setQueryTimeout(30); } return $connection; } public static function newRawConnection(array $options) { if (extension_loaded('mysqli')) { return new AphrontMySQLiDatabaseConnection($options); } else { return new AphrontMySQLDatabaseConnection($options); } } }