diff --git a/src/applications/conpherence/query/ConpherenceFulltextQuery.php b/src/applications/conpherence/query/ConpherenceFulltextQuery.php index 43de7455ae..ba734049f8 100644 --- a/src/applications/conpherence/query/ConpherenceFulltextQuery.php +++ b/src/applications/conpherence/query/ConpherenceFulltextQuery.php @@ -1,85 +1,85 @@ threadPHIDs = $phids; return $this; } public function withPreviousTransactionPHIDs(array $phids) { $this->previousTransactionPHIDs = $phids; return $this; } public function withFulltext($fulltext) { $this->fulltext = $fulltext; return $this; } public function execute() { $table = new ConpherenceIndex(); $conn_r = $table->establishConnection('r'); $rows = queryfx_all( $conn_r, 'SELECT threadPHID, transactionPHID, previousTransactionPHID FROM %T i %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderByClause($conn_r), $this->buildLimitClause($conn_r)); return $rows; } protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->threadPHIDs !== null) { $where[] = qsprintf( $conn_r, 'i.threadPHID IN (%Ls)', $this->threadPHIDs); } if ($this->previousTransactionPHIDs !== null) { $where[] = qsprintf( $conn_r, 'i.previousTransactionPHID IN (%Ls)', $this->previousTransactionPHIDs); } if (strlen($this->fulltext)) { - $compiled_query = PhabricatorSearchDocument::newQueryCompiler() - ->setQuery($this->fulltext) - ->compileQuery(); + $compiler = PhabricatorSearchDocument::newQueryCompiler(); + $tokens = $compiler->newTokens($this->fulltext); + $compiled_query = $compiler->compileQuery($tokens); $where[] = qsprintf( $conn_r, 'MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE)', $compiled_query); } return $this->formatWhereClause($where); } private function buildOrderByClause(AphrontDatabaseConnection $conn_r) { if (strlen($this->fulltext)) { return qsprintf( $conn_r, 'ORDER BY MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE) DESC', $this->fulltext); } else { return qsprintf( $conn_r, 'ORDER BY id DESC'); } } } diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php index 72c49576f0..626c7435c3 100644 --- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php @@ -1,419 +1,420 @@ getPHID(); if (!$phid) { throw new Exception(pht('Document has no PHID!')); } $store = new PhabricatorSearchDocument(); $store->setPHID($doc->getPHID()); $store->setDocumentType($doc->getDocumentType()); $store->setDocumentTitle($doc->getDocumentTitle()); $store->setDocumentCreated($doc->getDocumentCreated()); $store->setDocumentModified($doc->getDocumentModified()); $store->replace(); $conn_w = $store->establishConnection('w'); $stemmer = new PhutilSearchStemmer(); $field_dao = new PhabricatorSearchDocumentField(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $field_dao->getTableName(), $phid); foreach ($doc->getFieldData() as $field) { list($ftype, $corpus, $aux_phid) = $field; $stemmed_corpus = $stemmer->stemCorpus($corpus); queryfx( $conn_w, 'INSERT INTO %T (phid, phidType, field, auxPHID, corpus, stemmedCorpus) '. 'VALUES (%s, %s, %s, %ns, %s, %s)', $field_dao->getTableName(), $phid, $doc->getDocumentType(), $ftype, $aux_phid, $corpus, $stemmed_corpus); } $sql = array(); foreach ($doc->getRelationshipData() as $relationship) { list($rtype, $to_phid, $to_type, $time) = $relationship; $sql[] = qsprintf( $conn_w, '(%s, %s, %s, %s, %d)', $phid, $to_phid, $rtype, $to_type, $time); } $rship_dao = new PhabricatorSearchDocumentRelationship(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $rship_dao->getTableName(), $phid); if ($sql) { queryfx( $conn_w, 'INSERT INTO %T '. '(phid, relatedPHID, relation, relatedType, relatedTime) '. 'VALUES %Q', $rship_dao->getTableName(), implode(', ', $sql)); } } /** * Rebuild the PhabricatorSearchAbstractDocument that was used to index * an object out of the index itself. This is primarily useful for debugging, * as it allows you to inspect the search index representation of a * document. * * @param phid PHID of a document which exists in the search index. * @return null|PhabricatorSearchAbstractDocument Abstract document object * which corresponds to the original abstract document used to * build the document index. */ public function reconstructDocument($phid) { $dao_doc = new PhabricatorSearchDocument(); $dao_field = new PhabricatorSearchDocumentField(); $dao_relationship = new PhabricatorSearchDocumentRelationship(); $t_doc = $dao_doc->getTableName(); $t_field = $dao_field->getTableName(); $t_relationship = $dao_relationship->getTableName(); $doc = queryfx_one( $dao_doc->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_doc, $phid); if (!$doc) { return null; } $fields = queryfx_all( $dao_field->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_field, $phid); $relationships = queryfx_all( $dao_relationship->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_relationship, $phid); $adoc = id(new PhabricatorSearchAbstractDocument()) ->setPHID($phid) ->setDocumentType($doc['documentType']) ->setDocumentTitle($doc['documentTitle']) ->setDocumentCreated($doc['documentCreated']) ->setDocumentModified($doc['documentModified']); foreach ($fields as $field) { $adoc->addField( $field['field'], $field['corpus'], $field['auxPHID']); } foreach ($relationships as $relationship) { $adoc->addRelationship( $relationship['relation'], $relationship['relatedPHID'], $relationship['relatedType'], $relationship['relatedTime']); } return $adoc; } public function executeSearch(PhabricatorSavedQuery $query) { $table = new PhabricatorSearchDocument(); $document_table = $table->getTableName(); $conn = $table->establishConnection('r'); $subquery = $this->newFulltextSubquery($query, $conn); $offset = (int)$query->getParameter('offset', 0); $limit = (int)$query->getParameter('limit', 25); // NOTE: We must JOIN the subquery in order to apply a limit. $results = queryfx_all( $conn, 'SELECT documentPHID, MAX(fieldScore) AS documentScore FROM (%Q) query JOIN %T root ON query.documentPHID = root.phid GROUP BY documentPHID ORDER BY documentScore DESC LIMIT %d, %d', $subquery, $document_table, $offset, $limit); return ipull($results, 'documentPHID'); } private function newFulltextSubquery( PhabricatorSavedQuery $query, AphrontDatabaseConnection $conn) { $field = new PhabricatorSearchDocumentField(); $field_table = $field->getTableName(); $document = new PhabricatorSearchDocument(); $document_table = $document->getTableName(); $select = array(); $select[] = 'document.phid AS documentPHID'; $join = array(); $where = array(); $title_field = PhabricatorSearchDocumentFieldType::FIELD_TITLE; $title_boost = 1024; $raw_query = $query->getParameter('query'); $compiled_query = $this->compileQuery($raw_query); if (strlen($compiled_query)) { $select[] = qsprintf( $conn, 'IF(field.field = %s, %d, 0) + MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE) AS fieldScore', $title_field, $title_boost, $compiled_query); $join[] = qsprintf( $conn, '%T field ON field.phid = document.phid', $field_table); $where[] = qsprintf( $conn, 'MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE)', $compiled_query); if ($query->getParameter('field')) { $where[] = qsprintf( $conn, 'field.field = %s', $field); } } else { $select[] = qsprintf( $conn, 'document.documentCreated AS fieldScore'); } $exclude = $query->getParameter('exclude'); if ($exclude) { $where[] = qsprintf( $conn, 'document.phid != %s', $exclude); } $types = $query->getParameter('types'); if ($types) { if (strlen($compiled_query)) { $where[] = qsprintf( $conn, 'field.phidType IN (%Ls)', $types); } $where[] = qsprintf( $conn, 'document.documentType IN (%Ls)', $types); } $join[] = $this->joinRelationship( $conn, $query, 'authorPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR); $statuses = $query->getParameter('statuses', array()); $statuses = array_fuse($statuses); $open_rel = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; $closed_rel = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED; $include_open = !empty($statuses[$open_rel]); $include_closed = !empty($statuses[$closed_rel]); if ($include_open && !$include_closed) { $join[] = $this->joinRelationship( $conn, $query, 'statuses', $open_rel, true); } else if ($include_closed && !$include_open) { $join[] = $this->joinRelationship( $conn, $query, 'statuses', $closed_rel, true); } if ($query->getParameter('withAnyOwner')) { $join[] = $this->joinRelationship( $conn, $query, 'withAnyOwner', PhabricatorSearchRelationship::RELATIONSHIP_OWNER, true); } else if ($query->getParameter('withUnowned')) { $join[] = $this->joinRelationship( $conn, $query, 'withUnowned', PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED, true); } else { $join[] = $this->joinRelationship( $conn, $query, 'ownerPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_OWNER); } $join[] = $this->joinRelationship( $conn, $query, 'subscriberPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER); $join[] = $this->joinRelationship( $conn, $query, 'projectPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_PROJECT); $join[] = $this->joinRelationship( $conn, $query, 'repository', PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY); $select = implode(', ', $select); $join = array_filter($join); foreach ($join as $key => $clause) { $join[$key] = ' JOIN '.$clause; } $join = implode(' ', $join); if ($where) { $where = 'WHERE '.implode(' AND ', $where); } else { $where = ''; } if (strlen($compiled_query)) { $order = ''; } else { // When not executing a query, order by document creation date. This // is the default view in object browser dialogs, like "Close Duplicate". $order = qsprintf( $conn, 'ORDER BY document.documentCreated DESC'); } return qsprintf( $conn, 'SELECT %Q FROM %T document %Q %Q %Q LIMIT 1000', $select, $document_table, $join, $where, $order); } protected function joinRelationship( AphrontDatabaseConnection $conn, PhabricatorSavedQuery $query, $field, $type, $is_existence = false) { $sql = qsprintf( $conn, '%T AS %C ON %C.phid = document.phid AND %C.relation = %s', id(new PhabricatorSearchDocumentRelationship())->getTableName(), $field, $field, $field, $type); if (!$is_existence) { $phids = $query->getParameter($field, array()); if (!$phids) { return null; } $sql .= qsprintf( $conn, ' AND %C.relatedPHID in (%Ls)', $field, $phids); } return $sql; } private function compileQuery($raw_query) { $stemmer = new PhutilSearchStemmer(); $compiler = PhabricatorSearchDocument::newQueryCompiler() - ->setQuery($raw_query) ->setStemmer($stemmer); + $tokens = $compiler->newTokens($raw_query); + $queries = array(); - $queries[] = $compiler->compileLiteralQuery(); - $queries[] = $compiler->compileStemmedQuery(); + $queries[] = $compiler->compileLiteralQuery($tokens); + $queries[] = $compiler->compileStemmedQuery($tokens); return implode(' ', array_filter($queries)); } public function indexExists() { return true; } public function getIndexStats() { return false; } } diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php index a9ceb0e7e5..8e5b296132 100644 --- a/src/infrastructure/cluster/search/PhabricatorSearchService.php +++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php @@ -1,264 +1,268 @@ engine = $engine; $this->hostType = $engine->getHostType(); } /** * @throws Exception */ public function newHost($config) { $host = clone($this->hostType); $host_config = $this->config + $config; $host->setConfig($host_config); $this->hosts[] = $host; return $host; } public function getEngine() { return $this->engine; } public function getDisplayName() { return $this->hostType->getDisplayName(); } public function getStatusViewColumns() { return $this->hostType->getStatusViewColumns(); } public function setConfig($config) { $this->config = $config; if (!isset($config['hosts'])) { $config['hosts'] = array( array( 'host' => idx($config, 'host'), 'port' => idx($config, 'port'), 'protocol' => idx($config, 'protocol'), 'roles' => idx($config, 'roles'), ), ); } foreach ($config['hosts'] as $host) { $this->newHost($host); } } public function getConfig() { return $this->config; } public static function getConnectionStatusMap() { return array( self::STATUS_OKAY => array( 'icon' => 'fa-exchange', 'color' => 'green', 'label' => pht('Okay'), ), self::STATUS_FAIL => array( 'icon' => 'fa-times', 'color' => 'red', 'label' => pht('Failed'), ), ); } public function isWritable() { return (bool)$this->getAllHostsForRole(self::ROLE_WRITE); } public function isReadable() { return (bool)$this->getAllHostsForRole(self::ROLE_READ); } public function getPort() { return idx($this->config, 'port'); } public function getProtocol() { return idx($this->config, 'protocol'); } public function getVersion() { return idx($this->config, 'version'); } public function getHosts() { return $this->hosts; } /** * Get a random host reference with the specified role, skipping hosts which * failed recent health checks. * @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match. * @return PhabricatorSearchHost */ public function getAnyHostForRole($role) { $hosts = $this->getAllHostsForRole($role); shuffle($hosts); foreach ($hosts as $host) { $health = $host->getHealthRecord(); if ($health->getIsHealthy()) { return $host; } } throw new PhabricatorClusterNoHostForRoleException($role); } /** * Get all configured hosts for this service which have the specified role. * @return PhabricatorSearchHost[] */ public function getAllHostsForRole($role) { // if the role is explicitly set to false at the top level, then all hosts // have the role disabled. if (idx($this->config, $role) === false) { return array(); } $hosts = array(); foreach ($this->hosts as $host) { if ($host->hasRole($role)) { $hosts[] = $host; } } return $hosts; } /** * Get a reference to all configured fulltext search cluster services * @return PhabricatorSearchService[] */ public static function getAllServices() { $cache = PhabricatorCaches::getRequestCache(); $refs = $cache->getKey(self::KEY_REFS); if (!$refs) { $refs = self::newRefs(); $cache->setKey(self::KEY_REFS, $refs); } return $refs; } /** * Load all valid PhabricatorFulltextStorageEngine subclasses */ public static function loadAllFulltextStorageEngines() { return id(new PhutilClassMapQuery()) ->setAncestorClass('PhabricatorFulltextStorageEngine') ->setUniqueMethod('getEngineIdentifier') ->execute(); } /** * Create instances of PhabricatorSearchService based on configuration * @return PhabricatorSearchService[] */ public static function newRefs() { $services = PhabricatorEnv::getEnvConfig('cluster.search'); $engines = self::loadAllFulltextStorageEngines(); $refs = array(); foreach ($services as $config) { // Normally, we've validated configuration before we get this far, but // make sure we don't fatal if we end up here with a bogus configuration. if (!isset($engines[$config['type']])) { throw new Exception( pht( 'Configured search engine type "%s" is unknown. Valid engines '. 'are: %s.', $config['type'], implode(', ', array_keys($engines)))); } $engine = clone($engines[$config['type']]); $cluster = new self($engine); $cluster->setConfig($config); $engine->setService($cluster); $refs[] = $cluster; } return $refs; } /** * (re)index the document: attempt to pass the document to all writable * fulltext search hosts */ public static function reindexAbstractDocument( PhabricatorSearchAbstractDocument $document) { $exceptions = array(); foreach (self::getAllServices() as $service) { if (!$service->isWritable()) { continue; } $engine = $service->getEngine(); try { $engine->reindexAbstractDocument($document); } catch (Exception $ex) { $exceptions[] = $ex; } } if ($exceptions) { throw new PhutilAggregateException( pht( 'Writes to search services failed while reindexing document "%s".', $document->getPHID()), $exceptions); } } /** * Execute a full-text query and return a list of PHIDs of matching objects. * @return string[] * @throws PhutilAggregateException */ public static function executeSearch(PhabricatorSavedQuery $query) { $exceptions = array(); // try all services until one succeeds foreach (self::getAllServices() as $service) { try { $engine = $service->getEngine(); $res = $engine->executeSearch($query); // return immediately if we get results return $res; + } catch (PhutilSearchQueryCompilerSyntaxException $ex) { + // If there's a query compilation error, return it directly to the + // user: they issued a query with bad syntax. + throw $ex; } catch (Exception $ex) { $exceptions[] = $ex; } } $msg = pht('All of the configured Fulltext Search services failed.'); throw new PhutilAggregateException($msg, $exceptions); } }