diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php index d8864b5740..dab886b92a 100644 --- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -1,80 +1,82 @@ getAnyHostForRole('read'); } catch (PhabricatorClusterNoHostForRoleException $e) { // ignore the error continue; } if ($host instanceof PhabricatorElasticSearchHost) { $index_exists = null; $index_sane = null; try { $engine = $host->getEngine(); $index_exists = $engine->indexExists(); if ($index_exists) { $index_sane = $engine->indexIsSane(); } } catch (Exception $ex) { $summary = pht('Elasticsearch is not reachable as configured.'); $message = pht( 'Elasticsearch is configured (with the %s setting) but Phabricator'. ' encountered an exception when trying to test the index.'. "\n\n". '%s', phutil_tag('tt', array(), 'cluster.search'), phutil_tag('pre', array(), $ex->getMessage())); $this->newIssue('elastic.misconfigured') ->setName(pht('Elasticsearch Misconfigured')) ->setSummary($summary) ->setMessage($message) ->addRelatedPhabricatorConfig('cluster.search'); return; } if (!$index_exists) { $summary = pht( 'You enabled Elasticsearch but the index does not exist.'); $message = pht( 'You likely enabled cluster.search without creating the '. - 'index. Run `./bin/search init` to correct the index.'); + 'index. Use the following command to create a new index.'); $this ->newIssue('elastic.missing-index') - ->setName(pht('Elasticsearch index Not Found')) + ->setName(pht('Elasticsearch Index Not Found')) + ->addCommand('./bin/search init') ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('cluster.search'); + ->setMessage($message); + } else if (!$index_sane) { $summary = pht( 'Elasticsearch index exists but needs correction.'); $message = pht( 'Either the Phabricator schema for Elasticsearch has changed '. - 'or Elasticsearch created the index automatically. Run '. - '`./bin/search init` to correct the index.'); + 'or Elasticsearch created the index automatically. '. + 'Use the following command to rebuild the index.'); $this ->newIssue('elastic.broken-index') - ->setName(pht('Elasticsearch index Incorrect')) + ->setName(pht('Elasticsearch Index Schema Mismatch')) + ->addCommand('./bin/search init') ->setSummary($summary) ->setMessage($message); } } } } } diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php index bc32da5ef4..6d2cd9adc5 100644 --- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -1,542 +1,585 @@ service = $service; $config = $service->getConfig(); $index = idx($config, 'path', '/phabricator'); $this->index = str_replace('/', '', $index); $this->timeout = idx($config, 'timeout', 15); $this->version = (int)idx($config, 'version', 5); return $this; } public function getEngineIdentifier() { return 'elasticsearch'; } public function getTimestampField() { return $this->version < 2 ? '_timestamp' : 'lastModified'; } public function getTextFieldType() { return $this->version >= 5 ? 'text' : 'string'; } public function getHostType() { return new PhabricatorElasticSearchHost($this); } /** * @return PhabricatorElasticSearchHost */ public function getHostForRead() { return $this->getService()->getAnyHostForRole('read'); } /** * @return PhabricatorElasticSearchHost */ public function getHostForWrite() { return $this->getService()->getAnyHostForRole('write'); } public function setTimeout($timeout) { $this->timeout = $timeout; return $this; } public function getTimeout() { return $this->timeout; } public function getTypeConstants($class) { $relationship_class = new ReflectionClass($class); $typeconstants = $relationship_class->getConstants(); return array_unique(array_values($typeconstants)); } public function reindexAbstractDocument( PhabricatorSearchAbstractDocument $doc) { $host = $this->getHostForWrite(); $type = $doc->getDocumentType(); $phid = $doc->getPHID(); $handle = id(new PhabricatorHandleQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withPHIDs(array($phid)) ->executeOne(); $timestamp_key = $this->getTimestampField(); // URL is not used internally but it can be useful externally. $spec = array( 'title' => $doc->getDocumentTitle(), 'url' => PhabricatorEnv::getProductionURI($handle->getURI()), 'dateCreated' => $doc->getDocumentCreated(), $timestamp_key => $doc->getDocumentModified(), ); foreach ($doc->getFieldData() as $field) { list($field_name, $corpus, $aux) = $field; if (!isset($spec[$field_name])) { $spec[$field_name] = array($corpus); } else { $spec[$field_name][] = $corpus; } if ($aux != null) { $spec[$field_name][] = $aux; } } foreach ($doc->getRelationshipData() as $field) { list($field_name, $related_phid, $rtype, $time) = $field; if (!isset($spec[$field_name])) { $spec[$field_name] = array($related_phid); } else { $spec[$field_name][] = $related_phid; } if ($time) { $spec[$field_name.'_ts'] = $time; } } $this->executeRequest($host, "/{$type}/{$phid}/", $spec, 'PUT'); } public function reconstructDocument($phid) { $type = phid_get_type($phid); $host = $this->getHostForRead(); $response = $this->executeRequest($host, "/{$type}/{$phid}", array()); if (empty($response['exists'])) { return null; } $hit = $response['_source']; $doc = new PhabricatorSearchAbstractDocument(); $doc->setPHID($phid); $doc->setDocumentType($response['_type']); $doc->setDocumentTitle($hit['title']); $doc->setDocumentCreated($hit['dateCreated']); $doc->setDocumentModified($hit[$this->getTimestampField()]); foreach ($hit['field'] as $fdef) { $field_type = $fdef['type']; $doc->addField($field_type, $hit[$field_type], $fdef['aux']); } foreach ($hit['relationship'] as $rtype => $rships) { foreach ($rships as $rship) { $doc->addRelationship( $rtype, $rship['phid'], $rship['phidType'], $rship['when']); } } return $doc; } private function buildSpec(PhabricatorSavedQuery $query) { $q = new PhabricatorElasticSearchQueryBuilder('bool'); $query_string = $query->getParameter('query'); if (strlen($query_string)) { $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); // Build a simple_query_string query over all fields that must match all // of the words in the search string. $q->addMustClause(array( 'simple_query_string' => array( 'query' => $query_string, 'fields' => array( - '_all', + PhabricatorSearchDocumentFieldType::FIELD_TITLE.'.*', + PhabricatorSearchDocumentFieldType::FIELD_BODY.'.*', + PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'.*', ), - 'default_operator' => 'OR', + 'default_operator' => 'AND', ), )); // This second query clause is "SHOULD' so it only affects ranking of // documents which already matched the Must clause. This amplifies the // score of documents which have an exact match on title, body // or comments. $q->addShouldClause(array( 'simple_query_string' => array( 'query' => $query_string, 'fields' => array( + '*.raw', PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4', PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3', PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2', ), 'analyzer' => 'english_exact', 'default_operator' => 'and', ), )); } $exclude = $query->getParameter('exclude'); if ($exclude) { $q->addFilterClause(array( 'not' => array( 'ids' => array( 'values' => array($exclude), ), ), )); } $relationship_map = array( PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR => $query->getParameter('authorPHIDs', array()), PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER => $query->getParameter('subscriberPHIDs', array()), PhabricatorSearchRelationship::RELATIONSHIP_PROJECT => $query->getParameter('projectPHIDs', array()), PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY => $query->getParameter('repositoryPHIDs', array()), ); $statuses = $query->getParameter('statuses', array()); $statuses = array_fuse($statuses); $rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; $rel_closed = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED; $rel_unowned = PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED; $include_open = !empty($statuses[$rel_open]); $include_closed = !empty($statuses[$rel_closed]); if ($include_open && !$include_closed) { $q->addExistsClause($rel_open); } else if (!$include_open && $include_closed) { $q->addExistsClause($rel_closed); } if ($query->getParameter('withUnowned')) { $q->addExistsClause($rel_unowned); } $rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER; if ($query->getParameter('withAnyOwner')) { $q->addExistsClause($rel_owner); } else { $owner_phids = $query->getParameter('ownerPHIDs', array()); if (count($owner_phids)) { $q->addTermsClause($rel_owner, $owner_phids); } } foreach ($relationship_map as $field => $phids) { if (is_array($phids) && !empty($phids)) { $q->addTermsClause($field, $phids); } } if (!$q->getClauseCount('must')) { $q->addMustClause(array('match_all' => array('boost' => 1 ))); } $spec = array( '_source' => false, 'query' => array( 'bool' => $q->toArray(), ), ); if (!$query->getParameter('query')) { $spec['sort'] = array( array('dateCreated' => 'desc'), ); } $offset = (int)$query->getParameter('offset', 0); $limit = (int)$query->getParameter('limit', 101); if ($offset + $limit > 10000) { throw new Exception(pht( 'Query offset is too large. offset+limit=%s (max=%s)', $offset + $limit, 10000)); } $spec['from'] = $offset; $spec['size'] = $limit; return $spec; } public function executeSearch(PhabricatorSavedQuery $query) { $types = $query->getParameter('types'); if (!$types) { $types = array_keys( PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); } // Don't use '/_search' for the case that there is something // else in the index (for example if 'phabricator' is only an alias to // some bigger index). Use '/$types/_search' instead. $uri = '/'.implode(',', $types).'/_search'; $spec = $this->buildSpec($query); $exceptions = array(); foreach ($this->service->getAllHostsForRole('read') as $host) { try { $response = $this->executeRequest($host, $uri, $spec); $phids = ipull($response['hits']['hits'], '_id'); return $phids; } catch (Exception $e) { $exceptions[] = $e; } } throw new PhutilAggregateException('All search hosts failed:', $exceptions); } public function indexExists(PhabricatorElasticSearchHost $host = null) { if (!$host) { $host = $this->getHostForRead(); } try { if ($this->version >= 5) { $uri = '/_stats/'; $res = $this->executeRequest($host, $uri, array()); return isset($res['indices']['phabricator']); } else if ($this->version >= 2) { $uri = ''; } else { $uri = '/_status/'; } return (bool)$this->executeRequest($host, $uri, array()); } catch (HTTPFutureHTTPResponseStatus $e) { if ($e->getStatusCode() == 404) { return false; } throw $e; } } private function getIndexConfiguration() { $data = array(); $data['settings'] = array( 'index' => array( 'auto_expand_replicas' => '0-2', 'analysis' => array( + 'filter' => array( + 'english_stop' => array( + 'type' => 'stop', + 'stopwords' => '_english_', + ), + 'english_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'english', + ), + 'english_possessive_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'possessive_english', + ), + ), 'analyzer' => array( 'english_exact' => array( 'tokenizer' => 'standard', 'filter' => array('lowercase'), ), + 'letter_stop' => array( + 'tokenizer' => 'letter', + 'filter' => array('lowercase', 'english_stop'), + ), + 'english_stem' => array( + 'tokenizer' => 'standard', + 'filter' => array( + 'english_possessive_stemmer', + 'lowercase', + 'english_stop', + 'english_stemmer', + ), + ), ), ), ), ); $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); $relationships = $this->getTypeConstants('PhabricatorSearchRelationship'); $doc_types = array_keys( PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); $text_type = $this->getTextFieldType(); foreach ($doc_types as $type) { $properties = array(); foreach ($fields as $field) { // Use the custom analyzer for the corpus of text $properties[$field] = array( 'type' => $text_type, - 'analyzer' => 'english_exact', - 'search_analyzer' => 'english', - 'search_quote_analyzer' => 'english_exact', + 'fields' => array( + 'raw' => array( + 'type' => $text_type, + 'analyzer' => 'english_exact', + 'search_analyzer' => 'english', + 'search_quote_analyzer' => 'english_exact', + ), + 'keywords' => array( + 'type' => $text_type, + 'analyzer' => 'letter_stop', + ), + 'stems' => array( + 'type' => $text_type, + 'analyzer' => 'english_stem', + ), + ), ); } if ($this->version < 5) { foreach ($relationships as $rel) { $properties[$rel] = array( 'type' => 'string', 'index' => 'not_analyzed', 'include_in_all' => false, ); $properties[$rel.'_ts'] = array( 'type' => 'date', 'include_in_all' => false, ); } } else { foreach ($relationships as $rel) { $properties[$rel] = array( 'type' => 'keyword', 'include_in_all' => false, 'doc_values' => false, ); $properties[$rel.'_ts'] = array( 'type' => 'date', 'include_in_all' => false, ); } } // Ensure we have dateCreated since the default query requires it $properties['dateCreated']['type'] = 'date'; $properties['lastModified']['type'] = 'date'; $data['mappings'][$type]['properties'] = $properties; } return $data; } public function indexIsSane(PhabricatorElasticSearchHost $host = null) { if (!$host) { $host = $this->getHostForRead(); } if (!$this->indexExists($host)) { return false; } $cur_mapping = $this->executeRequest($host, '/_mapping/', array()); $cur_settings = $this->executeRequest($host, '/_settings/', array()); $actual = array_merge($cur_settings[$this->index], $cur_mapping[$this->index]); $res = $this->check($actual, $this->getIndexConfiguration()); return $res; } /** * Recursively check if two Elasticsearch configuration arrays are equal * * @param $actual * @param $required array * @return bool */ private function check($actual, $required, $path = '') { foreach ($required as $key => $value) { if (!array_key_exists($key, $actual)) { if ($key === '_all') { // The _all field never comes back so we just have to assume it // is set correctly. continue; } return false; } if (is_array($value)) { if (!is_array($actual[$key])) { return false; } if (!$this->check($actual[$key], $value, $path.'.'.$key)) { return false; } continue; } $actual[$key] = self::normalizeConfigValue($actual[$key]); $value = self::normalizeConfigValue($value); if ($actual[$key] != $value) { return false; } } return true; } /** * Normalize a config value for comparison. Elasticsearch accepts all kinds * of config values but it tends to throw back 'true' for true and 'false' for * false so we normalize everything. Sometimes, oddly, it'll throw back false * for false.... * * @param mixed $value config value * @return mixed value normalized */ private static function normalizeConfigValue($value) { if ($value === true) { return 'true'; } else if ($value === false) { return 'false'; } return $value; } public function initIndex() { $host = $this->getHostForWrite(); if ($this->indexExists()) { $this->executeRequest($host, '/', array(), 'DELETE'); } $data = $this->getIndexConfiguration(); $this->executeRequest($host, '/', $data, 'PUT'); } public function getIndexStats(PhabricatorElasticSearchHost $host = null) { if ($this->version < 2) { return false; } if (!$host) { $host = $this->getHostForRead(); } $uri = '/_stats/'; $host = $this->getHostForRead(); $res = $this->executeRequest($host, $uri, array()); $stats = $res['indices'][$this->index]; return array( pht('Queries') => idxv($stats, array('primaries', 'search', 'query_total')), pht('Documents') => idxv($stats, array('total', 'docs', 'count')), pht('Deleted') => idxv($stats, array('total', 'docs', 'deleted')), pht('Storage Used') => phutil_format_bytes(idxv($stats, array('total', 'store', 'size_in_bytes'))), ); } private function executeRequest(PhabricatorElasticSearchHost $host, $path, array $data, $method = 'GET') { $uri = $host->getURI($path); - $data = json_encode($data); + $data = phutil_json_encode($data); $future = new HTTPSFuture($uri, $data); if ($method != 'GET') { $future->setMethod($method); } if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } try { list($body) = $future->resolvex(); } catch (HTTPFutureResponseStatus $ex) { if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) { $host->didHealthCheck(false); } throw $ex; } if ($method != 'GET') { return null; } try { $data = phutil_json_decode($body); $host->didHealthCheck(true); return $data; } catch (PhutilJSONParserException $ex) { $host->didHealthCheck(false); throw new PhutilProxyException( pht('ElasticSearch server returned invalid JSON!'), $ex); } } } diff --git a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php index ced23cd542..742b5713d3 100644 --- a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php +++ b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php @@ -1,34 +1,43 @@ setRoles(idx($config, 'roles', array('read' => true, 'write' => true))); return $this; } public function getDisplayName() { return 'MySQL'; } public function getStatusViewColumns() { return array( pht('Protocol') => 'mysql', pht('Roles') => implode(', ', array_keys($this->getRoles())), ); } public function getProtocol() { return 'mysql'; } + public function getHealthRecord() { + if (!$this->healthRecord) { + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication( + 'search'); + $this->healthRecord = $ref->getHealthRecord(); + } + return $this->healthRecord; + } + public function getConnectionStatus() { PhabricatorDatabaseRef::queryAll(); $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); $status = $ref->getConnectionStatus(); return $status; } } diff --git a/src/infrastructure/cluster/search/PhabricatorSearchHost.php b/src/infrastructure/cluster/search/PhabricatorSearchHost.php index 834e786789..93c3c4d938 100644 --- a/src/infrastructure/cluster/search/PhabricatorSearchHost.php +++ b/src/infrastructure/cluster/search/PhabricatorSearchHost.php @@ -1,163 +1,123 @@ engine = $engine; } public function setDisabled($disabled) { $this->disabled = $disabled; return $this; } public function getDisabled() { return $this->disabled; } /** * @return PhabricatorFulltextStorageEngine */ public function getEngine() { return $this->engine; } public function isWritable() { return $this->hasRole('write'); } public function isReadable() { return $this->hasRole('read'); } public function hasRole($role) { return isset($this->roles[$role]) && $this->roles[$role] === true; } public function setRoles(array $roles) { foreach ($roles as $role => $val) { $this->roles[$role] = $val; } return $this; } public function getRoles() { $roles = array(); foreach ($this->roles as $key => $val) { if ($val) { $roles[$key] = $val; } } return $roles; } public function setPort($value) { $this->port = $value; return $this; } public function getPort() { return $this->port; } public function setHost($value) { $this->host = $value; return $this; } public function getHost() { return $this->host; } public function getHealthRecordCacheKey() { $host = $this->getHost(); $port = $this->getPort(); $key = self::KEY_HEALTH; return "{$key}({$host}, {$port})"; } /** * @return PhabricatorClusterServiceHealthRecord */ public function getHealthRecord() { if (!$this->healthRecord) { $this->healthRecord = new PhabricatorClusterServiceHealthRecord( $this->getHealthRecordCacheKey()); } return $this->healthRecord; } public function didHealthCheck($reachable) { $record = $this->getHealthRecord(); $should_check = $record->getShouldCheck(); if ($should_check) { $record->didHealthCheck($reachable); } } /** * @return string[] Get a list of fields to show in the status overview UI */ abstract public function getStatusViewColumns(); abstract public function getConnectionStatus(); - public static function reindexAbstractDocument( - PhabricatorSearchAbstractDocument $doc) { - - $services = self::getAllServices(); - $indexed = 0; - foreach (self::getWritableHostForEachService() as $host) { - $host->getEngine()->reindexAbstractDocument($doc); - $indexed++; - } - if ($indexed == 0) { - throw new PhabricatorClusterNoHostForRoleException('write'); - } - } - - public static function executeSearch(PhabricatorSavedQuery $query) { - $services = self::getAllServices(); - foreach ($services as $service) { - $hosts = $service->getAllHostsForRole('read'); - // try all hosts until one succeeds - foreach ($hosts as $host) { - $last_exception = null; - try { - $res = $host->getEngine()->executeSearch($query); - // return immediately if we get results without an exception - $host->didHealthCheck(true); - return $res; - } catch (Exception $ex) { - // try each server in turn, only throw if none succeed - $last_exception = $ex; - $host->didHealthCheck(false); - } - } - } - if ($last_exception) { - throw $last_exception; - } - return $res; - } - } diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php index 20c0664456..0a563207b1 100644 --- a/src/infrastructure/cluster/search/PhabricatorSearchService.php +++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php @@ -1,259 +1,260 @@ engine = $engine; $this->hostType = $engine->getHostType(); } /** * @throws Exception */ public function newHost($config) { $host = clone($this->hostType); $host_config = $this->config + $config; $host->setConfig($host_config); $this->hosts[] = $host; return $host; } public function getEngine() { return $this->engine; } public function getDisplayName() { return $this->hostType->getDisplayName(); } public function getStatusViewColumns() { return $this->hostType->getStatusViewColumns(); } public function setConfig($config) { $this->config = $config; + $this->setRoles(idx($config, 'roles', array())); if (!isset($config['hosts'])) { $config['hosts'] = array( array( 'host' => idx($config, 'host'), 'port' => idx($config, 'port'), 'protocol' => idx($config, 'protocol'), 'roles' => idx($config, 'roles'), ), ); } foreach ($config['hosts'] as $host) { $this->newHost($host); } } public function getConfig() { return $this->config; } - public function setDisabled($disabled) { - $this->disabled = $disabled; - return $this; - } - - public function getDisabled() { - return $this->disabled; - } - public static function getConnectionStatusMap() { return array( self::STATUS_OKAY => array( 'icon' => 'fa-exchange', 'color' => 'green', 'label' => pht('Okay'), ), self::STATUS_FAIL => array( 'icon' => 'fa-times', 'color' => 'red', 'label' => pht('Failed'), ), ); } public function isWritable() { return $this->hasRole('write'); } public function isReadable() { return $this->hasRole('read'); } public function hasRole($role) { - return isset($this->roles[$role]) && $this->roles[$role] === true; + return isset($this->roles[$role]) && $this->roles[$role] !== false; } public function setRoles(array $roles) { foreach ($roles as $role => $val) { if ($val === false && isset($this->roles[$role])) { unset($this->roles[$role]); } else { $this->roles[$role] = $val; } } return $this; } public function getRoles() { return $this->roles; } public function getPort() { return idx($this->config, 'port'); } public function getProtocol() { return idx($this->config, 'protocol'); } public function getVersion() { return idx($this->config, 'version'); } public function getHosts() { return $this->hosts; } /** * Get a random host reference with the specified role, skipping hosts which * failed recent health checks. * @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match. * @return PhabricatorSearchHost */ public function getAnyHostForRole($role) { $hosts = $this->getAllHostsForRole($role); shuffle($hosts); foreach ($hosts as $host) { $health = $host->getHealthRecord(); if ($health->getIsHealthy()) { return $host; } } throw new PhabricatorClusterNoHostForRoleException($role); } /** * Get all configured hosts for this service which have the specified role. * @return PhabricatorSearchHost[] */ public function getAllHostsForRole($role) { + // if the role is explicitly set to false at the top level, then all hosts + // have the role disabled. + if (idx($this->config, $role) === false) { + return array(); + } + $hosts = array(); foreach ($this->hosts as $host) { if ($host->hasRole($role)) { $hosts[] = $host; } } return $hosts; } /** * Get a reference to all configured fulltext search cluster services * @return PhabricatorSearchService[] */ public static function getAllServices() { $cache = PhabricatorCaches::getRequestCache(); $refs = $cache->getKey(self::KEY_REFS); if (!$refs) { $refs = self::newRefs(); $cache->setKey(self::KEY_REFS, $refs); } return $refs; } /** * Load all valid PhabricatorFulltextStorageEngine subclasses */ public static function loadAllFulltextStorageEngines() { return id(new PhutilClassMapQuery()) ->setAncestorClass('PhabricatorFulltextStorageEngine') ->setUniqueMethod('getEngineIdentifier') ->execute(); } /** * Create instances of PhabricatorSearchService based on configuration * @return PhabricatorSearchService[] */ public static function newRefs() { $services = PhabricatorEnv::getEnvConfig('cluster.search'); $engines = self::loadAllFulltextStorageEngines(); $refs = array(); foreach ($services as $config) { $engine = $engines[$config['type']]; $cluster = new self($engine); $cluster->setConfig($config); $engine->setService($cluster); $refs[] = $cluster; } return $refs; } /** * (re)index the document: attempt to pass the document to all writable * fulltext search hosts * @throws PhabricatorClusterNoHostForRoleException */ public static function reindexAbstractDocument( PhabricatorSearchAbstractDocument $doc) { $indexed = 0; foreach (self::getAllServices() as $service) { - $service->getEngine()->reindexAbstractDocument($doc); - $indexed++; + $hosts = $service->getAllHostsForRole('write'); + if (count($hosts)) { + $service->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } } if ($indexed == 0) { throw new PhabricatorClusterNoHostForRoleException('write'); } } /** * Execute a full-text query and return a list of PHIDs of matching objects. * @return string[] * @throws PhutilAggregateException */ public static function executeSearch(PhabricatorSavedQuery $query) { $services = self::getAllServices(); $exceptions = array(); foreach ($services as $service) { $engine = $service->getEngine(); // try all hosts until one succeeds try { $res = $engine->executeSearch($query); // return immediately if we get results without an exception return $res; } catch (Exception $ex) { $exceptions[] = $ex; } } throw new PhutilAggregateException('All search engines failed:', $exceptions); } }