diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php --- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -49,26 +49,28 @@ $message = pht( 'You likely enabled cluster.search without creating the '. - 'index. Run `./bin/search init` to correct the index.'); + 'index. Use the following command to create a new index.'); $this ->newIssue('elastic.missing-index') - ->setName(pht('Elasticsearch index Not Found')) + ->setName(pht('Elasticsearch Index Not Found')) + ->addCommand('./bin/search init') ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('cluster.search'); + ->setMessage($message); + } else if (!$index_sane) { $summary = pht( 'Elasticsearch index exists but needs correction.'); $message = pht( 'Either the Phabricator schema for Elasticsearch has changed '. - 'or Elasticsearch created the index automatically. Run '. - '`./bin/search init` to correct the index.'); + 'or Elasticsearch created the index automatically. '. + 'Use the following command to rebuild the index.'); $this ->newIssue('elastic.broken-index') - ->setName(pht('Elasticsearch index Incorrect')) + ->setName(pht('Elasticsearch Index Schema Mismatch')) + ->addCommand('./bin/search init') ->setSummary($summary) ->setMessage($message); } diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -161,9 +161,11 @@ 'simple_query_string' => array( 'query' => $query_string, 'fields' => array( - '_all', + PhabricatorSearchDocumentFieldType::FIELD_TITLE.'.*', + PhabricatorSearchDocumentFieldType::FIELD_BODY.'.*', + PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'.*', ), - 'default_operator' => 'OR', + 'default_operator' => 'AND', ), )); @@ -175,6 +177,7 @@ 'simple_query_string' => array( 'query' => $query_string, 'fields' => array( + '*.raw', PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4', PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3', PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2', @@ -332,11 +335,38 @@ 'index' => array( 'auto_expand_replicas' => '0-2', 'analysis' => array( + 'filter' => array( + 'english_stop' => array( + 'type' => 'stop', + 'stopwords' => '_english_', + ), + 'english_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'english', + ), + 'english_possessive_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'possessive_english', + ), + ), 'analyzer' => array( 'english_exact' => array( 'tokenizer' => 'standard', 'filter' => array('lowercase'), ), + 'letter_stop' => array( + 'tokenizer' => 'letter', + 'filter' => array('lowercase', 'english_stop'), + ), + 'english_stem' => array( + 'tokenizer' => 'standard', + 'filter' => array( + 'english_possessive_stemmer', + 'lowercase', + 'english_stop', + 'english_stemmer', + ), + ), ), ), ), @@ -356,9 +386,22 @@ // Use the custom analyzer for the corpus of text $properties[$field] = array( 'type' => $text_type, - 'analyzer' => 'english_exact', - 'search_analyzer' => 'english', - 'search_quote_analyzer' => 'english_exact', + 'fields' => array( + 'raw' => array( + 'type' => $text_type, + 'analyzer' => 'english_exact', + 'search_analyzer' => 'english', + 'search_quote_analyzer' => 'english_exact', + ), + 'keywords' => array( + 'type' => $text_type, + 'analyzer' => 'letter_stop', + ), + 'stems' => array( + 'type' => $text_type, + 'analyzer' => 'english_stem', + ), + ), ); } @@ -505,7 +548,7 @@ array $data, $method = 'GET') { $uri = $host->getURI($path); - $data = json_encode($data); + $data = phutil_json_encode($data); $future = new HTTPSFuture($uri, $data); if ($method != 'GET') { $future->setMethod($method); diff --git a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php --- a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php +++ b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php @@ -24,6 +24,15 @@ return 'mysql'; } + public function getHealthRecord() { + if (!$this->healthRecord) { + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication( + 'search'); + $this->healthRecord = $ref->getHealthRecord(); + } + return $this->healthRecord; + } + public function getConnectionStatus() { PhabricatorDatabaseRef::queryAll(); $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); diff --git a/src/infrastructure/cluster/search/PhabricatorSearchHost.php b/src/infrastructure/cluster/search/PhabricatorSearchHost.php --- a/src/infrastructure/cluster/search/PhabricatorSearchHost.php +++ b/src/infrastructure/cluster/search/PhabricatorSearchHost.php @@ -13,7 +13,6 @@ protected $disabled; protected $host; protected $port; - protected $hostRefs = array(); const STATUS_OKAY = 'okay'; const STATUS_FAIL = 'fail'; @@ -121,43 +120,4 @@ abstract public function getConnectionStatus(); - public static function reindexAbstractDocument( - PhabricatorSearchAbstractDocument $doc) { - - $services = self::getAllServices(); - $indexed = 0; - foreach (self::getWritableHostForEachService() as $host) { - $host->getEngine()->reindexAbstractDocument($doc); - $indexed++; - } - if ($indexed == 0) { - throw new PhabricatorClusterNoHostForRoleException('write'); - } - } - - public static function executeSearch(PhabricatorSavedQuery $query) { - $services = self::getAllServices(); - foreach ($services as $service) { - $hosts = $service->getAllHostsForRole('read'); - // try all hosts until one succeeds - foreach ($hosts as $host) { - $last_exception = null; - try { - $res = $host->getEngine()->executeSearch($query); - // return immediately if we get results without an exception - $host->didHealthCheck(true); - return $res; - } catch (Exception $ex) { - // try each server in turn, only throw if none succeed - $last_exception = $ex; - $host->didHealthCheck(false); - } - } - } - if ($last_exception) { - throw $last_exception; - } - return $res; - } - } diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php --- a/src/infrastructure/cluster/search/PhabricatorSearchService.php +++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php @@ -46,6 +46,7 @@ public function setConfig($config) { $this->config = $config; + $this->setRoles(idx($config, 'roles', array())); if (!isset($config['hosts'])) { $config['hosts'] = array( @@ -67,15 +68,6 @@ return $this->config; } - public function setDisabled($disabled) { - $this->disabled = $disabled; - return $this; - } - - public function getDisabled() { - return $this->disabled; - } - public static function getConnectionStatusMap() { return array( self::STATUS_OKAY => array( @@ -100,7 +92,7 @@ } public function hasRole($role) { - return isset($this->roles[$role]) && $this->roles[$role] === true; + return isset($this->roles[$role]) && $this->roles[$role] !== false; } public function setRoles(array $roles) { @@ -160,6 +152,12 @@ * @return PhabricatorSearchHost[] */ public function getAllHostsForRole($role) { + // if the role is explicitly set to false at the top level, then all hosts + // have the role disabled. + if (idx($this->config, $role) === false) { + return array(); + } + $hosts = array(); foreach ($this->hosts as $host) { if ($host->hasRole($role)) { @@ -225,8 +223,11 @@ PhabricatorSearchAbstractDocument $doc) { $indexed = 0; foreach (self::getAllServices() as $service) { - $service->getEngine()->reindexAbstractDocument($doc); - $indexed++; + $hosts = $service->getAllHostsForRole('write'); + if (count($hosts)) { + $service->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } } if ($indexed == 0) { throw new PhabricatorClusterNoHostForRoleException('write');