diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2292,6 +2292,7 @@ 'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php', 'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php', 'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php', + 'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php', 'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php', 'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php', 'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php', @@ -2331,6 +2332,7 @@ 'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php', 'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php', 'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php', + 'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php', 'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php', 'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php', 'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php', @@ -5471,6 +5473,7 @@ 'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine', 'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow', + 'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow', 'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchResultView' => 'AphrontView', @@ -5507,6 +5510,7 @@ 'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck', + 'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck', diff --git a/src/applications/config/check/PhabricatorSetupCheckElastic.php b/src/applications/config/check/PhabricatorSetupCheckElastic.php new file mode 100644 --- /dev/null +++ b/src/applications/config/check/PhabricatorSetupCheckElastic.php @@ -0,0 +1,39 @@ +newEngine(); + if (!$engine->indexExists()) { + $summary = pht( + 'You enabled Elasticsearch but the index does not exist.'); + + $message = pht( + 'You likely enabled search.elastic.host without creating the '. + 'index. Run `./bin/search init-index` to correct the index.'); + + $this + ->newIssue('elastic.missing-index') + ->setName(pht('Elasticsearch index Not Found')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('search.elastic.host'); + } else if (!$engine->indexIsSane()) { + $summary = pht( + 'Elasticsearch index exists but needs correction.'); + + $message = pht( + 'Either the Phabricator schema for Elasticsearch has changed '. + 'or Elasticsearch created the index automatically. Run '. + '`./bin/search init-index` to correct the index.'); + + $this + ->newIssue('elastic.broken-index') + ->setName(pht('Elasticsearch index Incorrect')) + ->setSummary($summary) + ->setMessage($message); + } + } + } +} diff --git a/src/applications/search/engine/PhabricatorSearchEngine.php b/src/applications/search/engine/PhabricatorSearchEngine.php --- a/src/applications/search/engine/PhabricatorSearchEngine.php +++ b/src/applications/search/engine/PhabricatorSearchEngine.php @@ -36,4 +36,10 @@ */ abstract public function executeSearch(PhabricatorSavedQuery $query); + /** + * Do any sort of setup for the search index + * + * @return void + */ + abstract public function initIndex(); } diff --git a/src/applications/search/engine/PhabricatorSearchEngineElastic.php b/src/applications/search/engine/PhabricatorSearchEngineElastic.php --- a/src/applications/search/engine/PhabricatorSearchEngineElastic.php +++ b/src/applications/search/engine/PhabricatorSearchEngineElastic.php @@ -52,10 +52,7 @@ ); } - $this->executeRequest( - "/{$type}/{$phid}/", - $spec, - $is_write = true); + $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT'); } public function reconstructDocument($phid) { @@ -238,22 +235,108 @@ return $phids; } - private function executeRequest($path, array $data, $is_write = false) { + public function indexExists() { + try { + (bool)$this->executeRequest('/_search/', array()); + } catch (HTTPFutureHTTPResponseStatus $e) { + if ($e->getStatusCode() == 404) { + return false; + } else if ($e->getStatusCode() == 400) { + return true; + } + throw $e; + } + } + + private function getIndexConfiguration() { + $data = array( + 'settings' => array( + 'auto_expand_replicas' => '0-1', + 'analysis' => array( + 'filter' => array( + 'english_stop' => array( + 'type' => 'stop', + 'stopwords' => '_english_', + ), + 'english_possessive_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'possessive_english', + ), + 'trigrams_filter' => array( + 'min_gram' => 3, + 'type' => 'ngram', + 'max_gram' => 3, + ), + 'english_stemmer' => array( + 'type' => 'stemmer', + 'language' => 'english', + ), + ), + 'analyzer' => array( + 'english_trigrams' => array( + 'type' => 'custom', + 'filter' => array( + 'english_possessive_stemmer', + 'lowercase', + 'english_stop', + 'english_stemmer', + 'trigrams_filter', + ), + 'tokenizer' => 'standard', + ), + ), + ), + ), + ); + + $types = array_keys( + PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); + foreach ($types as $type) { + $data['mappings'][$type]['properties']['field']['properties']['corpus'] = + array( 'type' => 'string', 'analyzer' => 'english_trigrams' ); + } + + return $data; + } + + public function indexIsSane() { + if (!$this->indexExists()) { + return false; + } + + $expected = array( $this->index => $this->getIndexConfiguration() ); + $actual = array_merge( + $this->executeRequest('/_settings/', array()), + $this->executeRequest('/_mapping/', array())); + + /** ideally we'd compare $expected and $actual here */ + return true; + } + + public function initIndex() { + if ($this->indexExists()) { + $this->executeRequest('/', array(), 'DELETE'); + } + $data = $this->getIndexConfiguration(); + $this->executeRequest('/', $data, 'PUT'); + } + + private function executeRequest($path, array $data, $method = 'GET') { $uri = new PhutilURI($this->uri); $uri->setPath($this->index); $uri->appendPath($path); $data = json_encode($data); $future = new HTTPSFuture($uri, $data); - if ($is_write) { - $future->setMethod('PUT'); + if ($method != 'GET') { + $future->setMethod($method); } if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } list($body) = $future->resolvex(); - if ($is_write) { + if ($method != 'GET') { return null; } diff --git a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php --- a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php +++ b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php @@ -331,4 +331,8 @@ return $sql; } + /** + * no-op for mysql + */ + public function initIndex() {} } diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php new file mode 100644 --- /dev/null +++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php @@ -0,0 +1,57 @@ +setName('init') + ->setSynopsis('Initialize or repair an index.') + ->setExamples('**init**'); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + if (!PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) { + $console->writeOut( + "%s\n", + pht('Index initialization only needed for Elasticsearch.')); + return; + } + + $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine(); + + $work_done = false; + if (!$engine->indexExists()) { + $console->writeOut( + '%s', + pht('Index does not exist, creating...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } else if (!$engine->indexIsSane()) { + $console->writeOut( + '%s', + pht('Index exists but is incorrect, fixing...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } + + if ($work_done) { + $console->writeOut( + "%s\n", + pht('Index maintenance complete. Run `./bin/search index` to '. + 'reindex documents')); + } else { + $console->writeOut( + "%s\n", + pht('Nothing to do.')); + } + } +}