diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2310,6 +2310,7 @@ 'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php', 'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php', 'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php', + 'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php', 'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php', 'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php', 'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php', @@ -2349,6 +2350,7 @@ 'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php', 'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php', 'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php', + 'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php', 'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php', 'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php', 'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php', @@ -5515,6 +5517,7 @@ 'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine', 'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow', + 'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow', 'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchResultView' => 'AphrontView', @@ -5551,6 +5554,7 @@ 'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck', + 'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck', 'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck', diff --git a/src/applications/config/check/PhabricatorSetupCheckElastic.php b/src/applications/config/check/PhabricatorSetupCheckElastic.php new file mode 100644 --- /dev/null +++ b/src/applications/config/check/PhabricatorSetupCheckElastic.php @@ -0,0 +1,39 @@ +newEngine(); + if (!$engine->indexExists()) { + $summary = pht( + 'You enabled Elasticsearch but the index does not exist.'); + + $message = pht( + 'You likely enabled search.elastic.host without creating the '. + 'index. Run `./bin/search init` to correct the index.'); + + $this + ->newIssue('elastic.missing-index') + ->setName(pht('Elasticsearch index Not Found')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('search.elastic.host'); + } else if (!$engine->indexIsSane()) { + $summary = pht( + 'Elasticsearch index exists but needs correction.'); + + $message = pht( + 'Either the Phabricator schema for Elasticsearch has changed '. + 'or Elasticsearch created the index automatically. Run '. + '`./bin/search init` to correct the index.'); + + $this + ->newIssue('elastic.broken-index') + ->setName(pht('Elasticsearch index Incorrect')) + ->setSummary($summary) + ->setMessage($message); + } + } + } +} diff --git a/src/applications/search/engine/PhabricatorSearchEngine.php b/src/applications/search/engine/PhabricatorSearchEngine.php --- a/src/applications/search/engine/PhabricatorSearchEngine.php +++ b/src/applications/search/engine/PhabricatorSearchEngine.php @@ -36,4 +36,26 @@ */ abstract public function executeSearch(PhabricatorSavedQuery $query); + /** + * Does the search index exist? + * + * @return bool + */ + abstract public function indexExists(); + + /** + * Is the index in a usable state? + * + * @return bool + */ + public function indexIsSane() { + return $this->indexExists(); + } + + /** + * Do any sort of setup for the search index + * + * @return void + */ + public function initIndex() {} } diff --git a/src/applications/search/engine/PhabricatorSearchEngineElastic.php b/src/applications/search/engine/PhabricatorSearchEngineElastic.php --- a/src/applications/search/engine/PhabricatorSearchEngineElastic.php +++ b/src/applications/search/engine/PhabricatorSearchEngineElastic.php @@ -52,10 +52,7 @@ ); } - $this->executeRequest( - "/{$type}/{$phid}/", - $spec, - $is_write = true); + $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT'); } public function reconstructDocument($phid) { @@ -236,22 +233,146 @@ return $phids; } - private function executeRequest($path, array $data, $is_write = false) { + public function indexExists() { + try { + return (bool)$this->executeRequest('/_status/', array()); + } catch (HTTPFutureHTTPResponseStatus $e) { + if ($e->getStatusCode() == 404) { + return false; + } + throw $e; + } + } + + private function getIndexConfiguration() { + $data = array(); + $data['settings'] = array( + 'index' => array( + 'auto_expand_replicas' => '0-2', + 'analysis' => array( + 'filter' => array( + 'trigrams_filter' => array( + 'min_gram' => 3, + 'type' => 'ngram', + 'max_gram' => 3, + ), + ), + 'analyzer' => array( + 'custom_trigrams' => array( + 'type' => 'custom', + 'filter' => array( + 'lowercase', + 'kstem', + 'trigrams_filter', + ), + 'tokenizer' => 'standard', + ), + ), + ), + ), + ); + + $types = array_keys( + PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); + foreach ($types as $type) { + $data['mappings'][$type]['properties']['field']['properties']['corpus'] = + array( 'type' => 'string', 'analyzer' => 'custom_trigrams' ); + } + + return $data; + } + + public function indexIsSane() { + if (!$this->indexExists()) { + return false; + } + + $cur_mapping = $this->executeRequest('/_mapping/', array()); + $cur_settings = $this->executeRequest('/_settings/', array()); + $actual = array_merge($cur_settings[$this->index], + $cur_mapping[$this->index]); + + return $this->check($actual, $this->getIndexConfiguration()); + } + + /** + * Recursively check if two Elasticsearch configuration arrays are equal + * + * @param $actual + * @param $required array + * @return bool + */ + private function check($actual, $required) { + foreach ($required as $key => $value) { + if (!array_key_exists($key, $actual)) { + if ($key === '_all') { + // The _all field never comes back so we just have to assume it + // is set correctly. + continue; + } + return false; + } + if (is_array($value)) { + if (!is_array($actual[$key])) { + return false; + } + if (!$this->check($actual[$key], $value)) { + return false; + } + continue; + } + + $actual[$key] = self::normalizeConfigValue($actual[$key]); + $value = self::normalizeConfigValue($value); + if ($actual[$key] != $value) { + return false; + } + } + return true; + } + + /** + * Normalize a config value for comparison. Elasticsearch accepts all kinds + * of config values but it tends to throw back 'true' for true and 'false' for + * false so we normalize everything. Sometimes, oddly, it'll throw back false + * for false.... + * + * @param mixed $value config value + * @return mixed value normalized + */ + private static function normalizeConfigValue($value) { + if ($value === true) { + return 'true'; + } else if ($value === false) { + return 'false'; + } + return $value; + } + + public function initIndex() { + if ($this->indexExists()) { + $this->executeRequest('/', array(), 'DELETE'); + } + $data = $this->getIndexConfiguration(); + $this->executeRequest('/', $data, 'PUT'); + } + + private function executeRequest($path, array $data, $method = 'GET') { $uri = new PhutilURI($this->uri); $uri->setPath($this->index); $uri->appendPath($path); $data = json_encode($data); $future = new HTTPSFuture($uri, $data); - if ($is_write) { - $future->setMethod('PUT'); + if ($method != 'GET') { + $future->setMethod($method); } if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } list($body) = $future->resolvex(); - if ($is_write) { + if ($method != 'GET') { return null; } diff --git a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php --- a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php +++ b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php @@ -331,4 +331,7 @@ return $sql; } + public function indexExists() { + return true; + } } diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php new file mode 100644 --- /dev/null +++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php @@ -0,0 +1,50 @@ +setName('init') + ->setSynopsis('Initialize or repair an index.') + ->setExamples('**init**'); + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine(); + + $work_done = false; + if (!$engine->indexExists()) { + $console->writeOut( + '%s', + pht('Index does not exist, creating...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } else if (!$engine->indexIsSane()) { + $console->writeOut( + '%s', + pht('Index exists but is incorrect, fixing...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } + + if ($work_done) { + $console->writeOut( + "%s\n", + pht('Index maintenance complete. Run `./bin/search index` to '. + 'reindex documents')); + } else { + $console->writeOut( + "%s\n", + pht('Nothing to do.')); + } + } +}