Page MenuHomePhabricator
Files F15383851

D10955.id26323.diff
No OneTemporary
Actions

D10955.id26323.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2292,6 +2292,7 @@
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
+ 'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php',
'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
@@ -2331,6 +2332,7 @@
'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php',
'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php',
'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php',
+ 'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php',
'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php',
'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php',
'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php',
@@ -5471,6 +5473,7 @@
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
+ 'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchResultView' => 'AphrontView',
@@ -5507,6 +5510,7 @@
'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck',
+ 'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck',
diff --git a/src/applications/config/check/PhabricatorSetupCheckElastic.php b/src/applications/config/check/PhabricatorSetupCheckElastic.php
new file mode 100644
--- /dev/null
+++ b/src/applications/config/check/PhabricatorSetupCheckElastic.php
@@ -0,0 +1,39 @@
+<?php
+
+final class PhabricatorSetupCheckElastic extends PhabricatorSetupCheck {
+
+ protected function executeChecks() {
+ if (PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
+ $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+ if (!$engine->indexExists()) {
+ $summary = pht(
+ 'You enabled Elasticsearch but the index does not exist.');
+
+ $message = pht(
+ 'You likely enabled search.elastic.host without creating the '.
+ 'index. Run `./bin/search init-index` to correct the index.');
+
+ $this
+ ->newIssue('elastic.missing-index')
+ ->setName(pht('Elasticsearch index Not Found'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->addRelatedPhabricatorConfig('search.elastic.host');
+ } else if (!$engine->indexIsSane()) {
+ $summary = pht(
+ 'Elasticsearch index exists but needs correction.');
+
+ $message = pht(
+ 'Either the Phabricator schema for Elasticsearch has changed '.
+ 'or Elasticsearch created the index automatically. Run '.
+ '`./bin/search init-index` to correct the index.');
+
+ $this
+ ->newIssue('elastic.broken-index')
+ ->setName(pht('Elasticsearch index Incorrect'))
+ ->setSummary($summary)
+ ->setMessage($message);
+ }
+ }
+ }
+}
diff --git a/src/applications/search/engine/PhabricatorSearchEngine.php b/src/applications/search/engine/PhabricatorSearchEngine.php
--- a/src/applications/search/engine/PhabricatorSearchEngine.php
+++ b/src/applications/search/engine/PhabricatorSearchEngine.php
@@ -36,4 +36,10 @@
*/
abstract public function executeSearch(PhabricatorSavedQuery $query);
+ /**
+ * Do any sort of setup for the search index
+ *
+ * @return void
+ */
+ abstract public function initIndex();
}
diff --git a/src/applications/search/engine/PhabricatorSearchEngineElastic.php b/src/applications/search/engine/PhabricatorSearchEngineElastic.php
--- a/src/applications/search/engine/PhabricatorSearchEngineElastic.php
+++ b/src/applications/search/engine/PhabricatorSearchEngineElastic.php
@@ -52,10 +52,7 @@
);
}
- $this->executeRequest(
- "/{$type}/{$phid}/",
- $spec,
- $is_write = true);
+ $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
}
public function reconstructDocument($phid) {
@@ -238,22 +235,108 @@
return $phids;
}
- private function executeRequest($path, array $data, $is_write = false) {
+ public function indexExists() {
+ try {
+ (bool)$this->executeRequest('/_search/', array());
+ } catch (HTTPFutureHTTPResponseStatus $e) {
+ if ($e->getStatusCode() == 404) {
+ return false;
+ } else if ($e->getStatusCode() == 400) {
+ return true;
+ }
+ throw $e;
+ }
+ }
+
+ private function getIndexConfiguration() {
+ $data = array(
+ 'settings' => array(
+ 'auto_expand_replicas' => '0-1',
+ 'analysis' => array(
+ 'filter' => array(
+ 'english_stop' => array(
+ 'type' => 'stop',
+ 'stopwords' => '_english_',
+ ),
+ 'english_possessive_stemmer' => array(
+ 'type' => 'stemmer',
+ 'language' => 'possessive_english',
+ ),
+ 'trigrams_filter' => array(
+ 'min_gram' => 3,
+ 'type' => 'ngram',
+ 'max_gram' => 3,
+ ),
+ 'english_stemmer' => array(
+ 'type' => 'stemmer',
+ 'language' => 'english',
+ ),
+ ),
+ 'analyzer' => array(
+ 'english_trigrams' => array(
+ 'type' => 'custom',
+ 'filter' => array(
+ 'english_possessive_stemmer',
+ 'lowercase',
+ 'english_stop',
+ 'english_stemmer',
+ 'trigrams_filter',
+ ),
+ 'tokenizer' => 'standard',
+ ),
+ ),
+ ),
+ ),
+ );
+
+ $types = array_keys(
+ PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
+ foreach ($types as $type) {
+ $data['mappings'][$type]['properties']['field']['properties']['corpus'] =
+ array( 'type' => 'string', 'analyzer' => 'english_trigrams' );
+ }
+
+ return $data;
+ }
+
+ public function indexIsSane() {
+ if (!$this->indexExists()) {
+ return false;
+ }
+
+ $expected = array( $this->index => $this->getIndexConfiguration() );
+ $actual = array_merge(
+ $this->executeRequest('/_settings/', array()),
+ $this->executeRequest('/_mapping/', array()));
+
+ /** ideally we'd compare $expected and $actual here */
+ return true;
+ }
+
+ public function initIndex() {
+ if ($this->indexExists()) {
+ $this->executeRequest('/', array(), 'DELETE');
+ }
+ $data = $this->getIndexConfiguration();
+ $this->executeRequest('/', $data, 'PUT');
+ }
+
+ private function executeRequest($path, array $data, $method = 'GET') {
$uri = new PhutilURI($this->uri);
$uri->setPath($this->index);
$uri->appendPath($path);
$data = json_encode($data);
$future = new HTTPSFuture($uri, $data);
- if ($is_write) {
- $future->setMethod('PUT');
+ if ($method != 'GET') {
+ $future->setMethod($method);
}
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
list($body) = $future->resolvex();
- if ($is_write) {
+ if ($method != 'GET') {
return null;
}
diff --git a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
--- a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
+++ b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
@@ -331,4 +331,8 @@
return $sql;
}
+ /**
+ * no-op for mysql
+ */
+ public function initIndex() {}
}
diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
new file mode 100644
--- /dev/null
+++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
@@ -0,0 +1,57 @@
+<?php
+
+final class PhabricatorSearchManagementInitWorkflow
+ extends PhabricatorSearchManagementWorkflow {
+
+ protected function didConstruct() {
+ $this
+ ->setName('init')
+ ->setSynopsis('Initialize or repair an index.')
+ ->setExamples('**init**');
+ }
+
+ public function execute(PhutilArgumentParser $args) {
+ $console = PhutilConsole::getConsole();
+
+ if (!PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
+ $console->writeOut(
+ "%s\n",
+ pht('Index initialization only needed for Elasticsearch.'));
+ return;
+ }
+
+ $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+
+ $work_done = false;
+ if (!$engine->indexExists()) {
+ $console->writeOut(
+ '%s',
+ pht('Index does not exist, creating...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ } else if (!$engine->indexIsSane()) {
+ $console->writeOut(
+ '%s',
+ pht('Index exists but is incorrect, fixing...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ }
+
+ if ($work_done) {
+ $console->writeOut(
+ "%s\n",
+ pht('Index maintenance complete. Run `./bin/search index` to '.
+ 'reindex documents'));
+ } else {
+ $console->writeOut(
+ "%s\n",
+ pht('Nothing to do.'));
+ }
+ }
+}

File Metadata

Mime Type
text/plain
Expires
Sat, Mar 15, 6:09 PM (2 w, 5 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7387765
Default Alt Text
D10955.id26323.diff (10 KB)

Event Timeline

Phabricator · Built by Phacility