Page MenuHomePhabricator
Files F15407130

D10955.id26491.diff
No OneTemporary
Actions

D10955.id26491.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2310,6 +2310,7 @@
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
+ 'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php',
'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
@@ -2349,6 +2350,7 @@
'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php',
'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php',
'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php',
+ 'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php',
'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php',
'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php',
'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php',
@@ -5515,6 +5517,7 @@
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
+ 'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchResultView' => 'AphrontView',
@@ -5551,6 +5554,7 @@
'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck',
+ 'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck',
'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck',
diff --git a/src/applications/config/check/PhabricatorSetupCheckElastic.php b/src/applications/config/check/PhabricatorSetupCheckElastic.php
new file mode 100644
--- /dev/null
+++ b/src/applications/config/check/PhabricatorSetupCheckElastic.php
@@ -0,0 +1,39 @@
+<?php
+
+final class PhabricatorSetupCheckElastic extends PhabricatorSetupCheck {
+
+ protected function executeChecks() {
+ if (PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
+ $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+ if (!$engine->indexExists()) {
+ $summary = pht(
+ 'You enabled Elasticsearch but the index does not exist.');
+
+ $message = pht(
+ 'You likely enabled search.elastic.host without creating the '.
+ 'index. Run `./bin/search init` to correct the index.');
+
+ $this
+ ->newIssue('elastic.missing-index')
+ ->setName(pht('Elasticsearch index Not Found'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->addRelatedPhabricatorConfig('search.elastic.host');
+ } else if (!$engine->indexIsSane()) {
+ $summary = pht(
+ 'Elasticsearch index exists but needs correction.');
+
+ $message = pht(
+ 'Either the Phabricator schema for Elasticsearch has changed '.
+ 'or Elasticsearch created the index automatically. Run '.
+ '`./bin/search init` to correct the index.');
+
+ $this
+ ->newIssue('elastic.broken-index')
+ ->setName(pht('Elasticsearch index Incorrect'))
+ ->setSummary($summary)
+ ->setMessage($message);
+ }
+ }
+ }
+}
diff --git a/src/applications/search/engine/PhabricatorSearchEngine.php b/src/applications/search/engine/PhabricatorSearchEngine.php
--- a/src/applications/search/engine/PhabricatorSearchEngine.php
+++ b/src/applications/search/engine/PhabricatorSearchEngine.php
@@ -36,4 +36,26 @@
*/
abstract public function executeSearch(PhabricatorSavedQuery $query);
+ /**
+ * Does the search index exist?
+ *
+ * @return bool
+ */
+ abstract public function indexExists();
+
+ /**
+ * Is the index in a usable state?
+ *
+ * @return bool
+ */
+ public function indexIsSane() {
+ return $this->indexExists();
+ }
+
+ /**
+ * Do any sort of setup for the search index
+ *
+ * @return void
+ */
+ public function initIndex() {}
}
diff --git a/src/applications/search/engine/PhabricatorSearchEngineElastic.php b/src/applications/search/engine/PhabricatorSearchEngineElastic.php
--- a/src/applications/search/engine/PhabricatorSearchEngineElastic.php
+++ b/src/applications/search/engine/PhabricatorSearchEngineElastic.php
@@ -52,10 +52,7 @@
);
}
- $this->executeRequest(
- "/{$type}/{$phid}/",
- $spec,
- $is_write = true);
+ $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
}
public function reconstructDocument($phid) {
@@ -236,22 +233,146 @@
return $phids;
}
- private function executeRequest($path, array $data, $is_write = false) {
+ public function indexExists() {
+ try {
+ return (bool)$this->executeRequest('/_status/', array());
+ } catch (HTTPFutureHTTPResponseStatus $e) {
+ if ($e->getStatusCode() == 404) {
+ return false;
+ }
+ throw $e;
+ }
+ }
+
+ private function getIndexConfiguration() {
+ $data = array();
+ $data['settings'] = array(
+ 'index' => array(
+ 'auto_expand_replicas' => '0-2',
+ 'analysis' => array(
+ 'filter' => array(
+ 'trigrams_filter' => array(
+ 'min_gram' => 3,
+ 'type' => 'ngram',
+ 'max_gram' => 3,
+ ),
+ ),
+ 'analyzer' => array(
+ 'custom_trigrams' => array(
+ 'type' => 'custom',
+ 'filter' => array(
+ 'lowercase',
+ 'kstem',
+ 'trigrams_filter',
+ ),
+ 'tokenizer' => 'standard',
+ ),
+ ),
+ ),
+ ),
+ );
+
+ $types = array_keys(
+ PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
+ foreach ($types as $type) {
+ $data['mappings'][$type]['properties']['field']['properties']['corpus'] =
+ array( 'type' => 'string', 'analyzer' => 'custom_trigrams' );
+ }
+
+ return $data;
+ }
+
+ public function indexIsSane() {
+ if (!$this->indexExists()) {
+ return false;
+ }
+
+ $cur_mapping = $this->executeRequest('/_mapping/', array());
+ $cur_settings = $this->executeRequest('/_settings/', array());
+ $actual = array_merge($cur_settings[$this->index],
+ $cur_mapping[$this->index]);
+
+ return $this->check($actual, $this->getIndexConfiguration());
+ }
+
+ /**
+ * Recursively check if two Elasticsearch configuration arrays are equal
+ *
+ * @param $actual
+ * @param $required array
+ * @return bool
+ */
+ private function check($actual, $required) {
+ foreach ($required as $key => $value) {
+ if (!array_key_exists($key, $actual)) {
+ if ($key === '_all') {
+ // The _all field never comes back so we just have to assume it
+ // is set correctly.
+ continue;
+ }
+ return false;
+ }
+ if (is_array($value)) {
+ if (!is_array($actual[$key])) {
+ return false;
+ }
+ if (!$this->check($actual[$key], $value)) {
+ return false;
+ }
+ continue;
+ }
+
+ $actual[$key] = self::normalizeConfigValue($actual[$key]);
+ $value = self::normalizeConfigValue($value);
+ if ($actual[$key] != $value) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Normalize a config value for comparison. Elasticsearch accepts all kinds
+ * of config values but it tends to throw back 'true' for true and 'false' for
+ * false so we normalize everything. Sometimes, oddly, it'll throw back false
+ * for false....
+ *
+ * @param mixed $value config value
+ * @return mixed value normalized
+ */
+ private static function normalizeConfigValue($value) {
+ if ($value === true) {
+ return 'true';
+ } else if ($value === false) {
+ return 'false';
+ }
+ return $value;
+ }
+
+ public function initIndex() {
+ if ($this->indexExists()) {
+ $this->executeRequest('/', array(), 'DELETE');
+ }
+ $data = $this->getIndexConfiguration();
+ $this->executeRequest('/', $data, 'PUT');
+ }
+
+ private function executeRequest($path, array $data, $method = 'GET') {
$uri = new PhutilURI($this->uri);
$uri->setPath($this->index);
$uri->appendPath($path);
$data = json_encode($data);
$future = new HTTPSFuture($uri, $data);
- if ($is_write) {
- $future->setMethod('PUT');
+ if ($method != 'GET') {
+ $future->setMethod($method);
}
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
list($body) = $future->resolvex();
- if ($is_write) {
+ if ($method != 'GET') {
return null;
}
diff --git a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
--- a/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
+++ b/src/applications/search/engine/PhabricatorSearchEngineMySQL.php
@@ -331,4 +331,7 @@
return $sql;
}
+ public function indexExists() {
+ return true;
+ }
}
diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
new file mode 100644
--- /dev/null
+++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
@@ -0,0 +1,50 @@
+<?php
+
+final class PhabricatorSearchManagementInitWorkflow
+ extends PhabricatorSearchManagementWorkflow {
+
+ protected function didConstruct() {
+ $this
+ ->setName('init')
+ ->setSynopsis('Initialize or repair an index.')
+ ->setExamples('**init**');
+ }
+
+ public function execute(PhutilArgumentParser $args) {
+ $console = PhutilConsole::getConsole();
+
+ $engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
+
+ $work_done = false;
+ if (!$engine->indexExists()) {
+ $console->writeOut(
+ '%s',
+ pht('Index does not exist, creating...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ } else if (!$engine->indexIsSane()) {
+ $console->writeOut(
+ '%s',
+ pht('Index exists but is incorrect, fixing...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ }
+
+ if ($work_done) {
+ $console->writeOut(
+ "%s\n",
+ pht('Index maintenance complete. Run `./bin/search index` to '.
+ 'reindex documents'));
+ } else {
+ $console->writeOut(
+ "%s\n",
+ pht('Nothing to do.'));
+ }
+ }
+}

File Metadata

Mime Type
text/plain
Expires
Wed, Mar 19, 3:55 PM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7375773
Default Alt Text
D10955.id26491.diff (11 KB)

Event Timeline

Phabricator · Built by Phacility