Page MenuHomePhabricator
Files F14037069

D17384.diff
No OneTemporary
Actions

D17384.diff
View Options

diff --git a/resources/sql/autopatches/20161130.search.02.rebuild.php b/resources/sql/autopatches/20161130.search.02.rebuild.php
--- a/resources/sql/autopatches/20161130.search.02.rebuild.php
+++ b/resources/sql/autopatches/20161130.search.02.rebuild.php
@@ -1,7 +1,15 @@
<?php
-$search_engine = PhabricatorFulltextStorageEngine::loadEngine();
-$use_mysql = ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine);
+
+$use_mysql = false;
+
+$services = PhabricatorSearchService::getAllServices();
+foreach ($services as $service) {
+ $engine = $service->getEngine();
+ if ($engine instanceof PhabricatorMySQLFulltextStorageEngine) {
+ $use_mysql = true;
+ }
+}
if ($use_mysql) {
$field = new PhabricatorSearchDocumentField();
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2259,12 +2259,15 @@
'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
- 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
- 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php',
- 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
- 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php',
- 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
- 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php',
+ 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php',
+ 'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php',
+ 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php',
+ 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php',
+ 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php',
+ 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php',
+ 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php',
+ 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php',
+ 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php',
'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@@ -2310,6 +2313,7 @@
'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php',
'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php',
+ 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php',
'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
@@ -2543,7 +2547,6 @@
'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
- 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php',
'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php',
'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
@@ -2651,6 +2654,8 @@
'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php',
'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php',
'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php',
+ 'PhabricatorElasticSearchHost' => 'infrastructure/cluster/search/PhabricatorElasticSearchHost.php',
+ 'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php',
'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php',
'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php',
'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php',
@@ -3073,6 +3078,7 @@
'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php',
'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php',
'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php',
+ 'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php',
'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php',
'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php',
'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php',
@@ -3762,7 +3768,6 @@
'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php',
'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php',
'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php',
- 'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php',
'PhabricatorSearchConstraintException' => 'applications/search/exception/PhabricatorSearchConstraintException.php',
'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php',
'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php',
@@ -3785,6 +3790,7 @@
'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php',
'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php',
'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php',
+ 'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php',
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
'PhabricatorSearchIndexVersion' => 'applications/search/storage/PhabricatorSearchIndexVersion.php',
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
@@ -3804,6 +3810,7 @@
'PhabricatorSearchSchemaSpec' => 'applications/search/storage/PhabricatorSearchSchemaSpec.php',
'PhabricatorSearchScopeSetting' => 'applications/settings/setting/PhabricatorSearchScopeSetting.php',
'PhabricatorSearchSelectField' => 'applications/search/field/PhabricatorSearchSelectField.php',
+ 'PhabricatorSearchService' => 'infrastructure/cluster/search/PhabricatorSearchService.php',
'PhabricatorSearchStringListField' => 'applications/search/field/PhabricatorSearchStringListField.php',
'PhabricatorSearchSubscribersField' => 'applications/search/field/PhabricatorSearchSubscribersField.php',
'PhabricatorSearchTextField' => 'applications/search/field/PhabricatorSearchTextField.php',
@@ -7303,6 +7310,9 @@
'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException',
'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
+ 'PhabricatorClusterNoHostForRoleException' => 'Exception',
+ 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType',
+ 'PhabricatorClusterServiceHealthRecord' => 'Phobject',
'PhabricatorClusterStrandedException' => 'PhabricatorClusterException',
'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
@@ -7354,6 +7364,7 @@
'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController',
+ 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController',
'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
@@ -7624,7 +7635,6 @@
'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController',
'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
'PhabricatorDataNotAttachedException' => 'Exception',
- 'PhabricatorDatabaseHealthRecord' => 'Phobject',
'PhabricatorDatabaseRef' => 'Phobject',
'PhabricatorDatabaseRefParser' => 'Phobject',
'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
@@ -7738,6 +7748,7 @@
'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting',
'PhabricatorEditorSetting' => 'PhabricatorStringSetting',
'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
+ 'PhabricatorElasticSearchHost' => 'PhabricatorSearchHost',
'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel',
'PhabricatorEmailContentSource' => 'PhabricatorContentSource',
@@ -8208,6 +8219,7 @@
'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
+ 'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost',
'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorNamedQuery' => array(
'PhabricatorSearchDAO',
@@ -9074,7 +9086,6 @@
'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
'PhabricatorSearchBaseController' => 'PhabricatorController',
'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField',
- 'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions',
'PhabricatorSearchConstraintException' => 'Exception',
'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField',
@@ -9097,6 +9108,7 @@
'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule',
'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase',
'PhabricatorSearchField' => 'Phobject',
+ 'PhabricatorSearchHost' => 'Phobject',
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
'PhabricatorSearchIndexVersion' => 'PhabricatorSearchDAO',
'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
@@ -9116,6 +9128,7 @@
'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec',
'PhabricatorSearchScopeSetting' => 'PhabricatorInternalSetting',
'PhabricatorSearchSelectField' => 'PhabricatorSearchField',
+ 'PhabricatorSearchService' => 'Phobject',
'PhabricatorSearchStringListField' => 'PhabricatorSearchField',
'PhabricatorSearchSubscribersField' => 'PhabricatorSearchTokenizerField',
'PhabricatorSearchTextField' => 'PhabricatorSearchField',
diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php
--- a/src/applications/config/application/PhabricatorConfigApplication.php
+++ b/src/applications/config/application/PhabricatorConfigApplication.php
@@ -69,6 +69,7 @@
'databases/' => 'PhabricatorConfigClusterDatabasesController',
'notifications/' => 'PhabricatorConfigClusterNotificationsController',
'repositories/' => 'PhabricatorConfigClusterRepositoriesController',
+ 'search/' => 'PhabricatorConfigClusterSearchController',
),
),
);
diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
--- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
+++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
@@ -7,71 +7,74 @@
}
protected function executeChecks() {
- if (!$this->shouldUseElasticSearchEngine()) {
- return;
- }
-
- $engine = new PhabricatorElasticFulltextStorageEngine();
+ $services = PhabricatorSearchService::getAllServices();
- $index_exists = null;
- $index_sane = null;
- try {
- $index_exists = $engine->indexExists();
- if ($index_exists) {
- $index_sane = $engine->indexIsSane();
+ foreach ($services as $service) {
+ try {
+ $host = $service->getAnyHostForRole('read');
+ } catch (PhabricatorClusterNoHostForRoleException $e) {
+ // ignore the error
+ continue;
}
- } catch (Exception $ex) {
- $summary = pht('Elasticsearch is not reachable as configured.');
- $message = pht(
- 'Elasticsearch is configured (with the %s setting) but Phabricator '.
- 'encountered an exception when trying to test the index.'.
- "\n\n".
- '%s',
- phutil_tag('tt', array(), 'search.elastic.host'),
- phutil_tag('pre', array(), $ex->getMessage()));
+ if ($host instanceof PhabricatorElasticSearchHost) {
+ $index_exists = null;
+ $index_sane = null;
+ try {
+ $engine = $host->getEngine();
+ $index_exists = $engine->indexExists();
+ if ($index_exists) {
+ $index_sane = $engine->indexIsSane();
+ }
+ } catch (Exception $ex) {
+ $summary = pht('Elasticsearch is not reachable as configured.');
+ $message = pht(
+ 'Elasticsearch is configured (with the %s setting) but Phabricator'.
+ ' encountered an exception when trying to test the index.'.
+ "\n\n".
+ '%s',
+ phutil_tag('tt', array(), 'cluster.search'),
+ phutil_tag('pre', array(), $ex->getMessage()));
- $this->newIssue('elastic.misconfigured')
- ->setName(pht('Elasticsearch Misconfigured'))
- ->setSummary($summary)
- ->setMessage($message)
- ->addRelatedPhabricatorConfig('search.elastic.host');
- return;
- }
+ $this->newIssue('elastic.misconfigured')
+ ->setName(pht('Elasticsearch Misconfigured'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->addRelatedPhabricatorConfig('cluster.search');
+ return;
+ }
- if (!$index_exists) {
- $summary = pht(
- 'You enabled Elasticsearch but the index does not exist.');
+ if (!$index_exists) {
+ $summary = pht(
+ 'You enabled Elasticsearch but the index does not exist.');
- $message = pht(
- 'You likely enabled search.elastic.host without creating the '.
- 'index. Run `./bin/search init` to correct the index.');
+ $message = pht(
+ 'You likely enabled cluster.search without creating the '.
+ 'index. Run `./bin/search init` to correct the index.');
- $this
- ->newIssue('elastic.missing-index')
- ->setName(pht('Elasticsearch index Not Found'))
- ->setSummary($summary)
- ->setMessage($message)
- ->addRelatedPhabricatorConfig('search.elastic.host');
- } else if (!$index_sane) {
- $summary = pht(
- 'Elasticsearch index exists but needs correction.');
+ $this
+ ->newIssue('elastic.missing-index')
+ ->setName(pht('Elasticsearch index Not Found'))
+ ->setSummary($summary)
+ ->setMessage($message)
+ ->addRelatedPhabricatorConfig('cluster.search');
+ } else if (!$index_sane) {
+ $summary = pht(
+ 'Elasticsearch index exists but needs correction.');
- $message = pht(
- 'Either the Phabricator schema for Elasticsearch has changed '.
- 'or Elasticsearch created the index automatically. Run '.
- '`./bin/search init` to correct the index.');
+ $message = pht(
+ 'Either the Phabricator schema for Elasticsearch has changed '.
+ 'or Elasticsearch created the index automatically. Run '.
+ '`./bin/search init` to correct the index.');
- $this
- ->newIssue('elastic.broken-index')
- ->setName(pht('Elasticsearch index Incorrect'))
- ->setSummary($summary)
- ->setMessage($message);
+ $this
+ ->newIssue('elastic.broken-index')
+ ->setName(pht('Elasticsearch index Incorrect'))
+ ->setSummary($summary)
+ ->setMessage($message);
+ }
+ }
}
}
- protected function shouldUseElasticSearchEngine() {
- $search_engine = PhabricatorFulltextStorageEngine::loadEngine();
- return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine);
- }
}
diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
--- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
+++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php
@@ -198,6 +198,10 @@
'This option has been removed, you can use Dashboards to provide '.
'homepage customization. See T11533 for more details.');
+ $elastic_reason = pht(
+ 'Elasticsearch is now configured with "%s".',
+ 'cluster.search');
+
$ancient_config += array(
'phid.external-loaders' =>
pht(
@@ -348,6 +352,10 @@
'mysql.configuration-provider' => pht(
'Phabricator now has application-level management of partitioning '.
'and replicas.'),
+
+ 'search.elastic.host' => $elastic_reason,
+ 'search.elastic.namespace' => $elastic_reason,
+
);
return $ancient_config;
diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
--- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php
+++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
@@ -379,8 +379,13 @@
}
protected function shouldUseMySQLSearchEngine() {
- $search_engine = PhabricatorFulltextStorageEngine::loadEngine();
- return ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine);
+ $services = PhabricatorSearchService::getAllServices();
+ foreach ($services as $service) {
+ if ($service instanceof PhabricatorMySQLSearchHost) {
+ return true;
+ }
+ }
+ return false;
}
}
diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
new file mode 100644
--- /dev/null
+++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
@@ -0,0 +1,129 @@
+<?php
+
+final class PhabricatorConfigClusterSearchController
+ extends PhabricatorConfigController {
+
+ public function handleRequest(AphrontRequest $request) {
+ $nav = $this->buildSideNavView();
+ $nav->selectFilter('cluster/search/');
+
+ $title = pht('Cluster Search');
+ $doc_href = PhabricatorEnv::getDoclink('Cluster: Search');
+
+ $header = id(new PHUIHeaderView())
+ ->setHeader($title)
+ ->setProfileHeader(true)
+ ->addActionLink(
+ id(new PHUIButtonView())
+ ->setIcon('fa-book')
+ ->setHref($doc_href)
+ ->setTag('a')
+ ->setText(pht('Documentation')));
+
+ $crumbs = $this
+ ->buildApplicationCrumbs($nav)
+ ->addTextCrumb($title)
+ ->setBorder(true);
+
+ $search_status = $this->buildClusterSearchStatus();
+
+ $content = id(new PhabricatorConfigPageView())
+ ->setHeader($header)
+ ->setContent($search_status);
+
+ return $this->newPage()
+ ->setTitle($title)
+ ->setCrumbs($crumbs)
+ ->setNavigation($nav)
+ ->appendChild($content)
+ ->addClass('white-background');
+ }
+
+ private function buildClusterSearchStatus() {
+ $viewer = $this->getViewer();
+
+ $services = PhabricatorSearchService::getAllServices();
+ Javelin::initBehavior('phabricator-tooltips');
+
+ $view = array();
+ foreach ($services as $service) {
+ $view[] = $this->renderStatusView($service);
+ }
+ return $view;
+ }
+
+ private function renderStatusView($service) {
+ $head = array_merge(
+ array(pht('Type')),
+ array_keys($service->getStatusViewColumns()),
+ array(pht('Status')));
+
+ $rows = array();
+
+ $status_map = PhabricatorSearchService::getConnectionStatusMap();
+ $stats = false;
+ $stats_view = false;
+
+ foreach ($service->getHosts() as $host) {
+ try {
+ $status = $host->getConnectionStatus();
+ $status = idx($status_map, $status, array());
+ } catch (Exception $ex) {
+ $status['icon'] = 'fa-times';
+ $status['label'] = pht('Connection Error');
+ $status['color'] = 'red';
+ $host->didHealthCheck(false);
+ }
+
+ if (!$stats_view) {
+ try {
+ $stats = $host->getEngine()->getIndexStats($host);
+ $stats_view = $this->renderIndexStats($stats);
+ } catch (Exception $e) {
+ $stats_view = false;
+ }
+ }
+
+ $type_icon = 'fa-search sky';
+ $type_tip = $host->getDisplayName();
+
+ $type_icon = id(new PHUIIconView())
+ ->setIcon($type_icon);
+ $status_view = array(
+ id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']),
+ ' ',
+ $status['label'],
+ );
+ $row = array(array($type_icon, ' ', $type_tip));
+ $row = array_merge($row, array_values(
+ $host->getStatusViewColumns()));
+ $row[] = $status_view;
+ $rows[] = $row;
+ }
+
+ $table = id(new AphrontTableView($rows))
+ ->setNoDataString(pht('No search servers are configured.'))
+ ->setHeaders($head);
+
+ $view = id(new PHUIObjectBoxView())
+ ->setHeaderText($service->getDisplayName())
+ ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
+ ->setTable($table);
+
+ if ($stats_view) {
+ $view->addPropertyList($stats_view);
+ }
+ return $view;
+ }
+
+ private function renderIndexStats($stats) {
+ $view = id(new PHUIPropertyListView());
+ if ($stats !== false) {
+ foreach ($stats as $label => $val) {
+ $view->addProperty($label, $val);
+ }
+ }
+ return $view;
+ }
+
+}
diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php
--- a/src/applications/config/controller/PhabricatorConfigController.php
+++ b/src/applications/config/controller/PhabricatorConfigController.php
@@ -42,8 +42,11 @@
pht('Notification Servers'), null, 'fa-bell-o');
$nav->addFilter('cluster/repositories/',
pht('Repository Servers'), null, 'fa-code');
+ $nav->addFilter('cluster/search/',
+ pht('Search Servers'), null, 'fa-search');
$nav->addLabel(pht('Modules'));
+
$modules = PhabricatorConfigModule::getAllModules();
foreach ($modules as $key => $module) {
$nav->addFilter('module/'.$key.'/',
diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php
--- a/src/applications/config/option/PhabricatorClusterConfigOptions.php
+++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php
@@ -38,6 +38,17 @@
$intro_href = PhabricatorEnv::getDoclink('Clustering Introduction');
$intro_name = pht('Clustering Introduction');
+ $search_type = 'custom:PhabricatorClusterSearchConfigOptionType';
+ $search_help = $this->deformat(pht(<<<EOTEXT
+Define one or more fulltext storage services. Here you can configure which
+hosts will handle fulltext search queries and indexing. For help with
+configuring fulltext search clusters, see **[[ %s | %s ]]** in the
+documentation.
+EOTEXT
+ ,
+ PhabricatorEnv::getDoclink('Cluster: Search'),
+ pht('Cluster: Search')));
+
return array(
$this->newOption('cluster.addresses', 'list<string>', array())
->setLocked(true)
@@ -114,6 +125,21 @@
->setSummary(
pht('Configure database read replicas.'))
->setDescription($databases_help),
+ $this->newOption('cluster.search', $search_type, array())
+ ->setLocked(true)
+ ->setSummary(
+ pht('Configure full-text search services.'))
+ ->setDescription($search_help)
+ ->setDefault(
+ array(
+ array(
+ 'type' => 'mysql',
+ 'roles' => array(
+ 'read' => true,
+ 'write' => true,
+ ),
+ ),
+ )),
);
}
diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php
--- a/src/applications/maniphest/query/ManiphestTaskQuery.php
+++ b/src/applications/maniphest/query/ManiphestTaskQuery.php
@@ -513,14 +513,14 @@
->setEngineClassName('PhabricatorSearchApplicationSearchEngine')
->setParameter('query', $this->fullTextSearch);
- // NOTE: Setting this to something larger than 2^53 will raise errors in
+ // NOTE: Setting this to something larger than 10,000 will raise errors in
// ElasticSearch, and billions of results won't fit in memory anyway.
- $fulltext_query->setParameter('limit', 100000);
+ $fulltext_query->setParameter('limit', 10000);
$fulltext_query->setParameter('types',
array(ManiphestTaskPHIDType::TYPECONST));
- $engine = PhabricatorFulltextStorageEngine::loadEngine();
- $fulltext_results = $engine->executeSearch($fulltext_query);
+ $fulltext_results = PhabricatorSearchService::executeSearch(
+ $fulltext_query);
if (empty($fulltext_results)) {
$fulltext_results = array(null);
diff --git a/src/applications/project/search/PhabricatorProjectFulltextEngine.php b/src/applications/project/search/PhabricatorProjectFulltextEngine.php
--- a/src/applications/project/search/PhabricatorProjectFulltextEngine.php
+++ b/src/applications/project/search/PhabricatorProjectFulltextEngine.php
@@ -10,7 +10,15 @@
$project = $object;
$project->updateDatasourceTokens();
- $document->setDocumentTitle($project->getName());
+ $document->setDocumentTitle($project->getDisplayName());
+ $document->addField(PhabricatorSearchDocumentFieldType::FIELD_KEYWORDS,
+ $project->getPrimarySlug());
+ try {
+ $slugs = $project->getSlugs();
+ foreach ($slugs as $slug) {}
+ } catch (PhabricatorDataNotAttachedException $e) {
+ // ignore
+ }
$document->addRelationship(
$project->isArchived()
diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php
deleted file mode 100644
--- a/src/applications/search/config/PhabricatorSearchConfigOptions.php
+++ /dev/null
@@ -1,35 +0,0 @@
-<?php
-
-final class PhabricatorSearchConfigOptions
- extends PhabricatorApplicationConfigOptions {
-
- public function getName() {
- return pht('Search');
- }
-
- public function getDescription() {
- return pht('Options relating to Search.');
- }
-
- public function getIcon() {
- return 'fa-search';
- }
-
- public function getGroup() {
- return 'apps';
- }
-
- public function getOptions() {
- return array(
- $this->newOption('search.elastic.host', 'string', null)
- ->setLocked(true)
- ->setDescription(pht('Elastic Search host.'))
- ->addExample('http://elastic.example.com:9200/', pht('Valid Setting')),
- $this->newOption('search.elastic.namespace', 'string', 'phabricator')
- ->setLocked(true)
- ->setDescription(pht('Elastic Search index.'))
- ->addExample('phabricator2', pht('Valid Setting')),
- );
- }
-
-}
diff --git a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
--- a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
+++ b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
@@ -5,5 +5,6 @@
const FIELD_TITLE = 'titl';
const FIELD_BODY = 'body';
const FIELD_COMMENT = 'cmnt';
+ const FIELD_KEYWORDS = 'kwrd';
}
diff --git a/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php b/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php
--- a/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php
+++ b/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php
@@ -3,8 +3,8 @@
final class PhabricatorSearchEngineTestCase extends PhabricatorTestCase {
public function testLoadAllEngines() {
- PhabricatorFulltextStorageEngine::loadAllEngines();
- $this->assertTrue(true);
+ $services = PhabricatorSearchService::getAllServices();
+ $this->assertTrue(!empty($services));
}
}
diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
--- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
+++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
@@ -1,37 +1,52 @@
<?php
-final class PhabricatorElasticFulltextStorageEngine
+class PhabricatorElasticFulltextStorageEngine
extends PhabricatorFulltextStorageEngine {
- private $uri;
private $index;
private $timeout;
-
- public function __construct() {
- $this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
- $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace');
+ private $version;
+
+ public function setService(PhabricatorSearchService $service) {
+ $this->service = $service;
+ $config = $service->getConfig();
+ $index = idx($config, 'path', '/phabricator');
+ $this->index = str_replace('/', '', $index);
+ $this->timeout = idx($config, 'timeout', 15);
+ $this->version = (int)idx($config, 'version', 5);
+ return $this;
}
public function getEngineIdentifier() {
return 'elasticsearch';
}
- public function getEnginePriority() {
- return 10;
+ public function getTimestampField() {
+ return $this->version < 2 ?
+ '_timestamp' : 'lastModified';
}
- public function isEnabled() {
- return (bool)$this->uri;
+ public function getTextFieldType() {
+ return $this->version >= 5
+ ? 'text' : 'string';
}
- public function setURI($uri) {
- $this->uri = $uri;
- return $this;
+ public function getHostType() {
+ return new PhabricatorElasticSearchHost($this);
}
- public function setIndex($index) {
- $this->index = $index;
- return $this;
+ /**
+ * @return PhabricatorElasticSearchHost
+ */
+ public function getHostForRead() {
+ return $this->getService()->getAnyHostForRole('read');
+ }
+
+ /**
+ * @return PhabricatorElasticSearchHost
+ */
+ public function getHostForWrite() {
+ return $this->getService()->getAnyHostForRole('write');
}
public function setTimeout($timeout) {
@@ -39,21 +54,21 @@
return $this;
}
- public function getURI() {
- return $this->uri;
- }
-
- public function getIndex() {
- return $this->index;
- }
-
public function getTimeout() {
return $this->timeout;
}
+ public function getTypeConstants($class) {
+ $relationship_class = new ReflectionClass($class);
+ $typeconstants = $relationship_class->getConstants();
+ return array_unique(array_values($typeconstants));
+ }
+
public function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
+ $host = $this->getHostForWrite();
+
$type = $doc->getDocumentType();
$phid = $doc->getPHID();
$handle = id(new PhabricatorHandleQuery())
@@ -61,36 +76,47 @@
->withPHIDs(array($phid))
->executeOne();
+ $timestamp_key = $this->getTimestampField();
+
// URL is not used internally but it can be useful externally.
$spec = array(
'title' => $doc->getDocumentTitle(),
'url' => PhabricatorEnv::getProductionURI($handle->getURI()),
'dateCreated' => $doc->getDocumentCreated(),
- '_timestamp' => $doc->getDocumentModified(),
- 'field' => array(),
- 'relationship' => array(),
+ $timestamp_key => $doc->getDocumentModified(),
);
foreach ($doc->getFieldData() as $field) {
- $spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
+ list($field_name, $corpus, $aux) = $field;
+ if (!isset($spec[$field_name])) {
+ $spec[$field_name] = array($corpus);
+ } else {
+ $spec[$field_name][] = $corpus;
+ }
+ if ($aux != null) {
+ $spec[$field_name][] = $aux;
+ }
}
- foreach ($doc->getRelationshipData() as $relationship) {
- list($rtype, $to_phid, $to_type, $time) = $relationship;
- $spec['relationship'][$rtype][] = array(
- 'phid' => $to_phid,
- 'phidType' => $to_type,
- 'when' => (int)$time,
- );
+ foreach ($doc->getRelationshipData() as $field) {
+ list($field_name, $related_phid, $rtype, $time) = $field;
+ if (!isset($spec[$field_name])) {
+ $spec[$field_name] = array($related_phid);
+ } else {
+ $spec[$field_name][] = $related_phid;
+ }
+ if ($time) {
+ $spec[$field_name.'_ts'] = $time;
+ }
}
- $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
+ $this->executeRequest($host, "/{$type}/{$phid}/", $spec, 'PUT');
}
public function reconstructDocument($phid) {
$type = phid_get_type($phid);
-
- $response = $this->executeRequest("/{$type}/{$phid}", array());
+ $host = $this->getHostForRead();
+ $response = $this->executeRequest($host, "/{$type}/{$phid}", array());
if (empty($response['exists'])) {
return null;
@@ -103,10 +129,11 @@
$doc->setDocumentType($response['_type']);
$doc->setDocumentTitle($hit['title']);
$doc->setDocumentCreated($hit['dateCreated']);
- $doc->setDocumentModified($hit['_timestamp']);
+ $doc->setDocumentModified($hit[$this->getTimestampField()]);
foreach ($hit['field'] as $fdef) {
- $doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
+ $field_type = $fdef['type'];
+ $doc->addField($field_type, $hit[$field_type], $fdef['aux']);
}
foreach ($hit['relationship'] as $rtype => $rships) {
@@ -123,35 +150,51 @@
}
private function buildSpec(PhabricatorSavedQuery $query) {
- $spec = array();
- $filter = array();
- $title_spec = array();
-
- if (strlen($query->getParameter('query'))) {
- $spec[] = array(
+ $q = new PhabricatorElasticSearchQueryBuilder('bool');
+ $query_string = $query->getParameter('query');
+ if (strlen($query_string)) {
+ $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');
+
+ // Build a simple_query_string query over all fields that must match all
+ // of the words in the search string.
+ $q->addMustClause(array(
'simple_query_string' => array(
- 'query' => $query->getParameter('query'),
- 'fields' => array('field.corpus'),
+ 'query' => $query_string,
+ 'fields' => array(
+ '_all',
+ ),
+ 'default_operator' => 'OR',
),
- );
+ ));
- $title_spec = array(
+ // This second query clause is "SHOULD' so it only affects ranking of
+ // documents which already matched the Must clause. This amplifies the
+ // score of documents which have an exact match on title, body
+ // or comments.
+ $q->addShouldClause(array(
'simple_query_string' => array(
- 'query' => $query->getParameter('query'),
- 'fields' => array('title'),
+ 'query' => $query_string,
+ 'fields' => array(
+ PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4',
+ PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3',
+ PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2',
+ ),
+ 'analyzer' => 'english_exact',
+ 'default_operator' => 'and',
),
- );
+ ));
+
}
$exclude = $query->getParameter('exclude');
if ($exclude) {
- $filter[] = array(
+ $q->addFilterClause(array(
'not' => array(
'ids' => array(
'values' => array($exclude),
),
),
- );
+ ));
}
$relationship_map = array(
@@ -176,75 +219,59 @@
$include_closed = !empty($statuses[$rel_closed]);
if ($include_open && !$include_closed) {
- $relationship_map[$rel_open] = true;
+ $q->addExistsClause($rel_open);
} else if (!$include_open && $include_closed) {
- $relationship_map[$rel_closed] = true;
+ $q->addExistsClause($rel_closed);
}
if ($query->getParameter('withUnowned')) {
- $relationship_map[$rel_unowned] = true;
+ $q->addExistsClause($rel_unowned);
}
$rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER;
if ($query->getParameter('withAnyOwner')) {
- $relationship_map[$rel_owner] = true;
+ $q->addExistsClause($rel_owner);
} else {
$owner_phids = $query->getParameter('ownerPHIDs', array());
- $relationship_map[$rel_owner] = $owner_phids;
- }
-
- foreach ($relationship_map as $field => $param) {
- if (is_array($param) && $param) {
- $should = array();
- foreach ($param as $val) {
- $should[] = array(
- 'match' => array(
- "relationship.{$field}.phid" => array(
- 'query' => $val,
- 'type' => 'phrase',
- ),
- ),
- );
- }
- // We couldn't solve it by minimum_number_should_match because it can
- // match multiple owners without matching author.
- $spec[] = array('bool' => array('should' => $should));
- } else if ($param) {
- $filter[] = array(
- 'exists' => array(
- 'field' => "relationship.{$field}.phid",
- ),
- );
+ if (count($owner_phids)) {
+ $q->addTermsClause($rel_owner, $owner_phids);
}
}
- if ($spec) {
- $spec = array('query' => array('bool' => array('must' => $spec)));
- if ($title_spec) {
- $spec['query']['bool']['should'] = $title_spec;
+ foreach ($relationship_map as $field => $phids) {
+ if (is_array($phids) && !empty($phids)) {
+ $q->addTermsClause($field, $phids);
}
}
- if ($filter) {
- $filter = array('filter' => array('and' => $filter));
- if (!$spec) {
- $spec = array('query' => array('match_all' => new stdClass()));
- }
- $spec = array(
- 'query' => array(
- 'filtered' => $spec + $filter,
- ),
- );
+ if (!$q->getClauseCount('must')) {
+ $q->addMustClause(array('match_all' => array('boost' => 1 )));
}
+ $spec = array(
+ '_source' => false,
+ 'query' => array(
+ 'bool' => $q->toArray(),
+ ),
+ );
+
+
if (!$query->getParameter('query')) {
$spec['sort'] = array(
array('dateCreated' => 'desc'),
);
}
- $spec['from'] = (int)$query->getParameter('offset', 0);
- $spec['size'] = (int)$query->getParameter('limit', 25);
+ $offset = (int)$query->getParameter('offset', 0);
+ $limit = (int)$query->getParameter('limit', 101);
+ if ($offset + $limit > 10000) {
+ throw new Exception(pht(
+ 'Query offset is too large. offset+limit=%s (max=%s)',
+ $offset + $limit,
+ 10000));
+ }
+ $spec['from'] = $offset;
+ $spec['size'] = $limit;
return $spec;
}
@@ -261,30 +288,36 @@
// some bigger index). Use '/$types/_search' instead.
$uri = '/'.implode(',', $types).'/_search';
- try {
- $response = $this->executeRequest($uri, $this->buildSpec($query));
- } catch (HTTPFutureHTTPResponseStatus $ex) {
- // elasticsearch probably uses Lucene query syntax:
- // http://lucene.apache.org/core/3_6_1/queryparsersyntax.html
- // Try literal search if operator search fails.
- if (!strlen($query->getParameter('query'))) {
- throw $ex;
+ $spec = $this->buildSpec($query);
+ $exceptions = array();
+
+ foreach ($this->service->getAllHostsForRole('read') as $host) {
+ try {
+ $response = $this->executeRequest($host, $uri, $spec);
+ $phids = ipull($response['hits']['hits'], '_id');
+ return $phids;
+ } catch (Exception $e) {
+ $exceptions[] = $e;
}
- $query = clone $query;
- $query->setParameter(
- 'query',
- addcslashes(
- $query->getParameter('query'), '+-&|!(){}[]^"~*?:\\'));
- $response = $this->executeRequest($uri, $this->buildSpec($query));
}
-
- $phids = ipull($response['hits']['hits'], '_id');
- return $phids;
+ throw new PhutilAggregateException('All search hosts failed:', $exceptions);
}
- public function indexExists() {
+ public function indexExists(PhabricatorElasticSearchHost $host = null) {
+ if (!$host) {
+ $host = $this->getHostForRead();
+ }
try {
- return (bool)$this->executeRequest('/_status/', array());
+ if ($this->version >= 5) {
+ $uri = '/_stats/';
+ $res = $this->executeRequest($host, $uri, array());
+ return isset($res['indices']['phabricator']);
+ } else if ($this->version >= 2) {
+ $uri = '';
+ } else {
+ $uri = '/_status/';
+ }
+ return (bool)$this->executeRequest($host, $uri, array());
} catch (HTTPFutureHTTPResponseStatus $e) {
if ($e->getStatusCode() == 404) {
return false;
@@ -299,53 +332,85 @@
'index' => array(
'auto_expand_replicas' => '0-2',
'analysis' => array(
- 'filter' => array(
- 'trigrams_filter' => array(
- 'min_gram' => 3,
- 'type' => 'ngram',
- 'max_gram' => 3,
- ),
- ),
'analyzer' => array(
- 'custom_trigrams' => array(
- 'type' => 'custom',
- 'filter' => array(
- 'lowercase',
- 'kstem',
- 'trigrams_filter',
- ),
+ 'english_exact' => array(
'tokenizer' => 'standard',
+ 'filter' => array('lowercase'),
),
),
),
),
);
- $types = array_keys(
+ $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');
+ $relationships = $this->getTypeConstants('PhabricatorSearchRelationship');
+
+ $doc_types = array_keys(
PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
- foreach ($types as $type) {
- // Use the custom trigram analyzer for the corpus of text
- $data['mappings'][$type]['properties']['field']['properties']['corpus'] =
- array('type' => 'string', 'analyzer' => 'custom_trigrams');
+
+ $text_type = $this->getTextFieldType();
+
+ foreach ($doc_types as $type) {
+ $properties = array();
+ foreach ($fields as $field) {
+ // Use the custom analyzer for the corpus of text
+ $properties[$field] = array(
+ 'type' => $text_type,
+ 'analyzer' => 'english_exact',
+ 'search_analyzer' => 'english',
+ 'search_quote_analyzer' => 'english_exact',
+ );
+ }
+
+ if ($this->version < 5) {
+ foreach ($relationships as $rel) {
+ $properties[$rel] = array(
+ 'type' => 'string',
+ 'index' => 'not_analyzed',
+ 'include_in_all' => false,
+ );
+ $properties[$rel.'_ts'] = array(
+ 'type' => 'date',
+ 'include_in_all' => false,
+ );
+ }
+ } else {
+ foreach ($relationships as $rel) {
+ $properties[$rel] = array(
+ 'type' => 'keyword',
+ 'include_in_all' => false,
+ 'doc_values' => false,
+ );
+ $properties[$rel.'_ts'] = array(
+ 'type' => 'date',
+ 'include_in_all' => false,
+ );
+ }
+ }
// Ensure we have dateCreated since the default query requires it
- $data['mappings'][$type]['properties']['dateCreated']['type'] = 'string';
- }
+ $properties['dateCreated']['type'] = 'date';
+ $properties['lastModified']['type'] = 'date';
+ $data['mappings'][$type]['properties'] = $properties;
+ }
return $data;
}
- public function indexIsSane() {
- if (!$this->indexExists()) {
+ public function indexIsSane(PhabricatorElasticSearchHost $host = null) {
+ if (!$host) {
+ $host = $this->getHostForRead();
+ }
+ if (!$this->indexExists($host)) {
return false;
}
-
- $cur_mapping = $this->executeRequest('/_mapping/', array());
- $cur_settings = $this->executeRequest('/_settings/', array());
+ $cur_mapping = $this->executeRequest($host, '/_mapping/', array());
+ $cur_settings = $this->executeRequest($host, '/_settings/', array());
$actual = array_merge($cur_settings[$this->index],
$cur_mapping[$this->index]);
- return $this->check($actual, $this->getIndexConfiguration());
+ $res = $this->check($actual, $this->getIndexConfiguration());
+ return $res;
}
/**
@@ -355,7 +420,7 @@
* @param $required array
* @return bool
*/
- private function check($actual, $required) {
+ private function check($actual, $required, $path = '') {
foreach ($required as $key => $value) {
if (!array_key_exists($key, $actual)) {
if ($key === '_all') {
@@ -369,7 +434,7 @@
if (!is_array($actual[$key])) {
return false;
}
- if (!$this->check($actual[$key], $value)) {
+ if (!$this->check($actual[$key], $value, $path.'.'.$key)) {
return false;
}
continue;
@@ -403,19 +468,44 @@
}
public function initIndex() {
+ $host = $this->getHostForWrite();
if ($this->indexExists()) {
- $this->executeRequest('/', array(), 'DELETE');
+ $this->executeRequest($host, '/', array(), 'DELETE');
}
$data = $this->getIndexConfiguration();
- $this->executeRequest('/', $data, 'PUT');
+ $this->executeRequest($host, '/', $data, 'PUT');
}
- private function executeRequest($path, array $data, $method = 'GET') {
- $uri = new PhutilURI($this->uri);
- $uri->setPath($this->index);
- $uri->appendPath($path);
- $data = json_encode($data);
+ public function getIndexStats(PhabricatorElasticSearchHost $host = null) {
+ if ($this->version < 2) {
+ return false;
+ }
+ if (!$host) {
+ $host = $this->getHostForRead();
+ }
+ $uri = '/_stats/';
+ $host = $this->getHostForRead();
+
+ $res = $this->executeRequest($host, $uri, array());
+ $stats = $res['indices'][$this->index];
+ return array(
+ pht('Queries') =>
+ idxv($stats, array('primaries', 'search', 'query_total')),
+ pht('Documents') =>
+ idxv($stats, array('total', 'docs', 'count')),
+ pht('Deleted') =>
+ idxv($stats, array('total', 'docs', 'deleted')),
+ pht('Storage Used') =>
+ phutil_format_bytes(idxv($stats,
+ array('total', 'store', 'size_in_bytes'))),
+ );
+ }
+
+ private function executeRequest(PhabricatorElasticSearchHost $host, $path,
+ array $data, $method = 'GET') {
+ $uri = $host->getURI($path);
+ $data = json_encode($data);
$future = new HTTPSFuture($uri, $data);
if ($method != 'GET') {
$future->setMethod($method);
@@ -423,19 +513,30 @@
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
- list($body) = $future->resolvex();
+ try {
+ list($body) = $future->resolvex();
+ } catch (HTTPFutureResponseStatus $ex) {
+ if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) {
+ $host->didHealthCheck(false);
+ }
+ throw $ex;
+ }
if ($method != 'GET') {
return null;
}
try {
- return phutil_json_decode($body);
+ $data = phutil_json_decode($body);
+ $host->didHealthCheck(true);
+ return $data;
} catch (PhutilJSONParserException $ex) {
+ $host->didHealthCheck(false);
throw new PhutilProxyException(
pht('ElasticSearch server returned invalid JSON!'),
$ex);
}
+
}
}
diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php
new file mode 100644
--- /dev/null
+++ b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php
@@ -0,0 +1,78 @@
+<?php
+
+class PhabricatorElasticSearchQueryBuilder {
+ protected $name;
+ protected $clauses = array();
+
+
+ public function getClauses($termkey = null) {
+ $clauses = $this->clauses;
+ if ($termkey == null) {
+ return $clauses;
+ }
+ if (isset($clauses[$termkey])) {
+ return $clauses[$termkey];
+ }
+ return array();
+ }
+
+ public function getClauseCount($clausekey) {
+ if (isset($this->clauses[$clausekey])) {
+ return count($this->clauses[$clausekey]);
+ } else {
+ return 0;
+ }
+ }
+
+ public function addExistsClause($field) {
+ return $this->addClause('filter', array(
+ 'exists' => array(
+ 'field' => $field,
+ ),
+ ));
+ }
+
+ public function addTermsClause($field, $values) {
+ return $this->addClause('filter', array(
+ 'terms' => array(
+ $field => array_values($values),
+ ),
+ ));
+ }
+
+ public function addMustClause($clause) {
+ return $this->addClause('must', $clause);
+ }
+
+ public function addFilterClause($clause) {
+ return $this->addClause('filter', $clause);
+ }
+
+ public function addShouldClause($clause) {
+ return $this->addClause('should', $clause);
+ }
+
+ public function addMustNotClause($clause) {
+ return $this->addClause('must_not', $clause);
+ }
+
+ public function addClause($clause, $terms) {
+ $this->clauses[$clause][] = $terms;
+ return $this;
+ }
+
+ public function toArray() {
+ $clauses = $this->getClauses();
+ return $clauses;
+ $cleaned = array();
+ foreach ($clauses as $clause => $subclauses) {
+ if (is_array($subclauses) && count($subclauses) == 1) {
+ $cleaned[$clause] = array_shift($subclauses);
+ } else {
+ $cleaned[$clause] = $subclauses;
+ }
+ }
+ return $cleaned;
+ }
+
+}
diff --git a/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php
--- a/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php
+++ b/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php
@@ -7,46 +7,40 @@
*/
abstract class PhabricatorFulltextStorageEngine extends Phobject {
-/* -( Engine Metadata )---------------------------------------------------- */
+ protected $service;
+
+ public function getHosts() {
+ return $this->service->getHosts();
+ }
+
+ public function setService(PhabricatorSearchService $service) {
+ $this->service = $service;
+ return $this;
+ }
/**
- * Return a unique, nonempty string which identifies this storage engine.
- *
- * @return string Unique string for this engine, max length 32.
- * @task meta
+ * @return PhabricatorSearchService
*/
- abstract public function getEngineIdentifier();
+ public function getService() {
+ return $this->service;
+ }
/**
- * Prioritize this engine relative to other engines.
- *
- * Engines with a smaller priority number get an opportunity to write files
- * first. Generally, lower-latency filestores should have lower priority
- * numbers, and higher-latency filestores should have higher priority
- * numbers. Setting priority to approximately the number of milliseconds of
- * read latency will generally produce reasonable results.
- *
- * In conjunction with filesize limits, the goal is to store small files like
- * profile images, thumbnails, and text snippets in lower-latency engines,
- * and store large files in higher-capacity engines.
- *
- * @return float Engine priority.
- * @task meta
+ * Implementations must return a prototype host instance which is cloned
+ * by the PhabricatorSearchService infrastructure to configure each engine.
+ * @return PhabricatorSearchHost
*/
- abstract public function getEnginePriority();
+ abstract public function getHostType();
+
+/* -( Engine Metadata )---------------------------------------------------- */
/**
- * Return `true` if the engine is currently writable.
- *
- * Engines that are disabled or missing configuration should return `false`
- * to prevent new writes. If writes were made with this engine in the past,
- * the application may still try to perform reads.
+ * Return a unique, nonempty string which identifies this storage engine.
*
- * @return bool True if this engine can support new writes.
+ * @return string Unique string for this engine, max length 32.
* @task meta
*/
- abstract public function isEnabled();
-
+ abstract public function getEngineIdentifier();
/* -( Managing Documents )------------------------------------------------- */
@@ -84,6 +78,13 @@
abstract public function indexExists();
/**
+ * Implementations should override this method to return a dictionary of
+ * stats which are suitable for display in the admin UI.
+ */
+ abstract public function getIndexStats();
+
+
+ /**
* Is the index in a usable state?
*
* @return bool
@@ -100,39 +101,4 @@
public function initIndex() {}
-/* -( Loading Storage Engines )-------------------------------------------- */
-
- /**
- * @task load
- */
- public static function loadAllEngines() {
- return id(new PhutilClassMapQuery())
- ->setAncestorClass(__CLASS__)
- ->setUniqueMethod('getEngineIdentifier')
- ->setSortMethod('getEnginePriority')
- ->execute();
- }
-
- /**
- * @task load
- */
- public static function loadActiveEngines() {
- $engines = self::loadAllEngines();
-
- $active = array();
- foreach ($engines as $key => $engine) {
- if (!$engine->isEnabled()) {
- continue;
- }
-
- $active[$key] = $engine;
- }
-
- return $active;
- }
-
- public static function loadEngine() {
- return head(self::loadActiveEngines());
- }
-
}
diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
--- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
+++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
@@ -7,12 +7,8 @@
return 'mysql';
}
- public function getEnginePriority() {
- return 100;
- }
-
- public function isEnabled() {
- return true;
+ public function getHostType() {
+ return new PhabricatorMySQLSearchHost($this);
}
public function reindexAbstractDocument(
@@ -415,4 +411,9 @@
public function indexExists() {
return true;
}
+
+ public function getIndexStats() {
+ return false;
+ }
+
}
diff --git a/src/applications/search/index/PhabricatorFulltextEngine.php b/src/applications/search/index/PhabricatorFulltextEngine.php
--- a/src/applications/search/index/PhabricatorFulltextEngine.php
+++ b/src/applications/search/index/PhabricatorFulltextEngine.php
@@ -40,8 +40,7 @@
$extension->indexFulltextObject($object, $document);
}
- $storage_engine = PhabricatorFulltextStorageEngine::loadEngine();
- $storage_engine->reindexAbstractDocument($document);
+ PhabricatorSearchService::reindexAbstractDocument($document);
}
protected function newAbstractDocument($object) {
diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
--- a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
+++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
@@ -13,27 +13,41 @@
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
- $engine = PhabricatorFulltextStorageEngine::loadEngine();
-
$work_done = false;
- if (!$engine->indexExists()) {
- $console->writeOut(
- '%s',
- pht('Index does not exist, creating...'));
- $engine->initIndex();
- $console->writeOut(
- "%s\n",
- pht('done.'));
- $work_done = true;
- } else if (!$engine->indexIsSane()) {
- $console->writeOut(
- '%s',
- pht('Index exists but is incorrect, fixing...'));
- $engine->initIndex();
+ foreach (PhabricatorSearchService::getAllServices() as $service) {
$console->writeOut(
"%s\n",
- pht('done.'));
- $work_done = true;
+ pht('Initializing search service "%s"', $service->getDisplayName()));
+
+ try {
+ $host = $service->getAnyHostForRole('write');
+ } catch (PhabricatorClusterNoHostForRoleException $e) {
+ // If there are no writable hosts for a given cluster, skip it
+ $console->writeOut("%s\n", $e->getMessage());
+ continue;
+ }
+
+ $engine = $host->getEngine();
+
+ if (!$engine->indexExists()) {
+ $console->writeOut(
+ '%s',
+ pht('Index does not exist, creating...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ } else if (!$engine->indexIsSane()) {
+ $console->writeOut(
+ '%s',
+ pht('Index exists but is incorrect, fixing...'));
+ $engine->initIndex();
+ $console->writeOut(
+ "%s\n",
+ pht('done.'));
+ $work_done = true;
+ }
}
if ($work_done) {
diff --git a/src/applications/search/query/PhabricatorSearchDocumentQuery.php b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
--- a/src/applications/search/query/PhabricatorSearchDocumentQuery.php
+++ b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
@@ -73,10 +73,7 @@
$query = id(clone($this->savedQuery))
->setParameter('offset', $this->getOffset())
->setParameter('limit', $this->getRawResultLimit());
-
- $engine = PhabricatorFulltextStorageEngine::loadEngine();
-
- return $engine->executeSearch($query);
+ return PhabricatorSearchService::executeSearch($query);
}
public function getQueryApplicationClass() {
diff --git a/src/docs/user/cluster/cluster_search.diviner b/src/docs/user/cluster/cluster_search.diviner
new file mode 100644
--- /dev/null
+++ b/src/docs/user/cluster/cluster_search.diviner
@@ -0,0 +1,76 @@
+@title Cluster: Search
+@group cluster
+
+Overview
+========
+
+You can configure phabricator to connect to one or more fulltext search clusters
+running either Elasticsearch or MySQL. By default and without further
+configuration, Phabricator will use MySQL for fulltext search. This will be
+adequate for the vast majority of users. Installs with a very large number of
+objects or specialized search needs can consider enabling Elasticsearch for
+better scalability and potentially better search results.
+
+Configuring Search Services
+===========================
+
+To configure an Elasticsearch service, use the `cluster.search` configuration
+option. A typical Elasticsearch configuration will probably look similar to
+the following example:
+
+```lang=json
+{
+ "cluster.search": [
+ {
+ "type": "elasticsearch",
+ "hosts": [
+ {
+ "host": "127.0.0.1",
+ "roles": { "write": true, "read": true }
+ }
+ ],
+ "port": 9200,
+ "protocol": "http",
+ "path": "/phabricator",
+ "version": 5
+ },
+ ],
+}
+```
+
+Supported Options
+-----------------
+| Key | Type |Comments|
+|`type` | String |Engine type. Currently, 'elasticsearch' or 'mysql'|
+|`protocol`| String |Either 'http' or 'https'|
+|`port`| Int |The TCP port that Elasticsearch is bound to|
+|`path`| String |The path portion of the url for phabricator's index.|
+|`version`| Int |The version of Elasticsearch server. Supports either 2 or 5.|
+|`hosts`| List |A list of one or more Elasticsearch host names / addresses.|
+
+Host Configuration
+------------------
+Each search service must have one or more hosts associated with it. Each host
+entry consists of a `host` key, a dictionary of roles and can optionally
+override any of the options that are valid at the service level (see above).
+
+Currently supported roles are `read` and `write`. These can be individually
+enabled or disabled on a per-host basis. A typical setup might include two
+elasticsearch clusters in two separate datacenters. You can configure one
+cluster for reads and both for writes. When one cluster is down for maintenance
+you can simply swap the read role over to the backup cluster and then proceed
+with maintenance without any service interruption.
+
+Monitoring Search Services
+==========================
+
+You can monitor fulltext search in {nav Config > Search Servers}. This interface
+shows you a quick overview of services and their health.
+
+The table on this page shows some basic stats for each configured service,
+followed by the configuration and current status of each host.
+
+NOTE: This page runs its diagnostics //from the web server that is serving the
+request//. If you are recovering from a disaster, the view this page shows
+may be partial or misleading, and two requests served by different servers may
+see different views of the cluster.
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
rename from src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
rename to src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
--- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
+++ b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
@@ -1,20 +1,19 @@
<?php
-final class PhabricatorDatabaseHealthRecord
+class PhabricatorClusterServiceHealthRecord
extends Phobject {
- private $ref;
+ private $cacheKey;
private $shouldCheck;
private $isHealthy;
private $upEventCount;
private $downEventCount;
- public function __construct(PhabricatorDatabaseRef $ref) {
- $this->ref = $ref;
+ public function __construct($cache_key) {
+ $this->cacheKey = $cache_key;
$this->readState();
}
-
/**
* Is the database currently healthy?
*/
@@ -153,18 +152,13 @@
}
}
- private function getHealthRecordCacheKey() {
- $ref = $this->ref;
-
- $host = $ref->getHost();
- $port = $ref->getPort();
-
- return "cluster.db.health({$host}, {$port})";
+ public function getCacheKey() {
+ return $this->cacheKey;
}
private function readHealthRecord() {
$cache = PhabricatorCaches::getSetupCache();
- $cache_key = $this->getHealthRecordCacheKey();
+ $cache_key = $this->getCacheKey();
$health_record = $cache->getKey($cache_key);
if (!is_array($health_record)) {
@@ -180,7 +174,7 @@
private function writeHealthRecord(array $record) {
$cache = PhabricatorCaches::getSetupCache();
- $cache_key = $this->getHealthRecordCacheKey();
+ $cache_key = $this->getCacheKey();
$cache->setKey($cache_key, $record);
}
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
@@ -14,6 +14,7 @@
const REPLICATION_SLOW = 'replica-slow';
const REPLICATION_NOT_REPLICATING = 'not-replicating';
+ const KEY_HEALTH = 'cluster.db.health';
const KEY_REFS = 'cluster.db.refs';
const KEY_INDIVIDUAL = 'cluster.db.individual';
@@ -489,9 +490,18 @@
return $this;
}
+ private function getHealthRecordCacheKey() {
+ $host = $this->getHost();
+ $port = $this->getPort();
+ $key = self::KEY_HEALTH;
+
+ return "{$key}({$host}, {$port})";
+ }
+
public function getHealthRecord() {
if (!$this->healthRecord) {
- $this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
+ $this->getHealthRecordCacheKey());
}
return $this->healthRecord;
}
diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php
rename from src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
rename to src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php
diff --git a/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php
@@ -0,0 +1,79 @@
+<?php
+
+final class PhabricatorClusterSearchConfigOptionType
+ extends PhabricatorConfigJSONOptionType {
+
+ public function validateOption(PhabricatorConfigOption $option, $value) {
+ if (!is_array($value)) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration is not valid: value must be a '.
+ 'list of search hosts.'));
+ }
+
+ $engines = PhabricatorSearchService::loadAllFulltextStorageEngines();
+
+ foreach ($value as $index => $spec) {
+ if (!is_array($spec)) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration is not valid: each entry in the '.
+ 'list must be a dictionary describing a search service, but '.
+ 'the value with index "%s" is not a dictionary.',
+ $index));
+ }
+
+ try {
+ PhutilTypeSpec::checkMap(
+ $spec,
+ array(
+ 'type' => 'string',
+ 'hosts' => 'optional list<map<string, wild>>',
+ 'roles' => 'optional map<string, wild>',
+ 'port' => 'optional int',
+ 'protocol' => 'optional string',
+ 'path' => 'optional string',
+ 'version' => 'optional int',
+ ));
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ 'Search engine configuration has an invalid service '.
+ 'specification (at index "%s"): %s.',
+ $index,
+ $ex->getMessage()));
+ }
+
+ if (!array_key_exists($spec['type'], $engines)) {
+ throw new Exception(
+ pht('Invalid search engine type: %s. Valid types include: %s',
+ $spec['type'],
+ implode(', ', array_keys($engines))));
+ }
+
+ if (isset($spec['hosts'])) {
+ foreach ($spec['hosts'] as $hostindex => $host) {
+ try {
+ PhutilTypeSpec::checkMap(
+ $host,
+ array(
+ 'host' => 'string',
+ 'roles' => 'optional map<string, wild>',
+ 'port' => 'optional int',
+ 'protocol' => 'optional string',
+ 'path' => 'optional string',
+ 'version' => 'optional int',
+ ));
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration has an invalid host '.
+ 'specification (at index "%s"): %s.',
+ $hostindex,
+ $ex->getMessage()));
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/infrastructure/cluster/PhabricatorClusterException.php b/src/infrastructure/cluster/exception/PhabricatorClusterException.php
rename from src/infrastructure/cluster/PhabricatorClusterException.php
rename to src/infrastructure/cluster/exception/PhabricatorClusterException.php
diff --git a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php b/src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php
rename from src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php
rename to src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php
diff --git a/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php
rename from src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php
rename to src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php
diff --git a/src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php
rename from src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php
rename to src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php
diff --git a/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php
@@ -0,0 +1,10 @@
+<?php
+
+final class PhabricatorClusterNoHostForRoleException
+ extends Exception {
+
+ public function __construct($role) {
+ parent::__construct(pht('Search cluster has no hosts for role "%s".',
+ $role));
+ }
+}
diff --git a/src/infrastructure/cluster/PhabricatorClusterStrandedException.php b/src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php
rename from src/infrastructure/cluster/PhabricatorClusterStrandedException.php
rename to src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php
diff --git a/src/infrastructure/cluster/search/PhabricatorElasticSearchHost.php b/src/infrastructure/cluster/search/PhabricatorElasticSearchHost.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/search/PhabricatorElasticSearchHost.php
@@ -0,0 +1,82 @@
+<?php
+
+final class PhabricatorElasticSearchHost
+ extends PhabricatorSearchHost {
+
+ private $version = 5;
+ private $path = 'phabricator/';
+ private $protocol = 'http';
+
+ const KEY_REFS = 'search.elastic.refs';
+
+
+ public function setConfig($config) {
+ $this->setRoles(idx($config, 'roles', $this->getRoles()))
+ ->setHost(idx($config, 'host', $this->host))
+ ->setPort(idx($config, 'port', $this->port))
+ ->setProtocol(idx($config, 'protocol', $this->protocol))
+ ->setPath(idx($config, 'path', $this->path))
+ ->setVersion(idx($config, 'version', $this->version));
+ return $this;
+ }
+
+ public function getDisplayName() {
+ return pht('ElasticSearch');
+ }
+
+ public function getStatusViewColumns() {
+ return array(
+ pht('Protocol') => $this->getProtocol(),
+ pht('Host') => $this->getHost(),
+ pht('Port') => $this->getPort(),
+ pht('Index Path') => $this->getPath(),
+ pht('Elastic Version') => $this->getVersion(),
+ pht('Roles') => implode(', ', array_keys($this->getRoles())),
+ );
+ }
+
+ public function setProtocol($protocol) {
+ $this->protocol = $protocol;
+ return $this;
+ }
+
+ public function getProtocol() {
+ return $this->protocol;
+ }
+
+ public function setPath($path) {
+ $this->path = $path;
+ return $this;
+ }
+
+ public function getPath() {
+ return $this->path;
+ }
+
+ public function setVersion($version) {
+ $this->version = $version;
+ return $this;
+ }
+
+ public function getVersion() {
+ return $this->version;
+ }
+
+ public function getURI($to_path = null) {
+ $uri = id(new PhutilURI('http://'.$this->getHost()))
+ ->setProtocol($this->getProtocol())
+ ->setPort($this->getPort())
+ ->setPath($this->getPath());
+
+ if ($to_path) {
+ $uri->appendPath($to_path);
+ }
+ return $uri;
+ }
+
+ public function getConnectionStatus() {
+ $status = $this->getEngine()->indexIsSane($this);
+ return $status ? parent::STATUS_OKAY : parent::STATUS_FAIL;
+ }
+
+}
diff --git a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php
@@ -0,0 +1,34 @@
+<?php
+
+final class PhabricatorMySQLSearchHost
+ extends PhabricatorSearchHost {
+
+ public function setConfig($config) {
+ $this->setRoles(idx($config, 'roles',
+ array('read' => true, 'write' => true)));
+ return $this;
+ }
+
+ public function getDisplayName() {
+ return 'MySQL';
+ }
+
+ public function getStatusViewColumns() {
+ return array(
+ pht('Protocol') => 'mysql',
+ pht('Roles') => implode(', ', array_keys($this->getRoles())),
+ );
+ }
+
+ public function getProtocol() {
+ return 'mysql';
+ }
+
+ public function getConnectionStatus() {
+ PhabricatorDatabaseRef::queryAll();
+ $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search');
+ $status = $ref->getConnectionStatus();
+ return $status;
+ }
+
+}
diff --git a/src/infrastructure/cluster/search/PhabricatorSearchHost.php b/src/infrastructure/cluster/search/PhabricatorSearchHost.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/search/PhabricatorSearchHost.php
@@ -0,0 +1,163 @@
+<?php
+
+abstract class PhabricatorSearchHost
+ extends Phobject {
+
+ const KEY_REFS = 'cluster.search.refs';
+ const KEY_HEALTH = 'cluster.search.health';
+
+ protected $engine;
+ protected $healthRecord;
+ protected $roles = array();
+
+ protected $disabled;
+ protected $host;
+ protected $port;
+ protected $hostRefs = array();
+
+ const STATUS_OKAY = 'okay';
+ const STATUS_FAIL = 'fail';
+
+ public function __construct(PhabricatorFulltextStorageEngine $engine) {
+ $this->engine = $engine;
+ }
+
+ public function setDisabled($disabled) {
+ $this->disabled = $disabled;
+ return $this;
+ }
+
+ public function getDisabled() {
+ return $this->disabled;
+ }
+
+ /**
+ * @return PhabricatorFulltextStorageEngine
+ */
+ public function getEngine() {
+ return $this->engine;
+ }
+
+ public function isWritable() {
+ return $this->hasRole('write');
+ }
+
+ public function isReadable() {
+ return $this->hasRole('read');
+ }
+
+ public function hasRole($role) {
+ return isset($this->roles[$role]) && $this->roles[$role] === true;
+ }
+
+ public function setRoles(array $roles) {
+ foreach ($roles as $role => $val) {
+ $this->roles[$role] = $val;
+ }
+ return $this;
+ }
+
+ public function getRoles() {
+ $roles = array();
+ foreach ($this->roles as $key => $val) {
+ if ($val) {
+ $roles[$key] = $val;
+ }
+ }
+ return $roles;
+ }
+
+ public function setPort($value) {
+ $this->port = $value;
+ return $this;
+ }
+
+ public function getPort() {
+ return $this->port;
+ }
+
+ public function setHost($value) {
+ $this->host = $value;
+ return $this;
+ }
+
+ public function getHost() {
+ return $this->host;
+ }
+
+
+ public function getHealthRecordCacheKey() {
+ $host = $this->getHost();
+ $port = $this->getPort();
+ $key = self::KEY_HEALTH;
+
+ return "{$key}({$host}, {$port})";
+ }
+
+/**
+ * @return PhabricatorClusterServiceHealthRecord
+ */
+ public function getHealthRecord() {
+ if (!$this->healthRecord) {
+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
+ $this->getHealthRecordCacheKey());
+ }
+ return $this->healthRecord;
+ }
+
+ public function didHealthCheck($reachable) {
+ $record = $this->getHealthRecord();
+ $should_check = $record->getShouldCheck();
+
+ if ($should_check) {
+ $record->didHealthCheck($reachable);
+ }
+ }
+
+ /**
+ * @return string[] Get a list of fields to show in the status overview UI
+ */
+ abstract public function getStatusViewColumns();
+
+ abstract public function getConnectionStatus();
+
+ public static function reindexAbstractDocument(
+ PhabricatorSearchAbstractDocument $doc) {
+
+ $services = self::getAllServices();
+ $indexed = 0;
+ foreach (self::getWritableHostForEachService() as $host) {
+ $host->getEngine()->reindexAbstractDocument($doc);
+ $indexed++;
+ }
+ if ($indexed == 0) {
+ throw new PhabricatorClusterNoHostForRoleException('write');
+ }
+ }
+
+ public static function executeSearch(PhabricatorSavedQuery $query) {
+ $services = self::getAllServices();
+ foreach ($services as $service) {
+ $hosts = $service->getAllHostsForRole('read');
+ // try all hosts until one succeeds
+ foreach ($hosts as $host) {
+ $last_exception = null;
+ try {
+ $res = $host->getEngine()->executeSearch($query);
+ // return immediately if we get results without an exception
+ $host->didHealthCheck(true);
+ return $res;
+ } catch (Exception $ex) {
+ // try each server in turn, only throw if none succeed
+ $last_exception = $ex;
+ $host->didHealthCheck(false);
+ }
+ }
+ }
+ if ($last_exception) {
+ throw $last_exception;
+ }
+ return $res;
+ }
+
+}
diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php
@@ -0,0 +1,259 @@
+<?php
+
+class PhabricatorSearchService
+ extends Phobject {
+
+ const KEY_REFS = 'cluster.search.refs';
+
+ protected $config;
+ protected $disabled;
+ protected $engine;
+ protected $hosts = array();
+ protected $hostsConfig;
+ protected $hostType;
+ protected $roles = array();
+
+ const STATUS_OKAY = 'okay';
+ const STATUS_FAIL = 'fail';
+
+ public function __construct(PhabricatorFulltextStorageEngine $engine) {
+ $this->engine = $engine;
+ $this->hostType = $engine->getHostType();
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function newHost($config) {
+ $host = clone($this->hostType);
+ $host_config = $this->config + $config;
+ $host->setConfig($host_config);
+ $this->hosts[] = $host;
+ return $host;
+ }
+
+ public function getEngine() {
+ return $this->engine;
+ }
+
+ public function getDisplayName() {
+ return $this->hostType->getDisplayName();
+ }
+
+ public function getStatusViewColumns() {
+ return $this->hostType->getStatusViewColumns();
+ }
+
+ public function setConfig($config) {
+ $this->config = $config;
+
+ if (!isset($config['hosts'])) {
+ $config['hosts'] = array(
+ array(
+ 'host' => idx($config, 'host'),
+ 'port' => idx($config, 'port'),
+ 'protocol' => idx($config, 'protocol'),
+ 'roles' => idx($config, 'roles'),
+ ),
+ );
+ }
+ foreach ($config['hosts'] as $host) {
+ $this->newHost($host);
+ }
+
+ }
+
+ public function getConfig() {
+ return $this->config;
+ }
+
+ public function setDisabled($disabled) {
+ $this->disabled = $disabled;
+ return $this;
+ }
+
+ public function getDisabled() {
+ return $this->disabled;
+ }
+
+ public static function getConnectionStatusMap() {
+ return array(
+ self::STATUS_OKAY => array(
+ 'icon' => 'fa-exchange',
+ 'color' => 'green',
+ 'label' => pht('Okay'),
+ ),
+ self::STATUS_FAIL => array(
+ 'icon' => 'fa-times',
+ 'color' => 'red',
+ 'label' => pht('Failed'),
+ ),
+ );
+ }
+
+ public function isWritable() {
+ return $this->hasRole('write');
+ }
+
+ public function isReadable() {
+ return $this->hasRole('read');
+ }
+
+ public function hasRole($role) {
+ return isset($this->roles[$role]) && $this->roles[$role] === true;
+ }
+
+ public function setRoles(array $roles) {
+ foreach ($roles as $role => $val) {
+ if ($val === false && isset($this->roles[$role])) {
+ unset($this->roles[$role]);
+ } else {
+ $this->roles[$role] = $val;
+ }
+ }
+ return $this;
+ }
+
+ public function getRoles() {
+ return $this->roles;
+ }
+
+ public function getPort() {
+ return idx($this->config, 'port');
+ }
+
+ public function getProtocol() {
+ return idx($this->config, 'protocol');
+ }
+
+
+ public function getVersion() {
+ return idx($this->config, 'version');
+ }
+
+ public function getHosts() {
+ return $this->hosts;
+ }
+
+
+ /**
+ * Get a random host reference with the specified role, skipping hosts which
+ * failed recent health checks.
+ * @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match.
+ * @return PhabricatorSearchHost
+ */
+ public function getAnyHostForRole($role) {
+ $hosts = $this->getAllHostsForRole($role);
+ shuffle($hosts);
+ foreach ($hosts as $host) {
+ $health = $host->getHealthRecord();
+ if ($health->getIsHealthy()) {
+ return $host;
+ }
+ }
+ throw new PhabricatorClusterNoHostForRoleException($role);
+ }
+
+
+ /**
+ * Get all configured hosts for this service which have the specified role.
+ * @return PhabricatorSearchHost[]
+ */
+ public function getAllHostsForRole($role) {
+ $hosts = array();
+ foreach ($this->hosts as $host) {
+ if ($host->hasRole($role)) {
+ $hosts[] = $host;
+ }
+ }
+ return $hosts;
+ }
+
+ /**
+ * Get a reference to all configured fulltext search cluster services
+ * @return PhabricatorSearchService[]
+ */
+ public static function getAllServices() {
+ $cache = PhabricatorCaches::getRequestCache();
+
+ $refs = $cache->getKey(self::KEY_REFS);
+ if (!$refs) {
+ $refs = self::newRefs();
+ $cache->setKey(self::KEY_REFS, $refs);
+ }
+
+ return $refs;
+ }
+
+ /**
+ * Load all valid PhabricatorFulltextStorageEngine subclasses
+ */
+ public static function loadAllFulltextStorageEngines() {
+ return id(new PhutilClassMapQuery())
+ ->setAncestorClass('PhabricatorFulltextStorageEngine')
+ ->setUniqueMethod('getEngineIdentifier')
+ ->execute();
+ }
+
+ /**
+ * Create instances of PhabricatorSearchService based on configuration
+ * @return PhabricatorSearchService[]
+ */
+ public static function newRefs() {
+ $services = PhabricatorEnv::getEnvConfig('cluster.search');
+ $engines = self::loadAllFulltextStorageEngines();
+ $refs = array();
+
+ foreach ($services as $config) {
+ $engine = $engines[$config['type']];
+ $cluster = new self($engine);
+ $cluster->setConfig($config);
+ $engine->setService($cluster);
+ $refs[] = $cluster;
+ }
+
+ return $refs;
+ }
+
+
+ /**
+ * (re)index the document: attempt to pass the document to all writable
+ * fulltext search hosts
+ * @throws PhabricatorClusterNoHostForRoleException
+ */
+ public static function reindexAbstractDocument(
+ PhabricatorSearchAbstractDocument $doc) {
+ $indexed = 0;
+ foreach (self::getAllServices() as $service) {
+ $service->getEngine()->reindexAbstractDocument($doc);
+ $indexed++;
+ }
+ if ($indexed == 0) {
+ throw new PhabricatorClusterNoHostForRoleException('write');
+ }
+ }
+
+ /**
+ * Execute a full-text query and return a list of PHIDs of matching objects.
+ * @return string[]
+ * @throws PhutilAggregateException
+ */
+ public static function executeSearch(PhabricatorSavedQuery $query) {
+ $services = self::getAllServices();
+ $exceptions = array();
+ foreach ($services as $service) {
+ $engine = $service->getEngine();
+ // try all hosts until one succeeds
+ try {
+ $res = $engine->executeSearch($query);
+ // return immediately if we get results without an exception
+ return $res;
+ } catch (Exception $ex) {
+ $exceptions[] = $ex;
+ }
+ }
+ throw new PhutilAggregateException('All search engines failed:',
+ $exceptions);
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Mon, Nov 11, 1:41 PM (5 d, 21 h ago)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/w2/wv/26lkkaakq5tmljez
Default Alt Text
D17384.diff (85 KB)

Event Timeline

Phabricator · Built by Phacility