diff --git a/resources/sql/autopatches/20161130.search.02.rebuild.php b/resources/sql/autopatches/20161130.search.02.rebuild.php --- a/resources/sql/autopatches/20161130.search.02.rebuild.php +++ b/resources/sql/autopatches/20161130.search.02.rebuild.php @@ -1,7 +1,15 @@ getEngine(); + if ($engine instanceof PhabricatorMySQLFulltextStorageEngine) { + $use_mysql = true; + } +} if ($use_mysql) { $field = new PhabricatorSearchDocumentField(); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2259,12 +2259,15 @@ 'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php', 'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php', 'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php', - 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php', - 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php', - 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php', - 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php', - 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php', - 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php', + 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php', + 'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php', + 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', + 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', + 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', + 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php', + 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', + 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', 'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php', @@ -2310,6 +2313,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php', 'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php', + 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -2543,7 +2547,6 @@ 'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php', 'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php', 'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php', - 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php', 'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php', 'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php', 'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php', @@ -2651,6 +2654,8 @@ 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', + 'PhabricatorElasticSearchHost' => 'infrastructure/cluster/search/PhabricatorElasticSearchHost.php', + 'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php', 'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php', 'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php', 'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php', @@ -3073,6 +3078,7 @@ 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php', + 'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php', 'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php', 'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php', 'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php', @@ -3762,7 +3768,6 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php', 'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php', 'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php', - 'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php', 'PhabricatorSearchConstraintException' => 'applications/search/exception/PhabricatorSearchConstraintException.php', 'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php', 'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php', @@ -3785,6 +3790,7 @@ 'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php', 'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php', 'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php', + 'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php', 'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php', 'PhabricatorSearchIndexVersion' => 'applications/search/storage/PhabricatorSearchIndexVersion.php', 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php', @@ -3804,6 +3810,7 @@ 'PhabricatorSearchSchemaSpec' => 'applications/search/storage/PhabricatorSearchSchemaSpec.php', 'PhabricatorSearchScopeSetting' => 'applications/settings/setting/PhabricatorSearchScopeSetting.php', 'PhabricatorSearchSelectField' => 'applications/search/field/PhabricatorSearchSelectField.php', + 'PhabricatorSearchService' => 'infrastructure/cluster/search/PhabricatorSearchService.php', 'PhabricatorSearchStringListField' => 'applications/search/field/PhabricatorSearchStringListField.php', 'PhabricatorSearchSubscribersField' => 'applications/search/field/PhabricatorSearchSubscribersField.php', 'PhabricatorSearchTextField' => 'applications/search/field/PhabricatorSearchTextField.php', @@ -7303,6 +7310,9 @@ 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterNoHostForRoleException' => 'Exception', + 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType', + 'PhabricatorClusterServiceHealthRecord' => 'Phobject', 'PhabricatorClusterStrandedException' => 'PhabricatorClusterException', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', @@ -7354,6 +7364,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', @@ -7624,7 +7635,6 @@ 'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController', 'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec', 'PhabricatorDataNotAttachedException' => 'Exception', - 'PhabricatorDatabaseHealthRecord' => 'Phobject', 'PhabricatorDatabaseRef' => 'Phobject', 'PhabricatorDatabaseRefParser' => 'Phobject', 'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck', @@ -7738,6 +7748,7 @@ 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorElasticSearchHost' => 'PhabricatorSearchHost', 'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailContentSource' => 'PhabricatorContentSource', @@ -8208,6 +8219,7 @@ 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost', 'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorNamedQuery' => array( 'PhabricatorSearchDAO', @@ -9074,7 +9086,6 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel', 'PhabricatorSearchBaseController' => 'PhabricatorController', 'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField', - 'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorSearchConstraintException' => 'Exception', 'PhabricatorSearchController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField', @@ -9097,6 +9108,7 @@ 'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule', 'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorSearchField' => 'Phobject', + 'PhabricatorSearchHost' => 'Phobject', 'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchIndexVersion' => 'PhabricatorSearchDAO', 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension', @@ -9116,6 +9128,7 @@ 'PhabricatorSearchSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorSearchScopeSetting' => 'PhabricatorInternalSetting', 'PhabricatorSearchSelectField' => 'PhabricatorSearchField', + 'PhabricatorSearchService' => 'Phobject', 'PhabricatorSearchStringListField' => 'PhabricatorSearchField', 'PhabricatorSearchSubscribersField' => 'PhabricatorSearchTokenizerField', 'PhabricatorSearchTextField' => 'PhabricatorSearchField', diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php --- a/src/applications/config/application/PhabricatorConfigApplication.php +++ b/src/applications/config/application/PhabricatorConfigApplication.php @@ -69,6 +69,7 @@ 'databases/' => 'PhabricatorConfigClusterDatabasesController', 'notifications/' => 'PhabricatorConfigClusterNotificationsController', 'repositories/' => 'PhabricatorConfigClusterRepositoriesController', + 'search/' => 'PhabricatorConfigClusterSearchController', ), ), ); diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php --- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -7,71 +7,74 @@ } protected function executeChecks() { - if (!$this->shouldUseElasticSearchEngine()) { - return; - } - - $engine = new PhabricatorElasticFulltextStorageEngine(); + $services = PhabricatorSearchService::getAllServices(); - $index_exists = null; - $index_sane = null; - try { - $index_exists = $engine->indexExists(); - if ($index_exists) { - $index_sane = $engine->indexIsSane(); + foreach ($services as $service) { + try { + $host = $service->getAnyHostForRole('read'); + } catch (PhabricatorClusterNoHostForRoleException $e) { + // ignore the error + continue; } - } catch (Exception $ex) { - $summary = pht('Elasticsearch is not reachable as configured.'); - $message = pht( - 'Elasticsearch is configured (with the %s setting) but Phabricator '. - 'encountered an exception when trying to test the index.'. - "\n\n". - '%s', - phutil_tag('tt', array(), 'search.elastic.host'), - phutil_tag('pre', array(), $ex->getMessage())); + if ($host instanceof PhabricatorElasticSearchHost) { + $index_exists = null; + $index_sane = null; + try { + $engine = $host->getEngine(); + $index_exists = $engine->indexExists(); + if ($index_exists) { + $index_sane = $engine->indexIsSane(); + } + } catch (Exception $ex) { + $summary = pht('Elasticsearch is not reachable as configured.'); + $message = pht( + 'Elasticsearch is configured (with the %s setting) but Phabricator'. + ' encountered an exception when trying to test the index.'. + "\n\n". + '%s', + phutil_tag('tt', array(), 'cluster.search'), + phutil_tag('pre', array(), $ex->getMessage())); - $this->newIssue('elastic.misconfigured') - ->setName(pht('Elasticsearch Misconfigured')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - return; - } + $this->newIssue('elastic.misconfigured') + ->setName(pht('Elasticsearch Misconfigured')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + return; + } - if (!$index_exists) { - $summary = pht( - 'You enabled Elasticsearch but the index does not exist.'); + if (!$index_exists) { + $summary = pht( + 'You enabled Elasticsearch but the index does not exist.'); - $message = pht( - 'You likely enabled search.elastic.host without creating the '. - 'index. Run `./bin/search init` to correct the index.'); + $message = pht( + 'You likely enabled cluster.search without creating the '. + 'index. Run `./bin/search init` to correct the index.'); - $this - ->newIssue('elastic.missing-index') - ->setName(pht('Elasticsearch index Not Found')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - } else if (!$index_sane) { - $summary = pht( - 'Elasticsearch index exists but needs correction.'); + $this + ->newIssue('elastic.missing-index') + ->setName(pht('Elasticsearch index Not Found')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + } else if (!$index_sane) { + $summary = pht( + 'Elasticsearch index exists but needs correction.'); - $message = pht( - 'Either the Phabricator schema for Elasticsearch has changed '. - 'or Elasticsearch created the index automatically. Run '. - '`./bin/search init` to correct the index.'); + $message = pht( + 'Either the Phabricator schema for Elasticsearch has changed '. + 'or Elasticsearch created the index automatically. Run '. + '`./bin/search init` to correct the index.'); - $this - ->newIssue('elastic.broken-index') - ->setName(pht('Elasticsearch index Incorrect')) - ->setSummary($summary) - ->setMessage($message); + $this + ->newIssue('elastic.broken-index') + ->setName(pht('Elasticsearch index Incorrect')) + ->setSummary($summary) + ->setMessage($message); + } + } } } - protected function shouldUseElasticSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine); - } } diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php --- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php +++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php @@ -198,6 +198,10 @@ 'This option has been removed, you can use Dashboards to provide '. 'homepage customization. See T11533 for more details.'); + $elastic_reason = pht( + 'Elasticsearch is now configured with "%s".', + 'cluster.search'); + $ancient_config += array( 'phid.external-loaders' => pht( @@ -348,6 +352,10 @@ 'mysql.configuration-provider' => pht( 'Phabricator now has application-level management of partitioning '. 'and replicas.'), + + 'search.elastic.host' => $elastic_reason, + 'search.elastic.namespace' => $elastic_reason, + ); return $ancient_config; diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php --- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php +++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php @@ -379,8 +379,13 @@ } protected function shouldUseMySQLSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine); + $services = PhabricatorSearchService::getAllServices(); + foreach ($services as $service) { + if ($service instanceof PhabricatorMySQLSearchHost) { + return true; + } + } + return false; } } diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php new file mode 100644 --- /dev/null +++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php @@ -0,0 +1,129 @@ +buildSideNavView(); + $nav->selectFilter('cluster/search/'); + + $title = pht('Cluster Search'); + $doc_href = PhabricatorEnv::getDoclink('Cluster: Search'); + + $header = id(new PHUIHeaderView()) + ->setHeader($title) + ->setProfileHeader(true) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb($title) + ->setBorder(true); + + $search_status = $this->buildClusterSearchStatus(); + + $content = id(new PhabricatorConfigPageView()) + ->setHeader($header) + ->setContent($search_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->setNavigation($nav) + ->appendChild($content) + ->addClass('white-background'); + } + + private function buildClusterSearchStatus() { + $viewer = $this->getViewer(); + + $services = PhabricatorSearchService::getAllServices(); + Javelin::initBehavior('phabricator-tooltips'); + + $view = array(); + foreach ($services as $service) { + $view[] = $this->renderStatusView($service); + } + return $view; + } + + private function renderStatusView($service) { + $head = array_merge( + array(pht('Type')), + array_keys($service->getStatusViewColumns()), + array(pht('Status'))); + + $rows = array(); + + $status_map = PhabricatorSearchService::getConnectionStatusMap(); + $stats = false; + $stats_view = false; + + foreach ($service->getHosts() as $host) { + try { + $status = $host->getConnectionStatus(); + $status = idx($status_map, $status, array()); + } catch (Exception $ex) { + $status['icon'] = 'fa-times'; + $status['label'] = pht('Connection Error'); + $status['color'] = 'red'; + $host->didHealthCheck(false); + } + + if (!$stats_view) { + try { + $stats = $host->getEngine()->getIndexStats($host); + $stats_view = $this->renderIndexStats($stats); + } catch (Exception $e) { + $stats_view = false; + } + } + + $type_icon = 'fa-search sky'; + $type_tip = $host->getDisplayName(); + + $type_icon = id(new PHUIIconView()) + ->setIcon($type_icon); + $status_view = array( + id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']), + ' ', + $status['label'], + ); + $row = array(array($type_icon, ' ', $type_tip)); + $row = array_merge($row, array_values( + $host->getStatusViewColumns())); + $row[] = $status_view; + $rows[] = $row; + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString(pht('No search servers are configured.')) + ->setHeaders($head); + + $view = id(new PHUIObjectBoxView()) + ->setHeaderText($service->getDisplayName()) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->setTable($table); + + if ($stats_view) { + $view->addPropertyList($stats_view); + } + return $view; + } + + private function renderIndexStats($stats) { + $view = id(new PHUIPropertyListView()); + if ($stats !== false) { + foreach ($stats as $label => $val) { + $view->addProperty($label, $val); + } + } + return $view; + } + +} diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php --- a/src/applications/config/controller/PhabricatorConfigController.php +++ b/src/applications/config/controller/PhabricatorConfigController.php @@ -42,8 +42,11 @@ pht('Notification Servers'), null, 'fa-bell-o'); $nav->addFilter('cluster/repositories/', pht('Repository Servers'), null, 'fa-code'); + $nav->addFilter('cluster/search/', + pht('Search Servers'), null, 'fa-search'); $nav->addLabel(pht('Modules')); + $modules = PhabricatorConfigModule::getAllModules(); foreach ($modules as $key => $module) { $nav->addFilter('module/'.$key.'/', diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php --- a/src/applications/config/option/PhabricatorClusterConfigOptions.php +++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php @@ -38,6 +38,17 @@ $intro_href = PhabricatorEnv::getDoclink('Clustering Introduction'); $intro_name = pht('Clustering Introduction'); + $search_type = 'custom:PhabricatorClusterSearchConfigOptionType'; + $search_help = $this->deformat(pht(<<newOption('cluster.addresses', 'list', array()) ->setLocked(true) @@ -114,6 +125,21 @@ ->setSummary( pht('Configure database read replicas.')) ->setDescription($databases_help), + $this->newOption('cluster.search', $search_type, array()) + ->setLocked(true) + ->setSummary( + pht('Configure full-text search services.')) + ->setDescription($search_help) + ->setDefault( + array( + array( + 'type' => 'mysql', + 'roles' => array( + 'read' => true, + 'write' => true, + ), + ), + )), ); } diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php --- a/src/applications/maniphest/query/ManiphestTaskQuery.php +++ b/src/applications/maniphest/query/ManiphestTaskQuery.php @@ -513,14 +513,14 @@ ->setEngineClassName('PhabricatorSearchApplicationSearchEngine') ->setParameter('query', $this->fullTextSearch); - // NOTE: Setting this to something larger than 2^53 will raise errors in + // NOTE: Setting this to something larger than 10,000 will raise errors in // ElasticSearch, and billions of results won't fit in memory anyway. - $fulltext_query->setParameter('limit', 100000); + $fulltext_query->setParameter('limit', 10000); $fulltext_query->setParameter('types', array(ManiphestTaskPHIDType::TYPECONST)); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $fulltext_results = $engine->executeSearch($fulltext_query); + $fulltext_results = PhabricatorSearchService::executeSearch( + $fulltext_query); if (empty($fulltext_results)) { $fulltext_results = array(null); diff --git a/src/applications/project/search/PhabricatorProjectFulltextEngine.php b/src/applications/project/search/PhabricatorProjectFulltextEngine.php --- a/src/applications/project/search/PhabricatorProjectFulltextEngine.php +++ b/src/applications/project/search/PhabricatorProjectFulltextEngine.php @@ -10,7 +10,15 @@ $project = $object; $project->updateDatasourceTokens(); - $document->setDocumentTitle($project->getName()); + $document->setDocumentTitle($project->getDisplayName()); + $document->addField(PhabricatorSearchDocumentFieldType::FIELD_KEYWORDS, + $project->getPrimarySlug()); + try { + $slugs = $project->getSlugs(); + foreach ($slugs as $slug) {} + } catch (PhabricatorDataNotAttachedException $e) { + // ignore + } $document->addRelationship( $project->isArchived() diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php deleted file mode 100644 --- a/src/applications/search/config/PhabricatorSearchConfigOptions.php +++ /dev/null @@ -1,35 +0,0 @@ -newOption('search.elastic.host', 'string', null) - ->setLocked(true) - ->setDescription(pht('Elastic Search host.')) - ->addExample('http://elastic.example.com:9200/', pht('Valid Setting')), - $this->newOption('search.elastic.namespace', 'string', 'phabricator') - ->setLocked(true) - ->setDescription(pht('Elastic Search index.')) - ->addExample('phabricator2', pht('Valid Setting')), - ); - } - -} diff --git a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php --- a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php +++ b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php @@ -5,5 +5,6 @@ const FIELD_TITLE = 'titl'; const FIELD_BODY = 'body'; const FIELD_COMMENT = 'cmnt'; + const FIELD_KEYWORDS = 'kwrd'; } diff --git a/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php b/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php --- a/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php +++ b/src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php @@ -3,8 +3,8 @@ final class PhabricatorSearchEngineTestCase extends PhabricatorTestCase { public function testLoadAllEngines() { - PhabricatorFulltextStorageEngine::loadAllEngines(); - $this->assertTrue(true); + $services = PhabricatorSearchService::getAllServices(); + $this->assertTrue(!empty($services)); } } diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -1,37 +1,52 @@ uri = PhabricatorEnv::getEnvConfig('search.elastic.host'); - $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace'); + private $version; + + public function setService(PhabricatorSearchService $service) { + $this->service = $service; + $config = $service->getConfig(); + $index = idx($config, 'path', '/phabricator'); + $this->index = str_replace('/', '', $index); + $this->timeout = idx($config, 'timeout', 15); + $this->version = (int)idx($config, 'version', 5); + return $this; } public function getEngineIdentifier() { return 'elasticsearch'; } - public function getEnginePriority() { - return 10; + public function getTimestampField() { + return $this->version < 2 ? + '_timestamp' : 'lastModified'; } - public function isEnabled() { - return (bool)$this->uri; + public function getTextFieldType() { + return $this->version >= 5 + ? 'text' : 'string'; } - public function setURI($uri) { - $this->uri = $uri; - return $this; + public function getHostType() { + return new PhabricatorElasticSearchHost($this); } - public function setIndex($index) { - $this->index = $index; - return $this; + /** + * @return PhabricatorElasticSearchHost + */ + public function getHostForRead() { + return $this->getService()->getAnyHostForRole('read'); + } + + /** + * @return PhabricatorElasticSearchHost + */ + public function getHostForWrite() { + return $this->getService()->getAnyHostForRole('write'); } public function setTimeout($timeout) { @@ -39,21 +54,21 @@ return $this; } - public function getURI() { - return $this->uri; - } - - public function getIndex() { - return $this->index; - } - public function getTimeout() { return $this->timeout; } + public function getTypeConstants($class) { + $relationship_class = new ReflectionClass($class); + $typeconstants = $relationship_class->getConstants(); + return array_unique(array_values($typeconstants)); + } + public function reindexAbstractDocument( PhabricatorSearchAbstractDocument $doc) { + $host = $this->getHostForWrite(); + $type = $doc->getDocumentType(); $phid = $doc->getPHID(); $handle = id(new PhabricatorHandleQuery()) @@ -61,36 +76,47 @@ ->withPHIDs(array($phid)) ->executeOne(); + $timestamp_key = $this->getTimestampField(); + // URL is not used internally but it can be useful externally. $spec = array( 'title' => $doc->getDocumentTitle(), 'url' => PhabricatorEnv::getProductionURI($handle->getURI()), 'dateCreated' => $doc->getDocumentCreated(), - '_timestamp' => $doc->getDocumentModified(), - 'field' => array(), - 'relationship' => array(), + $timestamp_key => $doc->getDocumentModified(), ); foreach ($doc->getFieldData() as $field) { - $spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field); + list($field_name, $corpus, $aux) = $field; + if (!isset($spec[$field_name])) { + $spec[$field_name] = array($corpus); + } else { + $spec[$field_name][] = $corpus; + } + if ($aux != null) { + $spec[$field_name][] = $aux; + } } - foreach ($doc->getRelationshipData() as $relationship) { - list($rtype, $to_phid, $to_type, $time) = $relationship; - $spec['relationship'][$rtype][] = array( - 'phid' => $to_phid, - 'phidType' => $to_type, - 'when' => (int)$time, - ); + foreach ($doc->getRelationshipData() as $field) { + list($field_name, $related_phid, $rtype, $time) = $field; + if (!isset($spec[$field_name])) { + $spec[$field_name] = array($related_phid); + } else { + $spec[$field_name][] = $related_phid; + } + if ($time) { + $spec[$field_name.'_ts'] = $time; + } } - $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT'); + $this->executeRequest($host, "/{$type}/{$phid}/", $spec, 'PUT'); } public function reconstructDocument($phid) { $type = phid_get_type($phid); - - $response = $this->executeRequest("/{$type}/{$phid}", array()); + $host = $this->getHostForRead(); + $response = $this->executeRequest($host, "/{$type}/{$phid}", array()); if (empty($response['exists'])) { return null; @@ -103,10 +129,11 @@ $doc->setDocumentType($response['_type']); $doc->setDocumentTitle($hit['title']); $doc->setDocumentCreated($hit['dateCreated']); - $doc->setDocumentModified($hit['_timestamp']); + $doc->setDocumentModified($hit[$this->getTimestampField()]); foreach ($hit['field'] as $fdef) { - $doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']); + $field_type = $fdef['type']; + $doc->addField($field_type, $hit[$field_type], $fdef['aux']); } foreach ($hit['relationship'] as $rtype => $rships) { @@ -123,35 +150,51 @@ } private function buildSpec(PhabricatorSavedQuery $query) { - $spec = array(); - $filter = array(); - $title_spec = array(); - - if (strlen($query->getParameter('query'))) { - $spec[] = array( + $q = new PhabricatorElasticSearchQueryBuilder('bool'); + $query_string = $query->getParameter('query'); + if (strlen($query_string)) { + $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); + + // Build a simple_query_string query over all fields that must match all + // of the words in the search string. + $q->addMustClause(array( 'simple_query_string' => array( - 'query' => $query->getParameter('query'), - 'fields' => array('field.corpus'), + 'query' => $query_string, + 'fields' => array( + '_all', + ), + 'default_operator' => 'OR', ), - ); + )); - $title_spec = array( + // This second query clause is "SHOULD' so it only affects ranking of + // documents which already matched the Must clause. This amplifies the + // score of documents which have an exact match on title, body + // or comments. + $q->addShouldClause(array( 'simple_query_string' => array( - 'query' => $query->getParameter('query'), - 'fields' => array('title'), + 'query' => $query_string, + 'fields' => array( + PhabricatorSearchDocumentFieldType::FIELD_TITLE.'^4', + PhabricatorSearchDocumentFieldType::FIELD_BODY.'^3', + PhabricatorSearchDocumentFieldType::FIELD_COMMENT.'^1.2', + ), + 'analyzer' => 'english_exact', + 'default_operator' => 'and', ), - ); + )); + } $exclude = $query->getParameter('exclude'); if ($exclude) { - $filter[] = array( + $q->addFilterClause(array( 'not' => array( 'ids' => array( 'values' => array($exclude), ), ), - ); + )); } $relationship_map = array( @@ -176,75 +219,59 @@ $include_closed = !empty($statuses[$rel_closed]); if ($include_open && !$include_closed) { - $relationship_map[$rel_open] = true; + $q->addExistsClause($rel_open); } else if (!$include_open && $include_closed) { - $relationship_map[$rel_closed] = true; + $q->addExistsClause($rel_closed); } if ($query->getParameter('withUnowned')) { - $relationship_map[$rel_unowned] = true; + $q->addExistsClause($rel_unowned); } $rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER; if ($query->getParameter('withAnyOwner')) { - $relationship_map[$rel_owner] = true; + $q->addExistsClause($rel_owner); } else { $owner_phids = $query->getParameter('ownerPHIDs', array()); - $relationship_map[$rel_owner] = $owner_phids; - } - - foreach ($relationship_map as $field => $param) { - if (is_array($param) && $param) { - $should = array(); - foreach ($param as $val) { - $should[] = array( - 'match' => array( - "relationship.{$field}.phid" => array( - 'query' => $val, - 'type' => 'phrase', - ), - ), - ); - } - // We couldn't solve it by minimum_number_should_match because it can - // match multiple owners without matching author. - $spec[] = array('bool' => array('should' => $should)); - } else if ($param) { - $filter[] = array( - 'exists' => array( - 'field' => "relationship.{$field}.phid", - ), - ); + if (count($owner_phids)) { + $q->addTermsClause($rel_owner, $owner_phids); } } - if ($spec) { - $spec = array('query' => array('bool' => array('must' => $spec))); - if ($title_spec) { - $spec['query']['bool']['should'] = $title_spec; + foreach ($relationship_map as $field => $phids) { + if (is_array($phids) && !empty($phids)) { + $q->addTermsClause($field, $phids); } } - if ($filter) { - $filter = array('filter' => array('and' => $filter)); - if (!$spec) { - $spec = array('query' => array('match_all' => new stdClass())); - } - $spec = array( - 'query' => array( - 'filtered' => $spec + $filter, - ), - ); + if (!$q->getClauseCount('must')) { + $q->addMustClause(array('match_all' => array('boost' => 1 ))); } + $spec = array( + '_source' => false, + 'query' => array( + 'bool' => $q->toArray(), + ), + ); + + if (!$query->getParameter('query')) { $spec['sort'] = array( array('dateCreated' => 'desc'), ); } - $spec['from'] = (int)$query->getParameter('offset', 0); - $spec['size'] = (int)$query->getParameter('limit', 25); + $offset = (int)$query->getParameter('offset', 0); + $limit = (int)$query->getParameter('limit', 101); + if ($offset + $limit > 10000) { + throw new Exception(pht( + 'Query offset is too large. offset+limit=%s (max=%s)', + $offset + $limit, + 10000)); + } + $spec['from'] = $offset; + $spec['size'] = $limit; return $spec; } @@ -261,30 +288,36 @@ // some bigger index). Use '/$types/_search' instead. $uri = '/'.implode(',', $types).'/_search'; - try { - $response = $this->executeRequest($uri, $this->buildSpec($query)); - } catch (HTTPFutureHTTPResponseStatus $ex) { - // elasticsearch probably uses Lucene query syntax: - // http://lucene.apache.org/core/3_6_1/queryparsersyntax.html - // Try literal search if operator search fails. - if (!strlen($query->getParameter('query'))) { - throw $ex; + $spec = $this->buildSpec($query); + $exceptions = array(); + + foreach ($this->service->getAllHostsForRole('read') as $host) { + try { + $response = $this->executeRequest($host, $uri, $spec); + $phids = ipull($response['hits']['hits'], '_id'); + return $phids; + } catch (Exception $e) { + $exceptions[] = $e; } - $query = clone $query; - $query->setParameter( - 'query', - addcslashes( - $query->getParameter('query'), '+-&|!(){}[]^"~*?:\\')); - $response = $this->executeRequest($uri, $this->buildSpec($query)); } - - $phids = ipull($response['hits']['hits'], '_id'); - return $phids; + throw new PhutilAggregateException('All search hosts failed:', $exceptions); } - public function indexExists() { + public function indexExists(PhabricatorElasticSearchHost $host = null) { + if (!$host) { + $host = $this->getHostForRead(); + } try { - return (bool)$this->executeRequest('/_status/', array()); + if ($this->version >= 5) { + $uri = '/_stats/'; + $res = $this->executeRequest($host, $uri, array()); + return isset($res['indices']['phabricator']); + } else if ($this->version >= 2) { + $uri = ''; + } else { + $uri = '/_status/'; + } + return (bool)$this->executeRequest($host, $uri, array()); } catch (HTTPFutureHTTPResponseStatus $e) { if ($e->getStatusCode() == 404) { return false; @@ -299,53 +332,85 @@ 'index' => array( 'auto_expand_replicas' => '0-2', 'analysis' => array( - 'filter' => array( - 'trigrams_filter' => array( - 'min_gram' => 3, - 'type' => 'ngram', - 'max_gram' => 3, - ), - ), 'analyzer' => array( - 'custom_trigrams' => array( - 'type' => 'custom', - 'filter' => array( - 'lowercase', - 'kstem', - 'trigrams_filter', - ), + 'english_exact' => array( 'tokenizer' => 'standard', + 'filter' => array('lowercase'), ), ), ), ), ); - $types = array_keys( + $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); + $relationships = $this->getTypeConstants('PhabricatorSearchRelationship'); + + $doc_types = array_keys( PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); - foreach ($types as $type) { - // Use the custom trigram analyzer for the corpus of text - $data['mappings'][$type]['properties']['field']['properties']['corpus'] = - array('type' => 'string', 'analyzer' => 'custom_trigrams'); + + $text_type = $this->getTextFieldType(); + + foreach ($doc_types as $type) { + $properties = array(); + foreach ($fields as $field) { + // Use the custom analyzer for the corpus of text + $properties[$field] = array( + 'type' => $text_type, + 'analyzer' => 'english_exact', + 'search_analyzer' => 'english', + 'search_quote_analyzer' => 'english_exact', + ); + } + + if ($this->version < 5) { + foreach ($relationships as $rel) { + $properties[$rel] = array( + 'type' => 'string', + 'index' => 'not_analyzed', + 'include_in_all' => false, + ); + $properties[$rel.'_ts'] = array( + 'type' => 'date', + 'include_in_all' => false, + ); + } + } else { + foreach ($relationships as $rel) { + $properties[$rel] = array( + 'type' => 'keyword', + 'include_in_all' => false, + 'doc_values' => false, + ); + $properties[$rel.'_ts'] = array( + 'type' => 'date', + 'include_in_all' => false, + ); + } + } // Ensure we have dateCreated since the default query requires it - $data['mappings'][$type]['properties']['dateCreated']['type'] = 'string'; - } + $properties['dateCreated']['type'] = 'date'; + $properties['lastModified']['type'] = 'date'; + $data['mappings'][$type]['properties'] = $properties; + } return $data; } - public function indexIsSane() { - if (!$this->indexExists()) { + public function indexIsSane(PhabricatorElasticSearchHost $host = null) { + if (!$host) { + $host = $this->getHostForRead(); + } + if (!$this->indexExists($host)) { return false; } - - $cur_mapping = $this->executeRequest('/_mapping/', array()); - $cur_settings = $this->executeRequest('/_settings/', array()); + $cur_mapping = $this->executeRequest($host, '/_mapping/', array()); + $cur_settings = $this->executeRequest($host, '/_settings/', array()); $actual = array_merge($cur_settings[$this->index], $cur_mapping[$this->index]); - return $this->check($actual, $this->getIndexConfiguration()); + $res = $this->check($actual, $this->getIndexConfiguration()); + return $res; } /** @@ -355,7 +420,7 @@ * @param $required array * @return bool */ - private function check($actual, $required) { + private function check($actual, $required, $path = '') { foreach ($required as $key => $value) { if (!array_key_exists($key, $actual)) { if ($key === '_all') { @@ -369,7 +434,7 @@ if (!is_array($actual[$key])) { return false; } - if (!$this->check($actual[$key], $value)) { + if (!$this->check($actual[$key], $value, $path.'.'.$key)) { return false; } continue; @@ -403,19 +468,44 @@ } public function initIndex() { + $host = $this->getHostForWrite(); if ($this->indexExists()) { - $this->executeRequest('/', array(), 'DELETE'); + $this->executeRequest($host, '/', array(), 'DELETE'); } $data = $this->getIndexConfiguration(); - $this->executeRequest('/', $data, 'PUT'); + $this->executeRequest($host, '/', $data, 'PUT'); } - private function executeRequest($path, array $data, $method = 'GET') { - $uri = new PhutilURI($this->uri); - $uri->setPath($this->index); - $uri->appendPath($path); - $data = json_encode($data); + public function getIndexStats(PhabricatorElasticSearchHost $host = null) { + if ($this->version < 2) { + return false; + } + if (!$host) { + $host = $this->getHostForRead(); + } + $uri = '/_stats/'; + $host = $this->getHostForRead(); + + $res = $this->executeRequest($host, $uri, array()); + $stats = $res['indices'][$this->index]; + return array( + pht('Queries') => + idxv($stats, array('primaries', 'search', 'query_total')), + pht('Documents') => + idxv($stats, array('total', 'docs', 'count')), + pht('Deleted') => + idxv($stats, array('total', 'docs', 'deleted')), + pht('Storage Used') => + phutil_format_bytes(idxv($stats, + array('total', 'store', 'size_in_bytes'))), + ); + } + + private function executeRequest(PhabricatorElasticSearchHost $host, $path, + array $data, $method = 'GET') { + $uri = $host->getURI($path); + $data = json_encode($data); $future = new HTTPSFuture($uri, $data); if ($method != 'GET') { $future->setMethod($method); @@ -423,19 +513,30 @@ if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } - list($body) = $future->resolvex(); + try { + list($body) = $future->resolvex(); + } catch (HTTPFutureResponseStatus $ex) { + if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) { + $host->didHealthCheck(false); + } + throw $ex; + } if ($method != 'GET') { return null; } try { - return phutil_json_decode($body); + $data = phutil_json_decode($body); + $host->didHealthCheck(true); + return $data; } catch (PhutilJSONParserException $ex) { + $host->didHealthCheck(false); throw new PhutilProxyException( pht('ElasticSearch server returned invalid JSON!'), $ex); } + } } diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php new file mode 100644 --- /dev/null +++ b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php @@ -0,0 +1,78 @@ +clauses; + if ($termkey == null) { + return $clauses; + } + if (isset($clauses[$termkey])) { + return $clauses[$termkey]; + } + return array(); + } + + public function getClauseCount($clausekey) { + if (isset($this->clauses[$clausekey])) { + return count($this->clauses[$clausekey]); + } else { + return 0; + } + } + + public function addExistsClause($field) { + return $this->addClause('filter', array( + 'exists' => array( + 'field' => $field, + ), + )); + } + + public function addTermsClause($field, $values) { + return $this->addClause('filter', array( + 'terms' => array( + $field => array_values($values), + ), + )); + } + + public function addMustClause($clause) { + return $this->addClause('must', $clause); + } + + public function addFilterClause($clause) { + return $this->addClause('filter', $clause); + } + + public function addShouldClause($clause) { + return $this->addClause('should', $clause); + } + + public function addMustNotClause($clause) { + return $this->addClause('must_not', $clause); + } + + public function addClause($clause, $terms) { + $this->clauses[$clause][] = $terms; + return $this; + } + + public function toArray() { + $clauses = $this->getClauses(); + return $clauses; + $cleaned = array(); + foreach ($clauses as $clause => $subclauses) { + if (is_array($subclauses) && count($subclauses) == 1) { + $cleaned[$clause] = array_shift($subclauses); + } else { + $cleaned[$clause] = $subclauses; + } + } + return $cleaned; + } + +} diff --git a/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php @@ -7,46 +7,40 @@ */ abstract class PhabricatorFulltextStorageEngine extends Phobject { -/* -( Engine Metadata )---------------------------------------------------- */ + protected $service; + + public function getHosts() { + return $this->service->getHosts(); + } + + public function setService(PhabricatorSearchService $service) { + $this->service = $service; + return $this; + } /** - * Return a unique, nonempty string which identifies this storage engine. - * - * @return string Unique string for this engine, max length 32. - * @task meta + * @return PhabricatorSearchService */ - abstract public function getEngineIdentifier(); + public function getService() { + return $this->service; + } /** - * Prioritize this engine relative to other engines. - * - * Engines with a smaller priority number get an opportunity to write files - * first. Generally, lower-latency filestores should have lower priority - * numbers, and higher-latency filestores should have higher priority - * numbers. Setting priority to approximately the number of milliseconds of - * read latency will generally produce reasonable results. - * - * In conjunction with filesize limits, the goal is to store small files like - * profile images, thumbnails, and text snippets in lower-latency engines, - * and store large files in higher-capacity engines. - * - * @return float Engine priority. - * @task meta + * Implementations must return a prototype host instance which is cloned + * by the PhabricatorSearchService infrastructure to configure each engine. + * @return PhabricatorSearchHost */ - abstract public function getEnginePriority(); + abstract public function getHostType(); + +/* -( Engine Metadata )---------------------------------------------------- */ /** - * Return `true` if the engine is currently writable. - * - * Engines that are disabled or missing configuration should return `false` - * to prevent new writes. If writes were made with this engine in the past, - * the application may still try to perform reads. + * Return a unique, nonempty string which identifies this storage engine. * - * @return bool True if this engine can support new writes. + * @return string Unique string for this engine, max length 32. * @task meta */ - abstract public function isEnabled(); - + abstract public function getEngineIdentifier(); /* -( Managing Documents )------------------------------------------------- */ @@ -84,6 +78,13 @@ abstract public function indexExists(); /** + * Implementations should override this method to return a dictionary of + * stats which are suitable for display in the admin UI. + */ + abstract public function getIndexStats(); + + + /** * Is the index in a usable state? * * @return bool @@ -100,39 +101,4 @@ public function initIndex() {} -/* -( Loading Storage Engines )-------------------------------------------- */ - - /** - * @task load - */ - public static function loadAllEngines() { - return id(new PhutilClassMapQuery()) - ->setAncestorClass(__CLASS__) - ->setUniqueMethod('getEngineIdentifier') - ->setSortMethod('getEnginePriority') - ->execute(); - } - - /** - * @task load - */ - public static function loadActiveEngines() { - $engines = self::loadAllEngines(); - - $active = array(); - foreach ($engines as $key => $engine) { - if (!$engine->isEnabled()) { - continue; - } - - $active[$key] = $engine; - } - - return $active; - } - - public static function loadEngine() { - return head(self::loadActiveEngines()); - } - } diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php @@ -7,12 +7,8 @@ return 'mysql'; } - public function getEnginePriority() { - return 100; - } - - public function isEnabled() { - return true; + public function getHostType() { + return new PhabricatorMySQLSearchHost($this); } public function reindexAbstractDocument( @@ -415,4 +411,9 @@ public function indexExists() { return true; } + + public function getIndexStats() { + return false; + } + } diff --git a/src/applications/search/index/PhabricatorFulltextEngine.php b/src/applications/search/index/PhabricatorFulltextEngine.php --- a/src/applications/search/index/PhabricatorFulltextEngine.php +++ b/src/applications/search/index/PhabricatorFulltextEngine.php @@ -40,8 +40,7 @@ $extension->indexFulltextObject($object, $document); } - $storage_engine = PhabricatorFulltextStorageEngine::loadEngine(); - $storage_engine->reindexAbstractDocument($document); + PhabricatorSearchService::reindexAbstractDocument($document); } protected function newAbstractDocument($object) { diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php --- a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php +++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php @@ -13,27 +13,41 @@ public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $work_done = false; - if (!$engine->indexExists()) { - $console->writeOut( - '%s', - pht('Index does not exist, creating...')); - $engine->initIndex(); - $console->writeOut( - "%s\n", - pht('done.')); - $work_done = true; - } else if (!$engine->indexIsSane()) { - $console->writeOut( - '%s', - pht('Index exists but is incorrect, fixing...')); - $engine->initIndex(); + foreach (PhabricatorSearchService::getAllServices() as $service) { $console->writeOut( "%s\n", - pht('done.')); - $work_done = true; + pht('Initializing search service "%s"', $service->getDisplayName())); + + try { + $host = $service->getAnyHostForRole('write'); + } catch (PhabricatorClusterNoHostForRoleException $e) { + // If there are no writable hosts for a given cluster, skip it + $console->writeOut("%s\n", $e->getMessage()); + continue; + } + + $engine = $host->getEngine(); + + if (!$engine->indexExists()) { + $console->writeOut( + '%s', + pht('Index does not exist, creating...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } else if (!$engine->indexIsSane()) { + $console->writeOut( + '%s', + pht('Index exists but is incorrect, fixing...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } } if ($work_done) { diff --git a/src/applications/search/query/PhabricatorSearchDocumentQuery.php b/src/applications/search/query/PhabricatorSearchDocumentQuery.php --- a/src/applications/search/query/PhabricatorSearchDocumentQuery.php +++ b/src/applications/search/query/PhabricatorSearchDocumentQuery.php @@ -73,10 +73,7 @@ $query = id(clone($this->savedQuery)) ->setParameter('offset', $this->getOffset()) ->setParameter('limit', $this->getRawResultLimit()); - - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - - return $engine->executeSearch($query); + return PhabricatorSearchService::executeSearch($query); } public function getQueryApplicationClass() { diff --git a/src/docs/user/cluster/cluster_search.diviner b/src/docs/user/cluster/cluster_search.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/cluster/cluster_search.diviner @@ -0,0 +1,76 @@ +@title Cluster: Search +@group cluster + +Overview +======== + +You can configure phabricator to connect to one or more fulltext search clusters +running either Elasticsearch or MySQL. By default and without further +configuration, Phabricator will use MySQL for fulltext search. This will be +adequate for the vast majority of users. Installs with a very large number of +objects or specialized search needs can consider enabling Elasticsearch for +better scalability and potentially better search results. + +Configuring Search Services +=========================== + +To configure an Elasticsearch service, use the `cluster.search` configuration +option. A typical Elasticsearch configuration will probably look similar to +the following example: + +```lang=json +{ + "cluster.search": [ + { + "type": "elasticsearch", + "hosts": [ + { + "host": "127.0.0.1", + "roles": { "write": true, "read": true } + } + ], + "port": 9200, + "protocol": "http", + "path": "/phabricator", + "version": 5 + }, + ], +} +``` + +Supported Options +----------------- +| Key | Type |Comments| +|`type` | String |Engine type. Currently, 'elasticsearch' or 'mysql'| +|`protocol`| String |Either 'http' or 'https'| +|`port`| Int |The TCP port that Elasticsearch is bound to| +|`path`| String |The path portion of the url for phabricator's index.| +|`version`| Int |The version of Elasticsearch server. Supports either 2 or 5.| +|`hosts`| List |A list of one or more Elasticsearch host names / addresses.| + +Host Configuration +------------------ +Each search service must have one or more hosts associated with it. Each host +entry consists of a `host` key, a dictionary of roles and can optionally +override any of the options that are valid at the service level (see above). + +Currently supported roles are `read` and `write`. These can be individually +enabled or disabled on a per-host basis. A typical setup might include two +elasticsearch clusters in two separate datacenters. You can configure one +cluster for reads and both for writes. When one cluster is down for maintenance +you can simply swap the read role over to the backup cluster and then proceed +with maintenance without any service interruption. + +Monitoring Search Services +========================== + +You can monitor fulltext search in {nav Config > Search Servers}. This interface +shows you a quick overview of services and their health. + +The table on this page shows some basic stats for each configured service, +followed by the configuration and current status of each host. + +NOTE: This page runs its diagnostics //from the web server that is serving the +request//. If you are recovering from a disaster, the view this page shows +may be partial or misleading, and two requests served by different servers may +see different views of the cluster. diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php rename from src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php rename to src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php --- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php +++ b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php @@ -1,20 +1,19 @@ ref = $ref; + public function __construct($cache_key) { + $this->cacheKey = $cache_key; $this->readState(); } - /** * Is the database currently healthy? */ @@ -153,18 +152,13 @@ } } - private function getHealthRecordCacheKey() { - $ref = $this->ref; - - $host = $ref->getHost(); - $port = $ref->getPort(); - - return "cluster.db.health({$host}, {$port})"; + public function getCacheKey() { + return $this->cacheKey; } private function readHealthRecord() { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $health_record = $cache->getKey($cache_key); if (!is_array($health_record)) { @@ -180,7 +174,7 @@ private function writeHealthRecord(array $record) { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $cache->setKey($cache_key, $record); } diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -14,6 +14,7 @@ const REPLICATION_SLOW = 'replica-slow'; const REPLICATION_NOT_REPLICATING = 'not-replicating'; + const KEY_HEALTH = 'cluster.db.health'; const KEY_REFS = 'cluster.db.refs'; const KEY_INDIVIDUAL = 'cluster.db.individual'; @@ -489,9 +490,18 @@ return $this; } + private function getHealthRecordCacheKey() { + $host = $this->getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + public function getHealthRecord() { if (!$this->healthRecord) { - $this->healthRecord = new PhabricatorDatabaseHealthRecord($this); + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); } return $this->healthRecord; } diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php rename from src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php rename to src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php diff --git a/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php @@ -0,0 +1,79 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Search cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a search service, but '. + 'the value with index "%s" is not a dictionary.', + $index)); + } + + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'type' => 'string', + 'hosts' => 'optional list>', + 'roles' => 'optional map', + 'port' => 'optional int', + 'protocol' => 'optional string', + 'path' => 'optional string', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search engine configuration has an invalid service '. + 'specification (at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + if (!array_key_exists($spec['type'], $engines)) { + throw new Exception( + pht('Invalid search engine type: %s. Valid types include: %s', + $spec['type'], + implode(', ', array_keys($engines)))); + } + + if (isset($spec['hosts'])) { + foreach ($spec['hosts'] as $hostindex => $host) { + try { + PhutilTypeSpec::checkMap( + $host, + array( + 'host' => 'string', + 'roles' => 'optional map', + 'port' => 'optional int', + 'protocol' => 'optional string', + 'path' => 'optional string', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search cluster configuration has an invalid host '. + 'specification (at index "%s"): %s.', + $hostindex, + $ex->getMessage())); + } + } + } + } + } +} diff --git a/src/infrastructure/cluster/PhabricatorClusterException.php b/src/infrastructure/cluster/exception/PhabricatorClusterException.php rename from src/infrastructure/cluster/PhabricatorClusterException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterException.php diff --git a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php b/src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php rename from src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php rename to src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php diff --git a/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php rename from src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php diff --git a/src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php rename from src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php diff --git a/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php @@ -0,0 +1,10 @@ +setRoles(idx($config, 'roles', $this->getRoles())) + ->setHost(idx($config, 'host', $this->host)) + ->setPort(idx($config, 'port', $this->port)) + ->setProtocol(idx($config, 'protocol', $this->protocol)) + ->setPath(idx($config, 'path', $this->path)) + ->setVersion(idx($config, 'version', $this->version)); + return $this; + } + + public function getDisplayName() { + return pht('ElasticSearch'); + } + + public function getStatusViewColumns() { + return array( + pht('Protocol') => $this->getProtocol(), + pht('Host') => $this->getHost(), + pht('Port') => $this->getPort(), + pht('Index Path') => $this->getPath(), + pht('Elastic Version') => $this->getVersion(), + pht('Roles') => implode(', ', array_keys($this->getRoles())), + ); + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setPath($path) { + $this->path = $path; + return $this; + } + + public function getPath() { + return $this->path; + } + + public function setVersion($version) { + $this->version = $version; + return $this; + } + + public function getVersion() { + return $this->version; + } + + public function getURI($to_path = null) { + $uri = id(new PhutilURI('http://'.$this->getHost())) + ->setProtocol($this->getProtocol()) + ->setPort($this->getPort()) + ->setPath($this->getPath()); + + if ($to_path) { + $uri->appendPath($to_path); + } + return $uri; + } + + public function getConnectionStatus() { + $status = $this->getEngine()->indexIsSane($this); + return $status ? parent::STATUS_OKAY : parent::STATUS_FAIL; + } + +} diff --git a/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php @@ -0,0 +1,34 @@ +setRoles(idx($config, 'roles', + array('read' => true, 'write' => true))); + return $this; + } + + public function getDisplayName() { + return 'MySQL'; + } + + public function getStatusViewColumns() { + return array( + pht('Protocol') => 'mysql', + pht('Roles') => implode(', ', array_keys($this->getRoles())), + ); + } + + public function getProtocol() { + return 'mysql'; + } + + public function getConnectionStatus() { + PhabricatorDatabaseRef::queryAll(); + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); + $status = $ref->getConnectionStatus(); + return $status; + } + +} diff --git a/src/infrastructure/cluster/search/PhabricatorSearchHost.php b/src/infrastructure/cluster/search/PhabricatorSearchHost.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/search/PhabricatorSearchHost.php @@ -0,0 +1,163 @@ +engine = $engine; + } + + public function setDisabled($disabled) { + $this->disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + /** + * @return PhabricatorFulltextStorageEngine + */ + public function getEngine() { + return $this->engine; + } + + public function isWritable() { + return $this->hasRole('write'); + } + + public function isReadable() { + return $this->hasRole('read'); + } + + public function hasRole($role) { + return isset($this->roles[$role]) && $this->roles[$role] === true; + } + + public function setRoles(array $roles) { + foreach ($roles as $role => $val) { + $this->roles[$role] = $val; + } + return $this; + } + + public function getRoles() { + $roles = array(); + foreach ($this->roles as $key => $val) { + if ($val) { + $roles[$key] = $val; + } + } + return $roles; + } + + public function setPort($value) { + $this->port = $value; + return $this; + } + + public function getPort() { + return $this->port; + } + + public function setHost($value) { + $this->host = $value; + return $this; + } + + public function getHost() { + return $this->host; + } + + + public function getHealthRecordCacheKey() { + $host = $this->getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + +/** + * @return PhabricatorClusterServiceHealthRecord + */ + public function getHealthRecord() { + if (!$this->healthRecord) { + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); + } + return $this->healthRecord; + } + + public function didHealthCheck($reachable) { + $record = $this->getHealthRecord(); + $should_check = $record->getShouldCheck(); + + if ($should_check) { + $record->didHealthCheck($reachable); + } + } + + /** + * @return string[] Get a list of fields to show in the status overview UI + */ + abstract public function getStatusViewColumns(); + + abstract public function getConnectionStatus(); + + public static function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + + $services = self::getAllServices(); + $indexed = 0; + foreach (self::getWritableHostForEachService() as $host) { + $host->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } + if ($indexed == 0) { + throw new PhabricatorClusterNoHostForRoleException('write'); + } + } + + public static function executeSearch(PhabricatorSavedQuery $query) { + $services = self::getAllServices(); + foreach ($services as $service) { + $hosts = $service->getAllHostsForRole('read'); + // try all hosts until one succeeds + foreach ($hosts as $host) { + $last_exception = null; + try { + $res = $host->getEngine()->executeSearch($query); + // return immediately if we get results without an exception + $host->didHealthCheck(true); + return $res; + } catch (Exception $ex) { + // try each server in turn, only throw if none succeed + $last_exception = $ex; + $host->didHealthCheck(false); + } + } + } + if ($last_exception) { + throw $last_exception; + } + return $res; + } + +} diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php @@ -0,0 +1,259 @@ +engine = $engine; + $this->hostType = $engine->getHostType(); + } + + /** + * @throws Exception + */ + public function newHost($config) { + $host = clone($this->hostType); + $host_config = $this->config + $config; + $host->setConfig($host_config); + $this->hosts[] = $host; + return $host; + } + + public function getEngine() { + return $this->engine; + } + + public function getDisplayName() { + return $this->hostType->getDisplayName(); + } + + public function getStatusViewColumns() { + return $this->hostType->getStatusViewColumns(); + } + + public function setConfig($config) { + $this->config = $config; + + if (!isset($config['hosts'])) { + $config['hosts'] = array( + array( + 'host' => idx($config, 'host'), + 'port' => idx($config, 'port'), + 'protocol' => idx($config, 'protocol'), + 'roles' => idx($config, 'roles'), + ), + ); + } + foreach ($config['hosts'] as $host) { + $this->newHost($host); + } + + } + + public function getConfig() { + return $this->config; + } + + public function setDisabled($disabled) { + $this->disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public static function getConnectionStatusMap() { + return array( + self::STATUS_OKAY => array( + 'icon' => 'fa-exchange', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::STATUS_FAIL => array( + 'icon' => 'fa-times', + 'color' => 'red', + 'label' => pht('Failed'), + ), + ); + } + + public function isWritable() { + return $this->hasRole('write'); + } + + public function isReadable() { + return $this->hasRole('read'); + } + + public function hasRole($role) { + return isset($this->roles[$role]) && $this->roles[$role] === true; + } + + public function setRoles(array $roles) { + foreach ($roles as $role => $val) { + if ($val === false && isset($this->roles[$role])) { + unset($this->roles[$role]); + } else { + $this->roles[$role] = $val; + } + } + return $this; + } + + public function getRoles() { + return $this->roles; + } + + public function getPort() { + return idx($this->config, 'port'); + } + + public function getProtocol() { + return idx($this->config, 'protocol'); + } + + + public function getVersion() { + return idx($this->config, 'version'); + } + + public function getHosts() { + return $this->hosts; + } + + + /** + * Get a random host reference with the specified role, skipping hosts which + * failed recent health checks. + * @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match. + * @return PhabricatorSearchHost + */ + public function getAnyHostForRole($role) { + $hosts = $this->getAllHostsForRole($role); + shuffle($hosts); + foreach ($hosts as $host) { + $health = $host->getHealthRecord(); + if ($health->getIsHealthy()) { + return $host; + } + } + throw new PhabricatorClusterNoHostForRoleException($role); + } + + + /** + * Get all configured hosts for this service which have the specified role. + * @return PhabricatorSearchHost[] + */ + public function getAllHostsForRole($role) { + $hosts = array(); + foreach ($this->hosts as $host) { + if ($host->hasRole($role)) { + $hosts[] = $host; + } + } + return $hosts; + } + + /** + * Get a reference to all configured fulltext search cluster services + * @return PhabricatorSearchService[] + */ + public static function getAllServices() { + $cache = PhabricatorCaches::getRequestCache(); + + $refs = $cache->getKey(self::KEY_REFS); + if (!$refs) { + $refs = self::newRefs(); + $cache->setKey(self::KEY_REFS, $refs); + } + + return $refs; + } + + /** + * Load all valid PhabricatorFulltextStorageEngine subclasses + */ + public static function loadAllFulltextStorageEngines() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorFulltextStorageEngine') + ->setUniqueMethod('getEngineIdentifier') + ->execute(); + } + + /** + * Create instances of PhabricatorSearchService based on configuration + * @return PhabricatorSearchService[] + */ + public static function newRefs() { + $services = PhabricatorEnv::getEnvConfig('cluster.search'); + $engines = self::loadAllFulltextStorageEngines(); + $refs = array(); + + foreach ($services as $config) { + $engine = $engines[$config['type']]; + $cluster = new self($engine); + $cluster->setConfig($config); + $engine->setService($cluster); + $refs[] = $cluster; + } + + return $refs; + } + + + /** + * (re)index the document: attempt to pass the document to all writable + * fulltext search hosts + * @throws PhabricatorClusterNoHostForRoleException + */ + public static function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + $indexed = 0; + foreach (self::getAllServices() as $service) { + $service->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } + if ($indexed == 0) { + throw new PhabricatorClusterNoHostForRoleException('write'); + } + } + + /** + * Execute a full-text query and return a list of PHIDs of matching objects. + * @return string[] + * @throws PhutilAggregateException + */ + public static function executeSearch(PhabricatorSavedQuery $query) { + $services = self::getAllServices(); + $exceptions = array(); + foreach ($services as $service) { + $engine = $service->getEngine(); + // try all hosts until one succeeds + try { + $res = $engine->executeSearch($query); + // return immediately if we get results without an exception + return $res; + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + throw new PhutilAggregateException('All search engines failed:', + $exceptions); + } + +}