Index: resources/sql/autopatches/20161130.search.02.rebuild.php =================================================================== --- resources/sql/autopatches/20161130.search.02.rebuild.php +++ resources/sql/autopatches/20161130.search.02.rebuild.php @@ -1,7 +1,14 @@ 'applications/chatlog/query/PhabricatorChatLogQuery.php', 'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php', 'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php', - 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php', - 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php', - 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php', - 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php', - 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php', - 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php', + 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php', + 'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php', + 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', + 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', + 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', + 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php', + 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', + 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', 'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php', @@ -2312,6 +2315,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php', 'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php', + 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -2544,7 +2548,6 @@ 'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php', 'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php', 'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php', - 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php', 'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php', 'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php', 'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php', @@ -2651,6 +2654,7 @@ 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', + 'PhabricatorElasticSearchHost' => 'infrastructure/cluster/search/PhabricatorElasticSearchHost.php', 'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php', 'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php', 'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php', @@ -3074,6 +3078,7 @@ 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php', + 'PhabricatorMySQLSearchHost' => 'infrastructure/cluster/search/PhabricatorMySQLSearchHost.php', 'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php', 'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php', 'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php', @@ -3763,7 +3768,7 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php', 'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php', 'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php', - 'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php', + 'PhabricatorSearchCluster' => 'infrastructure/cluster/search/PhabricatorSearchCluster.php', 'PhabricatorSearchConstraintException' => 'applications/search/exception/PhabricatorSearchConstraintException.php', 'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php', 'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php', @@ -3786,6 +3791,7 @@ 'PhabricatorSearchEngineExtensionModule' => 'applications/search/engineextension/PhabricatorSearchEngineExtensionModule.php', 'PhabricatorSearchEngineTestCase' => 'applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php', 'PhabricatorSearchField' => 'applications/search/field/PhabricatorSearchField.php', + 'PhabricatorSearchHost' => 'infrastructure/cluster/search/PhabricatorSearchHost.php', 'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php', 'PhabricatorSearchIndexVersion' => 'applications/search/storage/PhabricatorSearchIndexVersion.php', 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php', @@ -7305,6 +7311,9 @@ 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterNoHostForRoleException' => 'Exception', + 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType', + 'PhabricatorClusterServiceHealthRecord' => 'Phobject', 'PhabricatorClusterStrandedException' => 'PhabricatorClusterException', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', @@ -7356,6 +7365,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', @@ -7625,7 +7635,6 @@ 'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController', 'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec', 'PhabricatorDataNotAttachedException' => 'Exception', - 'PhabricatorDatabaseHealthRecord' => 'Phobject', 'PhabricatorDatabaseRef' => 'Phobject', 'PhabricatorDatabaseRefParser' => 'Phobject', 'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck', @@ -7738,6 +7747,7 @@ 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorElasticSearchHost' => 'PhabricatorSearchHost', 'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailContentSource' => 'PhabricatorContentSource', @@ -8208,6 +8218,7 @@ 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorMySQLSearchHost' => 'PhabricatorSearchHost', 'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorNamedQuery' => array( 'PhabricatorSearchDAO', @@ -9074,7 +9085,7 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel', 'PhabricatorSearchBaseController' => 'PhabricatorController', 'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField', - 'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions', + 'PhabricatorSearchCluster' => 'Phobject', 'PhabricatorSearchConstraintException' => 'Exception', 'PhabricatorSearchController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField', @@ -9097,6 +9108,7 @@ 'PhabricatorSearchEngineExtensionModule' => 'PhabricatorConfigModule', 'PhabricatorSearchEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorSearchField' => 'Phobject', + 'PhabricatorSearchHost' => 'Phobject', 'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchIndexVersion' => 'PhabricatorSearchDAO', 'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension', Index: src/applications/config/application/PhabricatorConfigApplication.php =================================================================== --- src/applications/config/application/PhabricatorConfigApplication.php +++ src/applications/config/application/PhabricatorConfigApplication.php @@ -69,6 +69,7 @@ 'databases/' => 'PhabricatorConfigClusterDatabasesController', 'notifications/' => 'PhabricatorConfigClusterNotificationsController', 'repositories/' => 'PhabricatorConfigClusterRepositoriesController', + 'search/' => 'PhabricatorConfigClusterSearchController', ), ), ); Index: src/applications/config/check/PhabricatorElasticSearchSetupCheck.php =================================================================== --- src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -7,71 +7,74 @@ } protected function executeChecks() { - if (!$this->shouldUseElasticSearchEngine()) { - return; - } - - $engine = new PhabricatorElasticFulltextStorageEngine(); + $services = PhabricatorSearchCluster::getAllServices(); - $index_exists = null; - $index_sane = null; - try { - $index_exists = $engine->indexExists(); - if ($index_exists) { - $index_sane = $engine->indexIsSane(); + foreach ($services as $service) { + try { + $host = $service->getAnyHostForRole('read'); + } catch (PhabricatorClusterNoHostForRoleException $e) { + // ignore the error + continue; } - } catch (Exception $ex) { - $summary = pht('Elasticsearch is not reachable as configured.'); - $message = pht( - 'Elasticsearch is configured (with the %s setting) but Phabricator '. - 'encountered an exception when trying to test the index.'. - "\n\n". - '%s', - phutil_tag('tt', array(), 'search.elastic.host'), - phutil_tag('pre', array(), $ex->getMessage())); + if ($host instanceof PhabricatorElasticSearchHost) { + $index_exists = null; + $index_sane = null; + try { + $engine = $host->getEngine(); + $index_exists = $engine->indexExists(); + if ($index_exists) { + $index_sane = $engine->indexIsSane(); + } + } catch (Exception $ex) { + $summary = pht('Elasticsearch is not reachable as configured.'); + $message = pht( + 'Elasticsearch is configured (with the %s setting) but Phabricator'. + ' encountered an exception when trying to test the index.'. + "\n\n". + '%s', + phutil_tag('tt', array(), 'cluster.search'), + phutil_tag('pre', array(), $ex->getMessage())); - $this->newIssue('elastic.misconfigured') - ->setName(pht('Elasticsearch Misconfigured')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - return; - } + $this->newIssue('elastic.misconfigured') + ->setName(pht('Elasticsearch Misconfigured')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + return; + } - if (!$index_exists) { - $summary = pht( - 'You enabled Elasticsearch but the index does not exist.'); + if (!$index_exists) { + $summary = pht( + 'You enabled Elasticsearch but the index does not exist.'); - $message = pht( - 'You likely enabled search.elastic.host without creating the '. - 'index. Run `./bin/search init` to correct the index.'); + $message = pht( + 'You likely enabled cluster.search without creating the '. + 'index. Run `./bin/search init` to correct the index.'); - $this - ->newIssue('elastic.missing-index') - ->setName(pht('Elasticsearch index Not Found')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - } else if (!$index_sane) { - $summary = pht( - 'Elasticsearch index exists but needs correction.'); + $this + ->newIssue('elastic.missing-index') + ->setName(pht('Elasticsearch index Not Found')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + } else if (!$index_sane) { + $summary = pht( + 'Elasticsearch index exists but needs correction.'); - $message = pht( - 'Either the Phabricator schema for Elasticsearch has changed '. - 'or Elasticsearch created the index automatically. Run '. - '`./bin/search init` to correct the index.'); + $message = pht( + 'Either the Phabricator schema for Elasticsearch has changed '. + 'or Elasticsearch created the index automatically. Run '. + '`./bin/search init` to correct the index.'); - $this - ->newIssue('elastic.broken-index') - ->setName(pht('Elasticsearch index Incorrect')) - ->setSummary($summary) - ->setMessage($message); + $this + ->newIssue('elastic.broken-index') + ->setName(pht('Elasticsearch index Incorrect')) + ->setSummary($summary) + ->setMessage($message); + } + } } } - protected function shouldUseElasticSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine); - } } Index: src/applications/config/check/PhabricatorExtraConfigSetupCheck.php =================================================================== --- src/applications/config/check/PhabricatorExtraConfigSetupCheck.php +++ src/applications/config/check/PhabricatorExtraConfigSetupCheck.php @@ -198,6 +198,10 @@ 'This option has been removed, you can use Dashboards to provide '. 'homepage customization. See T11533 for more details.'); + $elastic_reason = pht( + 'Elasticsearch is now configured with "%s"', + 'cluster.search'); + $ancient_config += array( 'phid.external-loaders' => pht( @@ -348,6 +352,11 @@ 'mysql.configuration-provider' => pht( 'Phabricator now has application-level management of partitioning '. 'and replicas.'), + + 'search.elastic.enabled' => $elastic_reason, + 'search.elastic.host' => $elastic_reason, + 'search.elastic.namespace' => $elastic_reason, + 'search.elastic.version' => $elastic_reason, ); return $ancient_config; Index: src/applications/config/check/PhabricatorMySQLSetupCheck.php =================================================================== --- src/applications/config/check/PhabricatorMySQLSetupCheck.php +++ src/applications/config/check/PhabricatorMySQLSetupCheck.php @@ -379,8 +379,13 @@ } protected function shouldUseMySQLSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine); + $services = PhabricatorSearchCluster::getAllServices(); + foreach ($services as $service) { + if ($service instanceof PhabricatorMySQLSearchHost) { + return true; + } + } + return false; } } Index: src/applications/config/controller/PhabricatorConfigClusterSearchController.php =================================================================== --- /dev/null +++ src/applications/config/controller/PhabricatorConfigClusterSearchController.php @@ -0,0 +1,146 @@ +buildSideNavView(); + $nav->selectFilter('cluster/search/'); + + $title = pht('Cluster Search'); + $doc_href = PhabricatorEnv::getDoclink('Cluster: Search'); + + $header = id(new PHUIHeaderView()) + ->setHeader($title) + ->setProfileHeader(true) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb($title) + ->setBorder(true); + + $search_status = $this->buildClusterSearchStatus(); + + $content = id(new PhabricatorConfigPageView()) + ->setHeader($header) + ->setContent($search_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->setNavigation($nav) + ->appendChild($content) + ->addClass('white-background'); + } + + private function buildClusterSearchStatus() { + $viewer = $this->getViewer(); + + $services = PhabricatorSearchCluster::getAllServices(); + Javelin::initBehavior('phabricator-tooltips'); + + $view = array(); + foreach ($services as $service) { + $view[] = $this->renderStatusView($service); + } + return $view; + } + + private function renderStatusView($service) { + $rows = array(); + + $head = array_merge( + array(pht('Type')), + array_keys($service->getStatusViewColumns()), + array(pht('Status'))); + + $status_map = PhabricatorSearchCluster::getConnectionStatusMap(); + foreach ($service->getHosts() as $host) { + $reachable = false; + try { + $engine = $host->getEngine(); + $reachable = $engine->indexExists(); + } catch (Exception $ex) { + $reachable = false; + } + $host->didHealthCheck($reachable); + try { + $status = $host->getConnectionStatus(); + $status = idx($status_map, $status, array()); + $stats = $engine->getIndexStats(); + } catch (Exception $ex) { + $status['icon'] = 'fa-times'; + $status['label'] = pht('Connection Error'); + $status['color'] = 'red'; + $stats = array(); + } + $stats_view = $this->renderIndexStats($stats); + $type_icon = 'fa-search sky'; + $type_tip = $host->getDisplayName(); + + $type_icon = id(new PHUIIconView()) + ->setIcon($type_icon); + $status_view = array( + id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']), + ' ', + $status['label'], + ); + $row = array(array($type_icon, ' ', $type_tip)); + $row = array_merge($row, array_values( + $host->getStatusViewColumns())); + $row[] = $status_view; + $rows[] = $row; + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString(pht('No search servers are configured.')) + ->setHeaders($head); + + return id(new PHUIObjectBoxView()) + ->setHeaderText($service->getDisplayName()) + ->addPropertyList($stats_view) + ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) + ->setTable($table); + } + + private function renderIndexStats($stats) { + $view = id(new PHUIPropertyListView()); + if ($stats !== false) { + foreach ($stats as $label => $val) { + $view->addProperty($label, $val); + } + } + return $view; + } + + private function renderYes($info) { + return array( + id(new PHUIIconView())->setIcon('fa-check', 'green'), + ' ', + $info, + ); + } + + private function renderNo($info) { + return array( + id(new PHUIIconView())->setIcon('fa-times-circle', 'red'), + ' ', + $info, + ); + } + + private function renderInfo($info) { + return array( + id(new PHUIIconView())->setIcon('fa-info-circle', 'grey'), + ' ', + $info, + ); + } + +} Index: src/applications/config/controller/PhabricatorConfigController.php =================================================================== --- src/applications/config/controller/PhabricatorConfigController.php +++ src/applications/config/controller/PhabricatorConfigController.php @@ -42,8 +42,11 @@ pht('Notification Servers'), null, 'fa-bell-o'); $nav->addFilter('cluster/repositories/', pht('Repository Servers'), null, 'fa-code'); + $nav->addFilter('cluster/search/', + pht('Search Servers'), null, 'fa-search'); $nav->addLabel(pht('Modules')); + $modules = PhabricatorConfigModule::getAllModules(); foreach ($modules as $key => $module) { $nav->addFilter('module/'.$key.'/', Index: src/applications/config/option/PhabricatorClusterConfigOptions.php =================================================================== --- src/applications/config/option/PhabricatorClusterConfigOptions.php +++ src/applications/config/option/PhabricatorClusterConfigOptions.php @@ -38,6 +38,17 @@ $intro_href = PhabricatorEnv::getDoclink('Clustering Introduction'); $intro_name = pht('Clustering Introduction'); + $search_type = 'custom:PhabricatorClusterSearchConfigOptionType'; + $search_help = $this->deformat(pht(<<newOption('cluster.addresses', 'list', array()) ->setLocked(true) @@ -114,6 +125,21 @@ ->setSummary( pht('Configure database read replicas.')) ->setDescription($databases_help), + $this->newOption('cluster.search', $search_type, array()) + ->setLocked(true) + ->setSummary( + pht('Configure full-text search services.')) + ->setDescription($search_help) + ->setDefault( + array( + array( + 'type' => 'mysql', + 'roles' => array( + 'read' => true, + 'write' => true, + ), + ), + )), ); } Index: src/applications/maniphest/query/ManiphestTaskQuery.php =================================================================== --- src/applications/maniphest/query/ManiphestTaskQuery.php +++ src/applications/maniphest/query/ManiphestTaskQuery.php @@ -513,14 +513,14 @@ ->setEngineClassName('PhabricatorSearchApplicationSearchEngine') ->setParameter('query', $this->fullTextSearch); - // NOTE: Setting this to something larger than 2^53 will raise errors in + // NOTE: Setting this to something larger than 10,000 will raise errors in // ElasticSearch, and billions of results won't fit in memory anyway. - $fulltext_query->setParameter('limit', 100000); + $fulltext_query->setParameter('limit', 10000); $fulltext_query->setParameter('types', array(ManiphestTaskPHIDType::TYPECONST)); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $fulltext_results = $engine->executeSearch($fulltext_query); + $fulltext_results = PhabricatorSearchCluster::executeSearch( + $fulltext_query); if (empty($fulltext_results)) { $fulltext_results = array(null); Index: src/applications/search/config/PhabricatorSearchConfigOptions.php =================================================================== --- src/applications/search/config/PhabricatorSearchConfigOptions.php +++ /dev/null @@ -1,35 +0,0 @@ -newOption('search.elastic.host', 'string', null) - ->setLocked(true) - ->setDescription(pht('Elastic Search host.')) - ->addExample('http://elastic.example.com:9200/', pht('Valid Setting')), - $this->newOption('search.elastic.namespace', 'string', 'phabricator') - ->setLocked(true) - ->setDescription(pht('Elastic Search index.')) - ->addExample('phabricator2', pht('Valid Setting')), - ); - } - -} Index: src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php =================================================================== --- src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php +++ src/applications/search/engine/__tests__/PhabricatorSearchEngineTestCase.php @@ -3,8 +3,8 @@ final class PhabricatorSearchEngineTestCase extends PhabricatorTestCase { public function testLoadAllEngines() { - PhabricatorFulltextStorageEngine::loadAllEngines(); - $this->assertTrue(true); + $services = PhabricatorSearchCluster::getAllServices(); + $this->assertTrue(!empty($services)); } } Index: src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php =================================================================== --- src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -3,7 +3,7 @@ final class PhabricatorElasticFulltextStorageEngine extends PhabricatorFulltextStorageEngine { - private $uri; + private $ref; private $index; private $timeout; /** elasticsearch version */ @@ -19,31 +19,13 @@ return 'elasticsearch'; } - public function getEnginePriority() { - return 10; - } - - public function isEnabled() { - return (bool)$this->uri; - } - - public function setURI($uri) { - $this->uri = $uri; - return $this; - } - - public function setIndex($index) { - $this->index = $index; - return $this; - } - public function setTimeout($timeout) { $this->timeout = $timeout; return $this; } - public function getURI() { - return $this->uri; + public function getURI($path = '') { + return $this->ref->getURI($path); } public function getIndex() { @@ -488,12 +470,40 @@ $this->executeRequest('/', $data, 'PUT'); } + public function didHealthCheck($reachable) { + static $cache=null; + if ($cache !== null) { + return; + } + + $cache = $reachable; + $this->ref->didHealthCheck($reachable); + return $this; + } + + public function getIndexStats() { + if ($this->version < 2) { + return false; + } + $uri = '/_stats/'; + $res = $this->executeRequest($uri, array()); + $stats = $res['indices'][$this->index]; + return array( + pht('Queries') => + idxv($stats, array('primaries', 'search', 'query_total')), + pht('Documents') => + idxv($stats, array('total', 'docs', 'count')), + pht('Deleted') => + idxv($stats, array('total', 'docs', 'deleted')), + pht('Storage Used') => + phutil_format_bytes(idxv($stats, + array('total', 'store', 'size_in_bytes'))), + ); + } + private function executeRequest($path, array $data, $method = 'GET') { - $uri = new PhutilURI($this->uri); - $uri->setPath($this->index); - $uri->appendPath($path); + $uri = $this->ref->getURI($path); $data = json_encode($data); - $future = new HTTPSFuture($uri, $data); if ($method != 'GET') { $future->setMethod($method); @@ -501,19 +511,30 @@ if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } - list($body) = $future->resolvex(); + try { + list($body) = $future->resolvex(); + } catch (HTTPFutureResponseStatus $ex) { + if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) { + $this->didHealthCheck(false); + } + throw $ex; + } if ($method != 'GET') { return null; } try { - return phutil_json_decode($body); + $data = phutil_json_decode($body); + $this->didHealthCheck(true); + return $data; } catch (PhutilJSONParserException $ex) { + $this->didHealthCheck(false); throw new PhutilProxyException( pht('ElasticSearch server returned invalid JSON!'), $ex); } + } } Index: src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php =================================================================== --- src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php +++ src/applications/search/fulltextstorage/PhabricatorFulltextStorageEngine.php @@ -17,37 +17,6 @@ */ abstract public function getEngineIdentifier(); - /** - * Prioritize this engine relative to other engines. - * - * Engines with a smaller priority number get an opportunity to write files - * first. Generally, lower-latency filestores should have lower priority - * numbers, and higher-latency filestores should have higher priority - * numbers. Setting priority to approximately the number of milliseconds of - * read latency will generally produce reasonable results. - * - * In conjunction with filesize limits, the goal is to store small files like - * profile images, thumbnails, and text snippets in lower-latency engines, - * and store large files in higher-capacity engines. - * - * @return float Engine priority. - * @task meta - */ - abstract public function getEnginePriority(); - - /** - * Return `true` if the engine is currently writable. - * - * Engines that are disabled or missing configuration should return `false` - * to prevent new writes. If writes were made with this engine in the past, - * the application may still try to perform reads. - * - * @return bool True if this engine can support new writes. - * @task meta - */ - abstract public function isEnabled(); - - /* -( Managing Documents )------------------------------------------------- */ /** @@ -84,6 +53,13 @@ abstract public function indexExists(); /** + * Implementations should override this method to return a dictionary of + * stats which are suitable for display in the admin UI. + */ + abstract public function getIndexStats(); + + + /** * Is the index in a usable state? * * @return bool @@ -100,39 +76,4 @@ public function initIndex() {} -/* -( Loading Storage Engines )-------------------------------------------- */ - - /** - * @task load - */ - public static function loadAllEngines() { - return id(new PhutilClassMapQuery()) - ->setAncestorClass(__CLASS__) - ->setUniqueMethod('getEngineIdentifier') - ->setSortMethod('getEnginePriority') - ->execute(); - } - - /** - * @task load - */ - public static function loadActiveEngines() { - $engines = self::loadAllEngines(); - - $active = array(); - foreach ($engines as $key => $engine) { - if (!$engine->isEnabled()) { - continue; - } - - $active[$key] = $engine; - } - - return $active; - } - - public static function loadEngine() { - return head(self::loadActiveEngines()); - } - } Index: src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php =================================================================== --- src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php +++ src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php @@ -7,14 +7,6 @@ return 'mysql'; } - public function getEnginePriority() { - return 100; - } - - public function isEnabled() { - return true; - } - public function reindexAbstractDocument( PhabricatorSearchAbstractDocument $doc) { @@ -415,4 +407,9 @@ public function indexExists() { return true; } + + public function getIndexStats() { + return false; + } + } Index: src/applications/search/index/PhabricatorFulltextEngine.php =================================================================== --- src/applications/search/index/PhabricatorFulltextEngine.php +++ src/applications/search/index/PhabricatorFulltextEngine.php @@ -40,8 +40,7 @@ $extension->indexFulltextObject($object, $document); } - $storage_engine = PhabricatorFulltextStorageEngine::loadEngine(); - $storage_engine->reindexAbstractDocument($document); + PhabricatorSearchCluster::reindexAbstractDocument($document); } protected function newAbstractDocument($object) { Index: src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php =================================================================== --- src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php +++ src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php @@ -13,27 +13,41 @@ public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $work_done = false; - if (!$engine->indexExists()) { - $console->writeOut( - '%s', - pht('Index does not exist, creating...')); - $engine->initIndex(); - $console->writeOut( - "%s\n", - pht('done.')); - $work_done = true; - } else if (!$engine->indexIsSane()) { - $console->writeOut( - '%s', - pht('Index exists but is incorrect, fixing...')); - $engine->initIndex(); + foreach (PhabricatorSearchCluster::getAllServices() as $service) { $console->writeOut( "%s\n", - pht('done.')); - $work_done = true; + pht('Initializing search service "%s"', $service->getDisplayName())); + + try { + $host = $service->getAnyHostForRole('write'); + } catch (PhabricatorClusterNoHostForRoleException $e) { + // If there are no writable hosts for a given cluster, skip it + $console->writeOut("%s\n", $e->getExceptionTitle()); + continue; + } + + $engine = $host->getEngine(); + + if (!$engine->indexExists()) { + $console->writeOut( + '%s', + pht('Index does not exist, creating...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } else if (!$engine->indexIsSane()) { + $console->writeOut( + '%s', + pht('Index exists but is incorrect, fixing...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } } if ($work_done) { Index: src/applications/search/query/PhabricatorSearchDocumentQuery.php =================================================================== --- src/applications/search/query/PhabricatorSearchDocumentQuery.php +++ src/applications/search/query/PhabricatorSearchDocumentQuery.php @@ -73,10 +73,7 @@ $query = id(clone($this->savedQuery)) ->setParameter('offset', $this->getOffset()) ->setParameter('limit', $this->getRawResultLimit()); - - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - - return $engine->executeSearch($query); + return PhabricatorSearchCluster::executeSearch($query); } public function getQueryApplicationClass() { Index: src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php @@ -1,93 +0,0 @@ - $spec) { - if (!is_array($spec)) { - throw new Exception( - pht( - 'Database cluster configuration is not valid: each entry in the '. - 'list must be a dictionary describing a database host, but '. - 'the value with index "%s" is not a dictionary.', - $index)); - } - } - - $masters = array(); - $map = array(); - foreach ($value as $index => $spec) { - try { - PhutilTypeSpec::checkMap( - $spec, - array( - 'host' => 'string', - 'role' => 'string', - 'port' => 'optional int', - 'user' => 'optional string', - 'pass' => 'optional string', - 'disabled' => 'optional bool', - 'master' => 'optional string', - 'partition' => 'optional list', - 'persistent' => 'optional bool', - )); - } catch (Exception $ex) { - throw new Exception( - pht( - 'Database cluster configuration has an invalid host '. - 'specification (at index "%s"): %s.', - $index, - $ex->getMessage())); - } - - $role = $spec['role']; - $host = $spec['host']; - $port = idx($spec, 'port'); - - switch ($role) { - case 'master': - case 'replica': - break; - default: - throw new Exception( - pht( - 'Database cluster configuration describes an invalid '. - 'host ("%s", at index "%s") with an unrecognized role ("%s"). '. - 'Valid roles are "%s" or "%s".', - $spec['host'], - $index, - $spec['role'], - 'master', - 'replica')); - } - - if ($role === 'master') { - $masters[] = $host; - } - - // We can't guarantee that you didn't just give the same host two - // different names in DNS, but this check can catch silly copy/paste - // mistakes. - $key = "{$host}:{$port}"; - if (isset($map[$key])) { - throw new Exception( - pht( - 'Database cluster configuration is invalid: it describes the '. - 'same host ("%s") multiple times. Each host should appear only '. - 'once in the list.', - $host)); - } - $map[$key] = true; - } - - } - -} Index: src/infrastructure/cluster/PhabricatorClusterException.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/PhabricatorClusterException.php @@ -1,8 +0,0 @@ -getViewer($request); - - $title = $ex->getExceptionTitle(); - - $dialog = id(new AphrontDialogView()) - ->setTitle($title) - ->setUser($viewer) - ->appendParagraph($ex->getMessage()) - ->addCancelButton('/', pht('Proceed With Caution')); - - return id(new AphrontDialogResponse()) - ->setDialog($dialog) - ->setHTTPResponseCode(500); - } - -} Index: src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php @@ -1,10 +0,0 @@ -ref = $ref; + public function __construct($cache_key) { + $this->cacheKey = $cache_key; $this->readState(); } - /** * Is the database currently healthy? */ @@ -153,18 +152,13 @@ } } - private function getHealthRecordCacheKey() { - $ref = $this->ref; - - $host = $ref->getHost(); - $port = $ref->getPort(); - - return "cluster.db.health({$host}, {$port})"; + public function getCacheKey() { + return $this->cacheKey; } private function readHealthRecord() { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $health_record = $cache->getKey($cache_key); if (!is_array($health_record)) { @@ -180,7 +174,7 @@ private function writeHealthRecord(array $record) { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $cache->setKey($cache_key, $record); } Index: src/infrastructure/cluster/PhabricatorClusterStrandedException.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/PhabricatorClusterStrandedException.php @@ -1,10 +0,0 @@ -getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + public function getHealthRecord() { if (!$this->healthRecord) { - $this->healthRecord = new PhabricatorDatabaseHealthRecord($this); + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); } return $this->healthRecord; } Index: src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php @@ -0,0 +1,79 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Search cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a search service, but '. + 'the value with index "%s" is not a dictionary.', + $index)); + } + + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'type' => 'string', + 'hosts' => 'optional list>', + 'roles' => 'optional map', + 'port' => 'optional int', + 'protocol' => 'optional string', + 'path' => 'optional string', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search cluster configuration has an invalid service '. + 'specification (at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + if (!array_key_exists($spec['type'], $types)) { + throw new Exception( + pht('Invalid search cluster type: %s. Valid types include: %s', + $spec['type'], + implode(', ', array_keys($types)))); + } + + if (isset($spec['hosts'])) { + foreach ($spec['hosts'] as $hostindex => $host) { + try { + PhutilTypeSpec::checkMap( + $host, + array( + 'host' => 'string', + 'roles' => 'optional map', + 'port' => 'optional int', + 'protocol' => 'optional string', + 'path' => 'optional string', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search cluster configuration has an invalid host '. + 'specification (at index "%s"): %s.', + $hostindex, + $ex->getMessage())); + } + } + } + } + } +} Index: src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php @@ -0,0 +1,10 @@ +setRoles(idx($config, 'roles', $this->getRoles())) + ->setHost(idx($config, 'host', $this->host)) + ->setPort(idx($config, 'port', $this->port)) + ->setProtocol(idx($config, 'protocol', $this->protocol)) + ->setPath(idx($config, 'path', $this->path)) + ->setVersion(idx($config, 'version', $this->version)); + return $this; + } + + public function getDisplayName() { + return pht('ElasticSearch'); + } + + public function getEngineIdentifier() { + return 'elasticsearch'; + } + + public function getStatusViewColumns() { + return array( + pht('Protocol') => $this->getEngineIdentifier(), + pht('Host') => $this->getHost(), + pht('Port') => $this->getPort(), + pht('Index Path') => $this->getPath(), + pht('Elastic Version') => $this->getVersion(), + pht('Roles') => implode(', ', array_keys($this->getRoles())), + ); + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setPath($path) { + $this->path = $path; + return $this; + } + + public function getPath() { + return $this->path; + } + + public function setVersion($version) { + $this->version = $version; + return $this; + } + + public function getVersion() { + return $this->version; + } + + public function getURI($to_path = null) { + $uri = id(new PhutilURI('http://'.$this->getHost())) + ->setProtocol($this->getProtocol()) + ->setPort($this->getPort()) + ->setPath($this->getPath()); + + if ($to_path) { + $uri->appendPath($to_path); + } + return $uri; + } + + /** + * @return PhabricatorElasticFulltextStorageEngine + */ + public function getEngine() { + if (!$this->engine) { + $engine = new PhabricatorElasticFulltextStorageEngine(); + $this->engine = $engine->setRef($this); + } + return $this->engine; + } + + public function getConnectionStatus() { + $status = $this->getEngine()->indexIsSane() + ? parent::STATUS_OKAY + : parent::STATUS_FAIL; + return $status; + } + +} Index: src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/search/PhabricatorMySQLSearchHost.php @@ -0,0 +1,48 @@ +engine = new PhabricatorMySQLFulltextStorageEngine(); + } + + public function setConfig($config) { + $this->setRoles(idx($config, 'roles', + array('read' => true, 'write' => true))); + return $this; + } + + public function getDisplayName() { + return 'MySQL'; + } + + public function getEngineIdentifier() { + return 'mysql'; + } + + public function getStatusViewColumns() { + return array( + pht('Protocol') => $this->getEngineIdentifier(), + pht('Roles') => implode(', ', array_keys($this->getRoles())), + ); + } + + public function getEngine() { + return $this->engine; + } + + public function getProtocol() { + return 'mysql'; + } + + public function getConnectionStatus() { + PhabricatorDatabaseRef::queryAll(); + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); + $status = $ref->getConnectionStatus(); + return $status; + } + +} Index: src/infrastructure/cluster/search/PhabricatorSearchCluster.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/search/PhabricatorSearchCluster.php @@ -0,0 +1,257 @@ +hostType = $host_type; + } + + /** + * @throws Exception + */ + public function newHost($config) { + $host = clone($this->hostType); + $host_config = $this->config + $config; + $host->setConfig($host_config); + $this->hosts[] = $host; + return $host; + } + + public function getDisplayName() { + return $this->hostType->getDisplayName(); + } + + public function getStatusViewColumns() { + return $this->hostType->getStatusViewColumns(); + } + + public function setConfig($config) { + $this->config = $config; + + if (!isset($config['hosts'])) { + $config['hosts'] = array( + array( + 'host' => idx($config, 'host'), + 'port' => idx($config, 'port'), + 'protocol' => idx($config, 'protocol'), + 'roles' => idx($config, 'roles'), + ), + ); + } + foreach ($config['hosts'] as $host) { + $this->newHost($host); + } + + } + + public function setDisabled($disabled) { + $this->disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public static function getConnectionStatusMap() { + return array( + self::STATUS_OKAY => array( + 'icon' => 'fa-exchange', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::STATUS_FAIL => array( + 'icon' => 'fa-times', + 'color' => 'red', + 'label' => pht('Failed'), + ), + ); + } + + public function isWritable() { + return $this->hasRole('write'); + } + + public function isReadable() { + return $this->hasRole('read'); + } + + public function hasRole($role) { + return isset($this->roles[$role]) && $this->roles[$role] === true; + } + + public function setRoles(array $roles) { + foreach ($roles as $role => $val) { + if ($val === false && isset($this->roles[$role])) { + unset($this->roles[$role]); + } else { + $this->roles[$role] = $val; + } + } + return $this; + } + + public function getRoles() { + return $this->roles; + } + + public function getPort() { + return idx($this->config, 'port'); + } + + public function getProtocol() { + return idx($this->config, 'protocol'); + } + + + public function getVersion() { + return idx($this->config, 'version'); + } + + public function getHosts() { + return $this->hosts; + } + + + /** @return PhabricatorSearchHost */ + public function getAnyHostForRole($role) { + $hosts = $this->getAllHostsForRole($role); + if (empty($hosts)) { + throw new PhabricatorClusterNoHostForRoleException($role); + } + $random = array_rand($hosts); + return $hosts[$random]; + } + + + /** @return PhabricatorSearchHost[] */ + public function getAllHostsForRole($role) { + $hosts = array(); + foreach ($this->hosts as $host) { + if ($host->hasRole($role)) { + $hosts[] = $host; + } + } + return $hosts; + } + + /** + * Get a reference to all configured fulltext search cluster services + * @return PhabricatorSearchCluster[] + */ + public static function getAllServices() { + $cache = PhabricatorCaches::getRequestCache(); + + $refs = $cache->getKey(self::KEY_REFS); + if (!$refs) { + $refs = self::newRefs(); + $cache->setKey(self::KEY_REFS, $refs); + } + + return $refs; + } + + /** find one random writable host from each service. + * @return PhabricatorSearchCluster[] writable cluster hosts + */ + public static function getAllWritableHosts() { + $services = self::getAllServices(); + $all_writable = array(); + foreach ($services as $service) { + $all_writable += $service->getAllHostsForRole('write'); + } + return $all_writable; + } + + + public static function getValidHostTypes() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorSearchHost') + ->setUniqueMethod('getEngineIdentifier') + ->execute(); + } + + /** + * Create instances of PhabricatorSearchCluster based on configuration + * @return PhabricatorSearchCluster[] + */ + public static function newRefs() { + $services = PhabricatorEnv::getEnvConfig('cluster.search'); + $types = self::getValidHostTypes(); + $refs = array(); + + foreach ($services as $config) { + if (!isset($types[$config['type']])) { + // this really should not happen as the value is validated by + // PhabricatorClusterSearchConfigOptionType + continue; + } + $type = $types[$config['type']]; + $cluster = new self($type); + $cluster->setConfig($config); + + $refs[] = $cluster; + } + + return $refs; + } + + + /** + * (re)index the document: attempt to pass the document to all writable + * fulltext search hosts + */ + public static function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + + $indexed = 0; + foreach (self::getAllWritableHosts() as $host) { + $host->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } + if ($indexed == 0) { + throw new PhabricatorClusterNoHostForRoleException('write'); + } + } + + /** + * Execute a full-text query and return a list of PHIDs of matching objects. + * @return string[] + */ + public static function executeSearch(PhabricatorSavedQuery $query) { + $services = self::getAllServices(); + foreach ($services as $service) { + $hosts = $service->getAllHostsForRole('read'); + // try all hosts until one succeeds + foreach ($hosts as $host) { + $last_exception = null; + try { + $res = $host->getEngine()->executeSearch($query); + // return immediately if we get results without an exception + return $res; + } catch (Exception $ex) { + // try each server in turn, only throw if none succeed + $last_exception = $ex; + } + } + } + if ($last_exception) { + throw $last_exception; + } + return $res; + } + +} Index: src/infrastructure/cluster/search/PhabricatorSearchHost.php =================================================================== --- /dev/null +++ src/infrastructure/cluster/search/PhabricatorSearchHost.php @@ -0,0 +1,149 @@ +disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public function isWritable() { + return $this->hasRole('write'); + } + + public function isReadable() { + return $this->hasRole('read'); + } + + public function hasRole($role) { + return isset($this->roles[$role]) && $this->roles[$role] === true; + } + + public function setRoles(array $roles) { + foreach ($roles as $role => $val) { + $this->roles[$role] = $val; + } + return $this; + } + + public function getRoles() { + return $this->roles; + } + + public function setPort($value) { + $this->port = $value; + return $this; + } + + public function getPort() { + return $this->port; + } + + public function setHost($value) { + $this->host = $value; + return $this; + } + + public function getHost() { + return $this->host; + } + + + public function getHealthRecordCacheKey() { + $host = $this->getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + +/** + * @return PhabricatorClusterServiceHealthRecord + */ + public function getHealthRecord() { + if (!$this->healthRecord) { + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); + } + return $this->healthRecord; + } + + public function didHealthCheck($reachable) { + $record = $this->getHealthRecord(); + $should_check = $record->getShouldCheck(); + + if ($should_check) { + $record->didHealthCheck($reachable); + } + } + + /** + * @return string[] Get a list of fields to show in the status overview UI + */ + abstract public function getStatusViewColumns(); + + /** + * @return PhabricatorFulltextStorageEngine + */ + abstract public function getEngine(); + + abstract public function getConnectionStatus(); + + public static function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + + $services = self::getAllServices(); + $indexed = 0; + foreach (self::getWritableHostForEachService() as $host) { + $host->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } + if ($indexed == 0) { + throw new PhabricatorClusterNoHostForRoleException('write'); + } + } + + public static function executeSearch(PhabricatorSavedQuery $query) { + $services = self::getAllServices(); + foreach ($services as $service) { + $hosts = $service->getAllHostsForRole('read'); + // try all hosts until one succeeds + foreach ($hosts as $host) { + $last_exception = null; + try { + $res = $host->getEngine()->executeSearch($query); + // return immediately if we get results without an exception + return $res; + } catch (Exception $ex) { + // try each server in turn, only throw if none succeed + $last_exception = $ex; + } + } + } + if ($last_exception) { + throw $last_exception; + } + return $res; + } + +}