D17384.id41972.diff
No OneTemporary
Actions

Size

63 KB

Referenced Files

None

Subscribers

None

D17384.id41972.diff
View Options

	diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
	--- a/src/__phutil_library_map__.php
	+++ b/src/__phutil_library_map__.php
	@@ -2255,12 +2255,15 @@
	'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
	'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
	'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
	- 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
	- 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php',
	- 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
	- 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php',
	- 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
	- 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php',
	+ 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php',
	+ 'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php',
	+ 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php',
	+ 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php',
	+ 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php',
	+ 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php',
	+ 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php',
	+ 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php',
	+ 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php',
	'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
	'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
	'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
	@@ -2306,6 +2309,7 @@
	'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
	'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php',
	'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php',
	+ 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php',
	'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
	'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
	'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
	@@ -2537,7 +2541,6 @@
	'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
	'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
	'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
	- 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php',
	'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
	'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php',
	'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
	@@ -2642,6 +2645,8 @@
	'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php',
	'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php',
	'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php',
	+ 'PhabricatorElasticSearchCluster' => 'infrastructure/cluster/PhabricatorElasticSearchCluster.php',
	+ 'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php',
	'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php',
	'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php',
	'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php',
	@@ -3064,6 +3069,7 @@
	'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php',
	'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php',
	'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php',
	+ 'PhabricatorMySQLSearchCluster' => 'infrastructure/cluster/PhabricatorMySQLSearchCluster.php',
	'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php',
	'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php',
	'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php',
	@@ -3746,6 +3752,7 @@
	'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php',
	'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php',
	'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php',
	+ 'PhabricatorSearchCluster' => 'infrastructure/cluster/PhabricatorSearchCluster.php',
	'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php',
	'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php',
	'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php',
	@@ -7280,6 +7287,9 @@
	'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
	'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException',
	'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
	+ 'PhabricatorClusterNoHostForRoleException' => 'Exception',
	+ 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType',
	+ 'PhabricatorClusterServiceHealthRecord' => 'Phobject',
	'PhabricatorClusterStrandedException' => 'PhabricatorClusterException',
	'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
	'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
	@@ -7331,6 +7341,7 @@
	'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
	'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController',
	'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController',
	+ 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController',
	'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
	'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
	'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
	@@ -7599,7 +7610,6 @@
	'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController',
	'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
	'PhabricatorDataNotAttachedException' => 'Exception',
	- 'PhabricatorDatabaseHealthRecord' => 'Phobject',
	'PhabricatorDatabaseRef' => 'Phobject',
	'PhabricatorDatabaseRefParser' => 'Phobject',
	'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
	@@ -7707,6 +7717,7 @@
	'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting',
	'PhabricatorEditorSetting' => 'PhabricatorStringSetting',
	'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
	+ 'PhabricatorElasticSearchCluster' => 'PhabricatorSearchCluster',
	'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck',
	'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel',
	'PhabricatorEmailContentSource' => 'PhabricatorContentSource',
	@@ -8177,6 +8188,7 @@
	'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions',
	'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine',
	'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
	+ 'PhabricatorMySQLSearchCluster' => 'PhabricatorSearchCluster',
	'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck',
	'PhabricatorNamedQuery' => array(
	'PhabricatorSearchDAO',
	@@ -9036,6 +9048,7 @@
	'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel',
	'PhabricatorSearchBaseController' => 'PhabricatorController',
	'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField',
	+ 'PhabricatorSearchCluster' => 'Phobject',
	'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions',
	'PhabricatorSearchController' => 'PhabricatorSearchBaseController',
	'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField',
	diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php
	--- a/src/applications/config/application/PhabricatorConfigApplication.php
	+++ b/src/applications/config/application/PhabricatorConfigApplication.php
	@@ -69,6 +69,7 @@
	'databases/' => 'PhabricatorConfigClusterDatabasesController',
	'notifications/' => 'PhabricatorConfigClusterNotificationsController',
	'repositories/' => 'PhabricatorConfigClusterRepositoriesController',
	+ 'search/' => 'PhabricatorConfigClusterSearchController',
	),
	),
	);
	diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
	--- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
	+++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
	@@ -11,67 +11,94 @@
	return;
	}

	- $engine = new PhabricatorElasticFulltextStorageEngine();
	+ $services = PhabricatorSearchCluster::getAllServices();

	- $index_exists = null;
	- $index_sane = null;
	- try {
	- $index_exists = $engine->indexExists();
	- if ($index_exists) {
	- $index_sane = $engine->indexIsSane();
	+ foreach ($services as $service) {
	+ if (!$service instanceof PhabricatorElasticSearchCluster) {
	+ continue;
	}
	- } catch (Exception $ex) {
	- $summary = pht('Elasticsearch is not reachable as configured.');
	- $message = pht(
	- 'Elasticsearch is configured (with the %s setting) but Phabricator '.
	- 'encountered an exception when trying to test the index.'.
	- "\n\n".
	- '%s',
	- phutil_tag('tt', array(), 'search.elastic.host'),
	- phutil_tag('pre', array(), $ex->getMessage()));
	+ $host = $service->getHostForRole('read');
	+ if (!$host) {
	+ $summary = pht('No readable Elasticsearch hosts configured.');
	+ $message = pht(
	+ 'Elasticsearch is configured (with the %s setting) but none are '.
	+ 'configured with the "read" role.',
	+ phutil_tag('tt', array(), 'cluster.search'));

	- $this->newIssue('elastic.misconfigured')
	- ->setName(pht('Elasticsearch Misconfigured'))
	- ->setSummary($summary)
	- ->setMessage($message)
	- ->addRelatedPhabricatorConfig('search.elastic.host');
	- return;
	- }
	+ $this->newIssue('elastic.notreadable')
	+ ->setName(pht('No readable Elasticsearch hosts'))
	+ ->setSummary($summary)
	+ ->setMessage($message)
	+ ->addRelatedPhabricatorConfig('cluster.search');

	- if (!$index_exists) {
	- $summary = pht(
	- 'You enabled Elasticsearch but the index does not exist.');
	+ continue;
	+ }
	+ $index_exists = null;
	+ $index_sane = null;
	+ try {
	+ $engine = $host->getEngine();
	+ $index_exists = $engine->indexExists();
	+ if ($index_exists) {
	+ $index_sane = $engine->indexIsSane();
	+ }
	+ } catch (Exception $ex) {
	+ $summary = pht('Elasticsearch is not reachable as configured.');
	+ $message = pht(
	+ 'Elasticsearch is configured (with the %s setting) but Phabricator '.
	+ 'encountered an exception when trying to test the index.'.
	+ "\n\n".
	+ '%s',
	+ phutil_tag('tt', array(), 'cluster.search'),
	+ phutil_tag('pre', array(), $ex->getMessage()));
	+
	+ $this->newIssue('elastic.misconfigured')
	+ ->setName(pht('Elasticsearch Misconfigured'))
	+ ->setSummary($summary)
	+ ->setMessage($message)
	+ ->addRelatedPhabricatorConfig('cluster.search');
	+ return;
	+ }

	- $message = pht(
	- 'You likely enabled search.elastic.host without creating the '.
	- 'index. Run `./bin/search init` to correct the index.');
	+ if (!$index_exists) {
	+ $summary = pht(
	+ 'You enabled Elasticsearch but the index does not exist.');

	- $this
	- ->newIssue('elastic.missing-index')
	- ->setName(pht('Elasticsearch index Not Found'))
	- ->setSummary($summary)
	- ->setMessage($message)
	- ->addRelatedPhabricatorConfig('search.elastic.host');
	- } else if (!$index_sane) {
	- $summary = pht(
	- 'Elasticsearch index exists but needs correction.');
	+ $message = pht(
	+ 'You likely enabled cluster.search without creating the '.
	+ 'index. Run `./bin/search init` to correct the index.');

	- $message = pht(
	- 'Either the Phabricator schema for Elasticsearch has changed '.
	- 'or Elasticsearch created the index automatically. Run '.
	- '`./bin/search init` to correct the index.');
	+ $this
	+ ->newIssue('elastic.missing-index')
	+ ->setName(pht('Elasticsearch index Not Found'))
	+ ->setSummary($summary)
	+ ->setMessage($message)
	+ ->addRelatedPhabricatorConfig('cluster.search');
	+ } else if (!$index_sane) {
	+ $summary = pht(
	+ 'Elasticsearch index exists but needs correction.');

	- $this
	- ->newIssue('elastic.broken-index')
	- ->setName(pht('Elasticsearch index Incorrect'))
	- ->setSummary($summary)
	- ->setMessage($message);
	+ $message = pht(
	+ 'Either the Phabricator schema for Elasticsearch has changed '.
	+ 'or Elasticsearch created the index automatically. Run '.
	+ '`./bin/search init` to correct the index.');
	+
	+ $this
	+ ->newIssue('elastic.broken-index')
	+ ->setName(pht('Elasticsearch index Incorrect'))
	+ ->setSummary($summary)
	+ ->setMessage($message);
	+ }
	}
	}

	protected function shouldUseElasticSearchEngine() {
	- $search_engine = PhabricatorFulltextStorageEngine::loadEngine();
	- return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine);
	+ $services = PhabricatorSearchCluster::getAllServices();
	+ foreach ($services as $service) {
	+ if ($service instanceof PhabricatorElasticSearchCluster) {
	+ return true;
	+ }
	+ }
	+ return false;
	}

	}
	diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
	new file mode 100644
	--- /dev/null
	+++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
	@@ -0,0 +1,149 @@
	+<?php
	+
	+final class PhabricatorConfigClusterSearchController
	+ extends PhabricatorConfigController {
	+
	+ public function handleRequest(AphrontRequest $request) {
	+ $nav = $this->buildSideNavView();
	+ $nav->selectFilter('cluster/search/');
	+
	+ $title = pht('Cluster Search');
	+ $doc_href = PhabricatorEnv::getDoclink('Cluster: Search');
	+
	+ $header = id(new PHUIHeaderView())
	+ ->setHeader($title)
	+ ->setProfileHeader(true)
	+ ->addActionLink(
	+ id(new PHUIButtonView())
	+ ->setIcon('fa-book')
	+ ->setHref($doc_href)
	+ ->setTag('a')
	+ ->setText(pht('Documentation')));
	+
	+ $crumbs = $this
	+ ->buildApplicationCrumbs($nav)
	+ ->addTextCrumb($title)
	+ ->setBorder(true);
	+
	+ $search_status = $this->buildClusterSearchStatus();
	+
	+ $content = id(new PhabricatorConfigPageView())
	+ ->setHeader($header)
	+ ->setContent($search_status);
	+
	+ return $this->newPage()
	+ ->setTitle($title)
	+ ->setCrumbs($crumbs)
	+ ->setNavigation($nav)
	+ ->appendChild($content)
	+ ->addClass('white-background');
	+ }
	+
	+ private function buildClusterSearchStatus() {
	+ $viewer = $this->getViewer();
	+
	+ $services = PhabricatorSearchCluster::getAllServices();
	+ Javelin::initBehavior('phabricator-tooltips');
	+ $status_map = PhabricatorDatabaseRef::getConnectionStatusMap();
	+ $rows = array();
	+ foreach ($services as $service) {
	+
	+ $reachable = false;
	+ try {
	+ $engine = $service->getEngine();
	+ $reachable = $engine->indexExists();
	+ } catch (Exception $ex) {
	+ $reachable = false;
	+ }
	+ $service->didHealthCheck($reachable);
	+
	+ try {
	+ $status = $service->loadServerStatus();
	+ $status = idx($status_map, $status, array());
	+ } catch (Exception $ex) {
	+ $status['icon'] = 'fa-times';
	+ $status['label'] = pht('Connection Error');
	+ $status['color'] = 'red';
	+ }
	+
	+ $type_icon = 'fa-search sky';
	+ $type_tip = $service->getDisplayName();
	+
	+ $type_icon = id(new PHUIIconView())
	+ ->setIcon($type_icon)
	+ ->addSigil('has-tooltip')
	+ ->setMetadata(
	+ array(
	+ 'tip' => $type_tip,
	+ ));
	+
	+ $hosts = $service->getHostRefs();
	+ if (!count($hosts)) {
	+ $hosts = array($service);
	+ }
	+ foreach ($hosts as $host) {
	+ $status_view = array(
	+ id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']),
	+ ' ',
	+ $status['label'],
	+ );
	+ $roles = implode(', ', array_keys($host->getRoles()));
	+ $rows[] = array(
	+ array($type_icon, ' '.$type_tip),
	+ $host->getProtocol(),
	+ $host->getHost(),
	+ $host->getPort(),
	+ $status_view,
	+ $roles,
	+ $engine->getEnginePriority(),
	+ );
	+ }
	+ }
	+
	+ $table = id(new AphrontTableView($rows))
	+ ->setNoDataString(
	+ pht('No search servers are configured.'))
	+ ->setHeaders(
	+ array(
	+ pht('Type'),
	+ pht('Protocol'),
	+ pht('Host'),
	+ pht('Port'),
	+ pht('Status'),
	+ pht('Roles'),
	+ pht('Priority'),
	+ null,
	+ ))
	+ ->setColumnClasses(
	+ array(
	+ null,
	+ null,
	+ null,
	+ null,
	+ null,
	+ null,
	+ null,
	+ 'wide',
	+ ));
	+
	+ return $table;
	+ }
	+
	+ private function checkIcon($check) {
	+ $icon = $check
	+ ? 'fa-check green'
	+ : 'fa-times red';
	+ $label = $check
	+ ? pht('Yes')
	+ : pht('No');
	+ $view = id(new PHUIIconView())
	+ ->setIcon($icon)
	+ ->addSigil('has-tooltip')
	+ ->setMetadata(
	+ array(
	+ 'tip' => $label,
	+ ));
	+
	+ return $view;
	+ }
	+}
	diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php
	--- a/src/applications/config/controller/PhabricatorConfigController.php
	+++ b/src/applications/config/controller/PhabricatorConfigController.php
	@@ -42,8 +42,11 @@
	pht('Notification Servers'), null, 'fa-bell-o');
	$nav->addFilter('cluster/repositories/',
	pht('Repository Servers'), null, 'fa-code');
	+ $nav->addFilter('cluster/search/',
	+ pht('Search Servers'), null, 'fa-search');
	$nav->addLabel(pht('Modules'));

	+
	$modules = PhabricatorConfigModule::getAllModules();
	foreach ($modules as $key => $module) {
	$nav->addFilter('module/'.$key.'/',
	diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php
	--- a/src/applications/config/option/PhabricatorClusterConfigOptions.php
	+++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php
	@@ -38,6 +38,9 @@
	$intro_href = PhabricatorEnv::getDoclink('Clustering Introduction');
	$intro_name = pht('Clustering Introduction');

	+ $search_type = 'custom:PhabricatorClusterSearchConfigOptionType';
	+ $search_help = 'TODO';
	+
	return array(
	$this->newOption('cluster.addresses', 'list<string>', array())
	->setLocked(true)
	@@ -114,6 +117,21 @@
	->setSummary(
	pht('Configure database read replicas.'))
	->setDescription($databases_help),
	+ $this->newOption('cluster.search', $search_type, array())
	+ ->setLocked(true)
	+ ->setSummary(
	+ pht('Configure full-text search services.'))
	+ ->setDescription($search_help)
	+ ->setDefault(
	+ array(
	+ array(
	+ 'type' => 'mysql',
	+ 'roles' => array(
	+ 'read' => true,
	+ 'write' => true,
	+ ),
	+ ),
	+ )),
	);
	}

	diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php
	--- a/src/applications/maniphest/query/ManiphestTaskQuery.php
	+++ b/src/applications/maniphest/query/ManiphestTaskQuery.php
	@@ -519,8 +519,8 @@
	$fulltext_query->setParameter('types',
	array(ManiphestTaskPHIDType::TYPECONST));

	- $engine = PhabricatorFulltextStorageEngine::loadEngine();
	- $fulltext_results = $engine->executeSearch($fulltext_query);
	+ $fulltext_results = PhabricatorSearchCluster::executeSearch(
	+ $fulltext_query);

	if (empty($fulltext_results)) {
	$fulltext_results = array(null);
	diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php
	--- a/src/applications/search/config/PhabricatorSearchConfigOptions.php
	+++ b/src/applications/search/config/PhabricatorSearchConfigOptions.php
	@@ -20,7 +20,15 @@
	}

	public function getOptions() {
	+ $servers_type = 'custom:PhabricatorClusterSearchConfigOptionType';
	+ $servers_help = 'TODO';
	+
	return array(
	+ $this->newOption('search.servers', $servers_type, array())
	+ ->setLocked(true)
	+ ->setSummary(
	+ pht('Configure full-text search servers.'))
	+ ->setDescription($servers_help),
	$this->newOption('search.elastic.host', 'string', null)
	->setLocked(true)
	->setDescription(pht('Elastic Search host.'))
	diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
	--- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
	+++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
	@@ -3,13 +3,31 @@
	final class PhabricatorElasticFulltextStorageEngine
	extends PhabricatorFulltextStorageEngine {

	- private $uri;
	+ private $ref;
	private $index;
	private $timeout;
	+ private $version;
	+ private $timestampFieldKey;
	+ private $textFieldType;
	+ private static $tagCache = array();
	+
	+ public function setRef(PhabricatorElasticSearchCluster $ref) {
	+ $this->ref = $ref;
	+ $this->index = str_replace('/', '', $ref->getPath());
	+ $this->version = (int)$ref->getVersion();
	+
	+ $this->timestampFieldKey = $this->version < 2
	+ ? '_timestamp'
	+ : 'lastModified';
	+
	+ $this->textFieldType = $this->version >= 5
	+ ? 'text'
	+ : 'string';
	+ return $this;
	+ }

	- public function __construct() {
	- $this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
	- $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace');
	+ public function isEnabled() {
	+ return true;
	}

	public function getEngineIdentifier() {
	@@ -20,27 +38,13 @@
	return 10;
	}

	- public function isEnabled() {
	- return (bool)$this->uri;
	- }
	-
	- public function setURI($uri) {
	- $this->uri = $uri;
	- return $this;
	- }
	-
	- public function setIndex($index) {
	- $this->index = $index;
	- return $this;
	- }
	-
	public function setTimeout($timeout) {
	$this->timeout = $timeout;
	return $this;
	}

	- public function getURI() {
	- return $this->uri;
	+ public function getURI($path = '') {
	+ return $this->ref->getURI($path);
	}

	public function getIndex() {
	@@ -51,6 +55,57 @@
	return $this->timeout;
	}

	+
	+ protected function resolveTags($tags) {
	+
	+ $lookup_phids = array();
	+ foreach ($tags as $phid) {
	+ if (!isset(self::$tagCache[$phid])) {
	+ $lookup_phids[] = $phid;
	+ }
	+ }
	+ if (count($lookup_phids)) {
	+ $projects = id(new PhabricatorProjectQuery())
	+ ->setViewer(PhabricatorUser::getOmnipotentUser())
	+ ->withPHIDs($lookup_phids)
	+ ->needSlugs(true)
	+ ->execute();
	+
	+ foreach ($projects as $project) {
	+ $phid = $project->getPHID();
	+ $slugs = $project->getSlugs();
	+ $slugs = mpull($slugs, 'getSlug');
	+ $keywords = $project->getDisplayName().' '.implode(' ', $slugs);
	+ $keywords = strtolower($keywords);
	+ $keywords = str_replace('_', ' ', $keywords);
	+ $keywords = explode(' ', $keywords);
	+ $keywords = array_unique($keywords);
	+ self::$tagCache[$phid] = $keywords;
	+ }
	+ }
	+
	+ $keywords = array();
	+ foreach ($tags as $phid) {
	+ if (isset(self::$tagCache[$phid])) {
	+ $keywords += self::$tagCache[$phid];
	+ }
	+ }
	+ $keywords = array_unique($keywords);
	+ // phlog($result);
	+ return implode(' ', $keywords);
	+ }
	+
	+ public function getTypeConstants($class) {
	+ static $typeconstants = array();
	+ if (!empty($typeconstants[$class])) {
	+ return $typeconstants[$class];
	+ }
	+
	+ $relationship_class = new ReflectionClass($class);
	+ $typeconstants[$class] = $relationship_class->getConstants();
	+ return array_unique(array_values($typeconstants[$class]));
	+ }
	+
	public function reindexAbstractDocument(
	PhabricatorSearchAbstractDocument $doc) {

	@@ -61,27 +116,42 @@
	->withPHIDs(array($phid))
	->executeOne();

	+ $timestamp_key = $this->timestampFieldKey;
	+
	// URL is not used internally but it can be useful externally.
	$spec = array(
	'title' => $doc->getDocumentTitle(),
	'url' => PhabricatorEnv::getProductionURI($handle->getURI()),
	'dateCreated' => $doc->getDocumentCreated(),
	- '_timestamp' => $doc->getDocumentModified(),
	- 'field' => array(),
	- 'relationship' => array(),
	+ $timestamp_key => $doc->getDocumentModified(),
	);

	foreach ($doc->getFieldData() as $field) {
	- $spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
	+ list($field_name, $corpus, $aux) = $field;
	+ if (!isset($spec[$field_name])) {
	+ $spec[$field_name] = $corpus;
	+ } else if (!is_array($spec[$field_name])) {
	+ $spec[$field_name] = array($spec[$field_name], $corpus);
	+ } else {
	+ $spec[$field_name][] = $corpus;
	+ }
	+ if ($aux != null) {
	+ $spec[$field_name.'_aux_phid'] = $aux;
	+ }
	}

	+ $tags = array();
	+
	foreach ($doc->getRelationshipData() as $relationship) {
	list($rtype, $to_phid, $to_type, $time) = $relationship;
	- $spec['relationship'][$rtype][] = array(
	- 'phid' => $to_phid,
	- 'phidType' => $to_type,
	- 'when' => (int)$time,
	- );
	+ $spec[$rtype][] = $to_phid;
	+ if ($rtype == PhabricatorSearchRelationship::RELATIONSHIP_PROJECT) {
	+ $tags[] = $to_phid;
	+ }
	+ }
	+
	+ if (!empty($tags)) {
	+ $spec['tags'] = $this->resolveTags($tags);
	}

	$this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
	@@ -103,10 +173,11 @@
	$doc->setDocumentType($response['_type']);
	$doc->setDocumentTitle($hit['title']);
	$doc->setDocumentCreated($hit['dateCreated']);
	- $doc->setDocumentModified($hit['_timestamp']);
	+ $doc->setDocumentModified($hit[$this->timestampFieldKey]);

	foreach ($hit['field'] as $fdef) {
	- $doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
	+ $field_type = $fdef['type'];
	+ $doc->addField($field_type, $hit[$field_type], $fdef['aux']);
	}

	foreach ($hit['relationship'] as $rtype => $rships) {
	@@ -123,35 +194,45 @@
	}

	private function buildSpec(PhabricatorSavedQuery $query) {
	- $spec = array();
	- $filter = array();
	- $title_spec = array();
	+ $q = new PhabricatorElasticSearchQueryBuilder('bool');
	+ $query_string = $query->getParameter('query');
	+ if (strlen($query_string)) {
	+ $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');

	- if (strlen($query->getParameter('query'))) {
	- $spec[] = array(
	+ $q->must(array(
	'simple_query_string' => array(
	- 'query' => $query->getParameter('query'),
	- 'fields' => array('field.corpus'),
	+ 'query' => $query_string,
	+ 'fields' => array(
	+ 'title^4',
	+ 'body^3',
	+ 'cmnt^2',
	+ 'tags',
	+ '_all',
	+ ),
	+ 'default_operator' => 'and',
	),
	- );
	+ ));

	- $title_spec = array(
	+ $q->should(array(
	'simple_query_string' => array(
	- 'query' => $query->getParameter('query'),
	- 'fields' => array('title'),
	+ 'query' => $query_string,
	+ 'fields' => array_values($fields),
	+ 'analyzer' => 'english_exact',
	+ 'default_operator' => 'and',
	),
	- );
	+ ));
	+
	}

	$exclude = $query->getParameter('exclude');
	if ($exclude) {
	- $filter[] = array(
	+ $q->filter(array(
	'not' => array(
	'ids' => array(
	'values' => array($exclude),
	),
	),
	- );
	+ ));
	}

	$relationship_map = array(
	@@ -176,67 +257,43 @@
	$include_closed = !empty($statuses[$rel_closed]);

	if ($include_open && !$include_closed) {
	- $relationship_map[$rel_open] = true;
	+ $q->exists($rel_open);
	} else if (!$include_open && $include_closed) {
	- $relationship_map[$rel_closed] = true;
	+ $q->exists($rel_closed);
	}

	if ($query->getParameter('withUnowned')) {
	- $relationship_map[$rel_unowned] = true;
	+ $q->exists($rel_unowned);
	}

	$rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER;
	if ($query->getParameter('withAnyOwner')) {
	- $relationship_map[$rel_owner] = true;
	+ $q->exists($rel_owner);
	} else {
	$owner_phids = $query->getParameter('ownerPHIDs', array());
	- $relationship_map[$rel_owner] = $owner_phids;
	- }
	-
	- foreach ($relationship_map as $field => $param) {
	- if (is_array($param) && $param) {
	- $should = array();
	- foreach ($param as $val) {
	- $should[] = array(
	- 'match' => array(
	- "relationship.{$field}.phid" => array(
	- 'query' => $val,
	- 'type' => 'phrase',
	- ),
	- ),
	- );
	- }
	- // We couldn't solve it by minimum_number_should_match because it can
	- // match multiple owners without matching author.
	- $spec[] = array('bool' => array('should' => $should));
	- } else if ($param) {
	- $filter[] = array(
	- 'exists' => array(
	- 'field' => "relationship.{$field}.phid",
	- ),
	- );
	+ if (count($owner_phids)) {
	+ $q->terms($rel_owner, $owner_phids);
	}
	}

	- if ($spec) {
	- $spec = array('query' => array('bool' => array('must' => $spec)));
	- if ($title_spec) {
	- $spec['query']['bool']['should'] = $title_spec;
	+ foreach ($relationship_map as $field => $phids) {
	+ if (is_array($phids) && !empty($phids)) {
	+ $q->terms($field, $phids);
	}
	}

	- if ($filter) {
	- $filter = array('filter' => array('and' => $filter));
	- if (!$spec) {
	- $spec = array('query' => array('match_all' => new stdClass()));
	- }
	- $spec = array(
	- 'query' => array(
	- 'filtered' => $spec + $filter,
	- ),
	- );
	+ if (!$q->clauseCount('must')) {
	+ $q->must(array('match_all' => array('boost' => 1 )));
	}

	+ $spec = array(
	+ '_source' => false,
	+ 'query' => array(
	+ 'bool' => $q->toArray(),
	+ ),
	+ );
	+
	+
	if (!$query->getParameter('query')) {
	$spec['sort'] = array(
	array('dateCreated' => 'desc'),
	@@ -244,8 +301,8 @@
	}

	$spec['from'] = (int)$query->getParameter('offset', 0);
	- $spec['size'] = (int)$query->getParameter('limit', 25);
	-
	+ $spec['size'] = min(10000, (int)$query->getParameter('limit', 25));
	+ // phlog(json_encode($spec));
	return $spec;
	}

	@@ -261,22 +318,9 @@
	// some bigger index). Use '/$types/_search' instead.
	$uri = '/'.implode(',', $types).'/_search';

	- try {
	- $response = $this->executeRequest($uri, $this->buildSpec($query));
	- } catch (HTTPFutureHTTPResponseStatus $ex) {
	- // elasticsearch probably uses Lucene query syntax:
	- // http://lucene.apache.org/core/3_6_1/queryparsersyntax.html
	- // Try literal search if operator search fails.
	- if (!strlen($query->getParameter('query'))) {
	- throw $ex;
	- }
	- $query = clone $query;
	- $query->setParameter(
	- 'query',
	- addcslashes(
	- $query->getParameter('query'), '+-&\|!(){}[]^"~*?:\\'));
	- $response = $this->executeRequest($uri, $this->buildSpec($query));
	- }
	+ $spec = $this->buildSpec($query);
	+ $response = $this->executeRequest($uri, $spec);
	+ // phlog($response);

	$phids = ipull($response['hits']['hits'], '_id');
	return $phids;
	@@ -284,7 +328,17 @@

	public function indexExists() {
	try {
	- return (bool)$this->executeRequest('/_status/', array());
	+
	+ if ($this->version >= 5) {
	+ $uri = '/_stats/';
	+ $res = $this->executeRequest($uri, array());
	+ return isset($res['indices']['phabricator']);
	+ } else if ($this->version >= 2) {
	+ $uri = '';
	+ } else {
	+ $uri = '/_status/';
	+ }
	+ return (bool)$this->executeRequest($uri, array());
	} catch (HTTPFutureHTTPResponseStatus $e) {
	if ($e->getStatusCode() == 404) {
	return false;
	@@ -299,43 +353,63 @@
	'index' => array(
	'auto_expand_replicas' => '0-2',
	'analysis' => array(
	- 'filter' => array(
	- 'trigrams_filter' => array(
	- 'min_gram' => 3,
	- 'type' => 'ngram',
	- 'max_gram' => 3,
	- ),
	- ),
	'analyzer' => array(
	- 'custom_trigrams' => array(
	- 'type' => 'custom',
	- 'filter' => array(
	- 'lowercase',
	- 'kstem',
	- 'trigrams_filter',
	- ),
	+ 'english_exact' => array(
	'tokenizer' => 'standard',
	+ 'filter' => array('lowercase'),
	),
	),
	),
	),
	);

	+ $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType');
	+ $relationships = $this->getTypeConstants('PhabricatorSearchRelationship');
	+
	$types = array_keys(
	PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
	+
	foreach ($types as $type) {
	- // Use the custom trigram analyzer for the corpus of text
	- $data['mappings'][$type]['properties']['field']['properties']['corpus'] =
	- array('type' => 'string', 'analyzer' => 'custom_trigrams');
	+ $properties = array();
	+ foreach ($fields as $field) {
	+ // Use the custom analyzer for the corpus of text
	+ $properties[$field] = array(
	+ 'type' => $this->textFieldType,
	+ 'analyzer' => 'english_exact',
	+ 'search_analyzer' => 'english',
	+ 'search_quote_analyzer' => 'english_exact',
	+ );
	+ }
	+
	+ foreach ($relationships as $rel) {
	+ $properties[$rel] = array(
	+ 'type' => $this->textFieldType,
	+ );
	+ if ($this->version < 5) {
	+ $properties[$rel]['index'] = 'not_analyzed';
	+ }
	+ }

	// Ensure we have dateCreated since the default query requires it
	- $data['mappings'][$type]['properties']['dateCreated']['type'] = 'string';
	- }
	+ $properties['dateCreated']['type'] = 'date';

	+ // Replaces deprecated _timestamp for elasticsearch 2
	+ if ((int)$this->version >= 2) {
	+ $properties['lastModified']['type'] = 'date';
	+ }
	+
	+ $properties['tags'] = array(
	+ 'type' => $this->textFieldType,
	+ 'analyzer' => 'english',
	+ 'store' => true,
	+ );
	+ $data['mappings'][$type]['properties'] = $properties;
	+ }
	return $data;
	}

	public function indexIsSane() {
	+
	if (!$this->indexExists()) {
	return false;
	}
	@@ -345,7 +419,8 @@
	$actual = array_merge($cur_settings[$this->index],
	$cur_mapping[$this->index]);

	- return $this->check($actual, $this->getIndexConfiguration());
	+ $res = $this->check($actual, $this->getIndexConfiguration());
	+ return $res;
	}

	/**
	@@ -410,12 +485,20 @@
	$this->executeRequest('/', $data, 'PUT');
	}

	+ public function didHealthCheck($reachable) {
	+ static $cache=null;
	+ if ($cache !== null) {
	+ return;
	+ }
	+
	+ $cache = $reachable;
	+ $this->ref->didHealthCheck($reachable);
	+ return $this;
	+ }
	+
	private function executeRequest($path, array $data, $method = 'GET') {
	- $uri = new PhutilURI($this->uri);
	- $uri->setPath($this->index);
	- $uri->appendPath($path);
	+ $uri = $this->ref->getURI($path);
	$data = json_encode($data);
	-
	$future = new HTTPSFuture($uri, $data);
	if ($method != 'GET') {
	$future->setMethod($method);
	@@ -423,19 +506,30 @@
	if ($this->getTimeout()) {
	$future->setTimeout($this->getTimeout());
	}
	- list($body) = $future->resolvex();
	+ try {
	+ list($body) = $future->resolvex();
	+ } catch (HTTPFutureResponseStatus $ex) {
	+ if ($ex->isTimeout() \|\| (int)$ex->getStatusCode() > 499) {
	+ $this->didHealthCheck(false);
	+ }
	+ throw $ex;
	+ }

	if ($method != 'GET') {
	return null;
	}

	try {
	- return phutil_json_decode($body);
	+ $data = phutil_json_decode($body);
	+ $this->didHealthCheck(true);
	+ return $data;
	} catch (PhutilJSONParserException $ex) {
	+ $this->didHealthCheck(false);
	throw new PhutilProxyException(
	pht('ElasticSearch server returned invalid JSON!'),
	$ex);
	}
	+
	}

	}
	diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php
	new file mode 100644
	--- /dev/null
	+++ b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php
	@@ -0,0 +1,78 @@
	+<?php
	+
	+class PhabricatorElasticSearchQueryBuilder {
	+ protected $name;
	+ protected $clauses = array();
	+
	+
	+ public function getClauses($termkey = null) {
	+ $clauses = $this->clauses;
	+ if ($termkey == null) {
	+ return $clauses;
	+ }
	+ if (isset($clauses[$termkey])) {
	+ return $clauses[$termkey];
	+ }
	+ return array();
	+ }
	+
	+ public function clauseCount($clausekey) {
	+ if (isset($this->clauses[$clausekey])) {
	+ return count($this->clauses[$clausekey]);
	+ } else {
	+ return 0;
	+ }
	+ }
	+
	+ public function exists($field) {
	+ return $this->addClause('filter', array(
	+ 'exists' => array(
	+ 'field' => $field,
	+ ),
	+ ));
	+ }
	+
	+ public function terms($field, $values) {
	+ return $this->addClause('filter', array(
	+ 'terms' => array(
	+ $field => array_values($values),
	+ ),
	+ ));
	+ }
	+
	+ public function must($clause) {
	+ return $this->addClause('must', $clause);
	+ }
	+
	+ public function filter($clause) {
	+ return $this->addClause('filter', $clause);
	+ }
	+
	+ public function should($clause) {
	+ return $this->addClause('should', $clause);
	+ }
	+
	+ public function mustNot($clause) {
	+ return $this->addClause('must_not', $clause);
	+ }
	+
	+ public function addClause($clause, $terms) {
	+ $this->clauses[$clause][] = $terms;
	+ return $this;
	+ }
	+
	+ public function toArray() {
	+ $clauses = $this->getClauses();
	+ return $clauses;
	+ $cleaned = array();
	+ foreach ($clauses as $clause => $subclauses) {
	+ if (is_array($subclauses) && count($subclauses) == 1) {
	+ $cleaned[$clause] = array_shift($subclauses);
	+ } else {
	+ $cleaned[$clause] = $subclauses;
	+ }
	+ }
	+ return $cleaned;
	+ }
	+
	+}
	diff --git a/src/applications/search/index/PhabricatorFulltextEngine.php b/src/applications/search/index/PhabricatorFulltextEngine.php
	--- a/src/applications/search/index/PhabricatorFulltextEngine.php
	+++ b/src/applications/search/index/PhabricatorFulltextEngine.php
	@@ -40,8 +40,7 @@
	$extension->indexFulltextObject($object, $document);
	}

	- $storage_engine = PhabricatorFulltextStorageEngine::loadEngine();
	- $storage_engine->reindexAbstractDocument($document);
	+ PhabricatorSearchCluster::reindexAbstractDocument($document);
	}

	protected function newAbstractDocument($object) {
	diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
	--- a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
	+++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
	@@ -13,27 +13,41 @@
	public function execute(PhutilArgumentParser $args) {
	$console = PhutilConsole::getConsole();

	- $engine = PhabricatorFulltextStorageEngine::loadEngine();
	-
	$work_done = false;
	- if (!$engine->indexExists()) {
	- $console->writeOut(
	- '%s',
	- pht('Index does not exist, creating...'));
	- $engine->initIndex();
	- $console->writeOut(
	- "%s\n",
	- pht('done.'));
	- $work_done = true;
	- } else if (!$engine->indexIsSane()) {
	- $console->writeOut(
	- '%s',
	- pht('Index exists but is incorrect, fixing...'));
	- $engine->initIndex();
	+ foreach (PhabricatorSearchCluster::getAllServices() as $service) {
	$console->writeOut(
	"%s\n",
	- pht('done.'));
	- $work_done = true;
	+ pht('Initializing search service "%s"', $service->getDisplayName()));
	+
	+ try {
	+ $host = $service->getHostForRole('write');
	+ } catch (PhabricatorClusterNoHostForRoleException $e) {
	+ // If there are no writable hosts for a given cluster, skip it
	+ $console->writeOut("%s\n", $e->getExceptionTitle());
	+ continue;
	+ }
	+
	+ $engine = $host->getEngine();
	+
	+ if (!$engine->indexExists()) {
	+ $console->writeOut(
	+ '%s',
	+ pht('Index does not exist, creating...'));
	+ $engine->initIndex();
	+ $console->writeOut(
	+ "%s\n",
	+ pht('done.'));
	+ $work_done = true;
	+ } else if (!$engine->indexIsSane()) {
	+ $console->writeOut(
	+ '%s',
	+ pht('Index exists but is incorrect, fixing...'));
	+ $engine->initIndex();
	+ $console->writeOut(
	+ "%s\n",
	+ pht('done.'));
	+ $work_done = true;
	+ }
	}

	if ($work_done) {
	diff --git a/src/applications/search/query/PhabricatorSearchDocumentQuery.php b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
	--- a/src/applications/search/query/PhabricatorSearchDocumentQuery.php
	+++ b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
	@@ -73,10 +73,7 @@
	$query = id(clone($this->savedQuery))
	->setParameter('offset', $this->getOffset())
	->setParameter('limit', $this->getRawResultLimit());
	-
	- $engine = PhabricatorFulltextStorageEngine::loadEngine();
	-
	- return $engine->executeSearch($query);
	+ return PhabricatorSearchCluster::executeSearch($query);
	}

	public function getQueryApplicationClass() {
	diff --git a/src/applications/search/worker/PhabricatorSearchWorker.php b/src/applications/search/worker/PhabricatorSearchWorker.php
	--- a/src/applications/search/worker/PhabricatorSearchWorker.php
	+++ b/src/applications/search/worker/PhabricatorSearchWorker.php
	@@ -51,6 +51,7 @@
	$lock->unlock();

	if (!($ex instanceof PhabricatorWorkerPermanentFailureException)) {
	+ phlog($ex);
	$ex = new PhabricatorWorkerPermanentFailureException(
	pht(
	'Failed to update search index for document "%s": %s',
	diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
	rename from src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
	rename to src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
	--- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
	+++ b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
	@@ -1,20 +1,19 @@
	<?php

	-final class PhabricatorDatabaseHealthRecord
	+class PhabricatorClusterServiceHealthRecord
	extends Phobject {

	- private $ref;
	+ private $cacheKey;
	private $shouldCheck;
	private $isHealthy;
	private $upEventCount;
	private $downEventCount;

	- public function __construct(PhabricatorDatabaseRef $ref) {
	- $this->ref = $ref;
	+ public function __construct($cache_key) {
	+ $this->cacheKey = $cache_key;
	$this->readState();
	}

	-
	/**
	* Is the database currently healthy?
	*/
	@@ -153,18 +152,13 @@
	}
	}

	- private function getHealthRecordCacheKey() {
	- $ref = $this->ref;
	-
	- $host = $ref->getHost();
	- $port = $ref->getPort();
	-
	- return "cluster.db.health({$host}, {$port})";
	+ public function getCacheKey() {
	+ return $this->cacheKey;
	}

	private function readHealthRecord() {
	$cache = PhabricatorCaches::getSetupCache();
	- $cache_key = $this->getHealthRecordCacheKey();
	+ $cache_key = $this->getCacheKey();
	$health_record = $cache->getKey($cache_key);

	if (!is_array($health_record)) {
	@@ -180,7 +174,7 @@

	private function writeHealthRecord(array $record) {
	$cache = PhabricatorCaches::getSetupCache();
	- $cache_key = $this->getHealthRecordCacheKey();
	+ $cache_key = $this->getCacheKey();
	$cache->setKey($cache_key, $record);
	}

	diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	@@ -14,6 +14,7 @@
	const REPLICATION_SLOW = 'replica-slow';
	const REPLICATION_NOT_REPLICATING = 'not-replicating';

	+ const KEY_HEALTH = 'cluster.db.health';
	const KEY_REFS = 'cluster.db.refs';
	const KEY_INDIVIDUAL = 'cluster.db.individual';

	@@ -489,9 +490,18 @@
	return $this;
	}

	+ private function getHealthRecordCacheKey() {
	+ $host = $this->getHost();
	+ $port = $this->getPort();
	+ $key = self::KEY_HEALTH;
	+
	+ return "{$key}({$host}, {$port})";
	+ }
	+
	public function getHealthRecord() {
	if (!$this->healthRecord) {
	- $this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
	+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
	+ $this->getHealthRecordCacheKey());
	}
	return $this->healthRecord;
	}
	diff --git a/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php b/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php
	new file mode 100644
	--- /dev/null
	+++ b/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php
	@@ -0,0 +1,93 @@
	+<?php
	+
	+final class PhabricatorElasticSearchCluster
	+ extends PhabricatorSearchCluster {
	+
	+ private $engine;
	+ private $version = 5;
	+ private $path = 'phabricator/';
	+ private $protocol = 'http';
	+
	+ const KEY_REFS = 'search.elastic.refs';
	+
	+ public function setConfig($config) {
	+ $this->setRoles(idx($config, 'roles',
	+ array('read' => true, 'write' => true)))
	+ ->setHostRefs(idx($config, 'hosts', array()))
	+ ->setPort(idx($config, 'port', $this->port))
	+ ->setProtocol(idx($config, 'protocol', $this->protocol))
	+ ->setPath(idx($config, 'path', $this->path))
	+ ->setVersion(idx($config, 'version', $this->version));
	+ return $this;
	+ }
	+
	+ public function getDisplayName() {
	+ return pht('ElasticSearch');
	+ }
	+
	+ public function getEngineIdentifier() {
	+ return 'elasticsearch';
	+ }
	+
	+ public function setProtocol($protocol) {
	+ $this->protocol = $protocol;
	+ return $this;
	+ }
	+
	+ public function getProtocol() {
	+ return $this->protocol;
	+ }
	+
	+ public function setPath($path) {
	+ $this->path = $path;
	+ return $this;
	+ }
	+
	+ public function getPath() {
	+ return $this->path;
	+ }
	+
	+ public function setVersion($version) {
	+ $this->version = $version;
	+ return $this;
	+ }
	+
	+ public function getVersion() {
	+ return $this->version;
	+ }
	+
	+ public function getURI($to_path = null) {
	+ $uri = id(new PhutilURI('http://'.$this->getHost()))
	+ ->setProtocol($this->getProtocol())
	+ ->setPort($this->getPort())
	+ ->setPath($this->getPath());
	+
	+ if ($to_path) {
	+ $uri->appendPath($to_path);
	+ }
	+ return $uri;
	+ }
	+
	+ /**
	+ * @return PhabricatorElasticFulltextStorageEngine
	+ */
	+ public function getEngine() {
	+ if (!$this->engine) {
	+ $engine = new PhabricatorElasticFulltextStorageEngine();
	+ $this->engine = $engine->setRef($this);
	+ }
	+ return $this->engine;
	+ }
	+
	+ public function loadServerStatus() {
	+ $status = $this->getEngine()->indexIsSane()
	+ ? 'okay'
	+ : 'fail';
	+ $engine = $this->getEngine();
	+ $status = $engine->indexIsSane()
	+ ? 'okay'
	+ : 'fail';
	+ return $status;
	+ }
	+
	+}
	diff --git a/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php b/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php
	new file mode 100644
	--- /dev/null
	+++ b/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php
	@@ -0,0 +1,41 @@
	+<?php
	+
	+final class PhabricatorMySQLSearchCluster
	+ extends PhabricatorSearchCluster {
	+
	+ private $engine;
	+
	+ public function __construct() {
	+ $this->engine = new PhabricatorMySQLFulltextStorageEngine();
	+ }
	+
	+ public function setConfig($config) {
	+ $this->setRoles(idx($config, 'roles',
	+ array('read' => true, 'write' => true)));
	+ return $this;
	+ }
	+
	+ public function getDisplayName() {
	+ return 'MySQL';
	+ }
	+
	+ public function getEngineIdentifier() {
	+ return 'mysql';
	+ }
	+
	+ public function getEngine() {
	+ return $this->engine;
	+ }
	+
	+ public function getProtocol() {
	+ return 'mysql';
	+ }
	+
	+ public function loadServerStatus() {
	+ PhabricatorDatabaseRef::queryAll();
	+ $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search');
	+ $status = $ref->getConnectionStatus();
	+ return $status;
	+ }
	+
	+}
	diff --git a/src/infrastructure/cluster/PhabricatorSearchCluster.php b/src/infrastructure/cluster/PhabricatorSearchCluster.php
	new file mode 100644
	--- /dev/null
	+++ b/src/infrastructure/cluster/PhabricatorSearchCluster.php
	@@ -0,0 +1,253 @@
	+<?php
	+
	+abstract class PhabricatorSearchCluster
	+ extends Phobject {
	+
	+ const KEY_REFS = 'cluster.search.refs';
	+ const KEY_HEALTH = 'cluster.search.health';
	+
	+ protected $healthRecord;
	+ protected $roles = array();
	+
	+ protected $disabled;
	+ protected $host;
	+ protected $port;
	+ protected $hostRefs = array();
	+
	+ const STATUS_OKAY = 'okay';
	+ const STATUS_FAIL = 'fail';
	+
	+ public function setDisabled($disabled) {
	+ $this->disabled = $disabled;
	+ return $this;
	+ }
	+
	+ public function getDisabled() {
	+ return $this->disabled;
	+ }
	+
	+ public static function getConnectionStatusMap() {
	+ return array(
	+ self::STATUS_OKAY => array(
	+ 'icon' => 'fa-exchange',
	+ 'color' => 'green',
	+ 'label' => pht('Okay'),
	+ ),
	+ self::STATUS_FAIL => array(
	+ 'icon' => 'fa-times',
	+ 'color' => 'red',
	+ 'label' => pht('Failed'),
	+ ),
	+ );
	+ }
	+
	+ public function isWritable() {
	+ return $this->hasRole('write');
	+ }
	+
	+ public function isReadable() {
	+ return $this->hasRole('read');
	+ }
	+
	+ public function hasRole($role) {
	+ return isset($this->roles[$role]) && $this->roles[$role] === true;
	+ }
	+
	+ public function setRoles(array $roles) {
	+ foreach ($roles as $role => $val) {
	+ $this->roles[$role] = $val;
	+ }
	+ return $this;
	+ }
	+
	+ public function getRoles() {
	+ return $this->roles;
	+ }
	+
	+ public function setPort($value) {
	+ $this->port = $value;
	+ return $this;
	+ }
	+
	+ public function getPort() {
	+ return $this->port;
	+ }
	+
	+ public function setHost($value) {
	+ $this->host = $value;
	+ return $this;
	+ }
	+
	+ public function getHost() {
	+ return $this->host;
	+ }
	+
	+ public function setHostRefs(array $refs) {
	+ $this->hostRefs = $refs;
	+ return $this;
	+ }
	+
	+ public function getHostRefs() {
	+ return $this->hostRefs;
	+ }
	+
	+ /** @return PhabricatorSearchCluster */
	+ public function getHostForRole($role) {
	+ $hosts = $this->getAllHostsForRole($role);
	+ if (empty($hosts)) {
	+ throw new PhabricatorClusterNoHostForRoleException($role);
	+ }
	+ $random = array_rand($hosts);
	+ return $hosts[$random];
	+ }
	+
	+ public function getAllHostsForRole($role) {
	+ $hosts = array();
	+ foreach ($this->hostRefs as $host) {
	+ if ($host->hasRole($role)) {
	+ $hosts[] = $host;
	+ }
	+ }
	+ return $hosts;
	+ }
	+
	+ public function getHealthRecordCacheKey() {
	+ $host = $this->getHost();
	+ $port = $this->getPort();
	+ $key = self::KEY_HEALTH;
	+
	+ return "{$key}({$host}, {$port})";
	+ }
	+
	+/**
	+ * @return PhabricatorClusterServiceHealthRecord
	+ */
	+ public function getHealthRecord() {
	+ if (!$this->healthRecord) {
	+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
	+ $this->getHealthRecordCacheKey());
	+ }
	+ return $this->healthRecord;
	+ }
	+
	+ public function didHealthCheck($reachable) {
	+ $record = $this->getHealthRecord();
	+ $should_check = $record->getShouldCheck();
	+
	+ if ($should_check) {
	+ $record->didHealthCheck($reachable);
	+ }
	+ }
	+
	+ /** @return PhabricatorSearchCluster[] */
	+ public static function getAllServices() {
	+ $cache = PhabricatorCaches::getRequestCache();
	+
	+ $refs = $cache->getKey(self::KEY_REFS);
	+ if (!$refs) {
	+ $refs = self::newRefs();
	+ $cache->setKey(self::KEY_REFS, $refs);
	+ }
	+
	+ return $refs;
	+ }
	+
	+ /** find one random writable host from each service.
	+ * @return PhabricatorSearchCluster[] writable cluster hosts
	+ */
	+ public static function getWritableHostForEachService() {
	+ $services = self::getAllServices();
	+ $all_writable = array();
	+ foreach ($services as $service) {
	+ $hosts = $service->getAllHostsForRole('write');
	+ if (count($hosts)) {
	+ $index = array_rand($hosts);
	+ $all_writable[] = $hosts[$index];
	+ }
	+ }
	+ return $all_writable;
	+ }
	+
	+
	+ /**
	+ * @return PhabricatorSearchCluster[]
	+ */
	+ public static function newRefs() {
	+ $services = PhabricatorEnv::getEnvConfig('cluster.search');
	+ $types = id(new PhutilClassMapQuery())
	+ ->setAncestorClass(__CLASS__)
	+ ->setUniqueMethod('getEngineIdentifier')
	+ ->execute();
	+ $refs = array();
	+
	+ foreach ($services as $config) {
	+ if (!isset($types[$config['type']])) {
	+ throw new Exception(pht('Configured search server type is invalid: %s',
	+ $config['type']));
	+ }
	+ $type = $types[$config['type']];
	+ $service = clone($type);
	+ $service->setConfig($config);
	+ if (isset($config['hosts']) && count($config['hosts'])) {
	+ $hosts = array();
	+ foreach ($config['hosts'] as $host) {
	+ $hostref = clone($service);
	+ $hostref->setConfig($host);
	+ $hostref->setPort(idx($host, 'port', $service->getPort()))
	+ ->setHost(idx($host, 'host'))
	+ ->setRoles(idx($host, 'roles', $service->getRoles()));
	+ $hosts[] = $hostref;
	+ }
	+ $service->setHostRefs($hosts);
	+ }
	+ $refs[] = $service;
	+ }
	+
	+ return $refs;
	+ }
	+
	+ /**
	+ * @return PhabricatorFulltextStorageEngine
	+ */
	+ abstract public function getEngine();
	+
	+ abstract public function loadServerStatus();
	+
	+ public static function reindexAbstractDocument(
	+ PhabricatorSearchAbstractDocument $doc) {
	+
	+ $services = self::getAllServices();
	+ $indexed = 0;
	+ foreach (self::getWritableHostForEachService() as $host) {
	+ $host->getEngine()->reindexAbstractDocument($doc);
	+ $indexed++;
	+ }
	+ if ($indexed == 0) {
	+ throw new PhabricatorClusterNoHostForRoleException('write');
	+ }
	+ }
	+
	+ public static function executeSearch(PhabricatorSavedQuery $query) {
	+ $services = self::getAllServices();
	+ foreach ($services as $service) {
	+ $hosts = $service->getAllHostsForRole('read');
	+ // try all hosts until one succeeds
	+ foreach ($hosts as $host) {
	+ $last_exception = null;
	+ try {
	+ $res = $host->getEngine()->executeSearch($query);
	+ // return immediately if we get results without an exception
	+ return $res;
	+ } catch (Exception $ex) {
	+ // try each server in turn, only throw if none succeed
	+ $last_exception = $ex;
	+ }
	+ }
	+ }
	+ if ($last_exception) {
	+ throw $last_exception;
	+ }
	+ return $res;
	+ }
	+
	+}
	diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php
	rename from src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
	rename to src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php
	diff --git a/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php
	new file mode 100644
	--- /dev/null
	+++ b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php
	@@ -0,0 +1,54 @@
	+<?php
	+
	+final class PhabricatorClusterSearchConfigOptionType
	+ extends PhabricatorConfigJSONOptionType {
	+
	+ public function validateOption(PhabricatorConfigOption $option, $value) {
	+ if (!is_array($value)) {
	+ throw new Exception(
	+ pht(
	+ 'Search cluster configuration is not valid: value must be a '.
	+ 'list of search hosts.'));
	+ }
	+
	+ foreach ($value as $index => $spec) {
	+ if (!is_array($spec)) {
	+ throw new Exception(
	+ pht(
	+ 'Search cluster configuration is not valid: each entry in the '.
	+ 'list must be a dictionary describing a search service, but '.
	+ 'the value with index "%s" is not a dictionary.',
	+ $index));
	+
	+ try {
	+ PhutilTypeSpec::checkMap(
	+ $spec,
	+ array(
	+ 'type' => 'string',
	+ 'hosts' => 'optional list<map<string, wild>>',
	+ 'roles' => 'optional map<string, wild>',
	+ 'port' => 'optional int',
	+ 'protocol' => 'optional string',
	+ 'path' => 'optional string',
	+ 'version' => 'optional int',
	+ ));
	+ } catch (Exception $ex) {
	+ throw new Exception(
	+ pht(
	+ 'Search cluster configuration has an invalid service '.
	+ 'specification (at index "%s"): %s.',
	+ $index,
	+ $ex->getMessage()));
	+ }
	+ if (isset($spec['hosts']) && !is_array($spec['hosts'])) {
	+ throw new Exception(
	+ pht(
	+ 'Search cluster configuration has invalid "hosts" specification '.
	+ 'for the service at index "%s": Hosts must be a list with at '.
	+ 'least one dictionary describing a host for this service.'),
	+ $index);
	+ }
	+ }
	+ }
	+ }
	+}
	diff --git a/src/infrastructure/cluster/PhabricatorClusterException.php b/src/infrastructure/cluster/exception/PhabricatorClusterException.php
	rename from src/infrastructure/cluster/PhabricatorClusterException.php
	rename to src/infrastructure/cluster/exception/PhabricatorClusterException.php
	diff --git a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php b/src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php
	rename from src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php
	rename to src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php
	diff --git a/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php
	rename from src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php
	rename to src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php
	diff --git a/src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php
	rename from src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php
	rename to src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php
	diff --git a/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php
	new file mode 100644
	--- /dev/null
	+++ b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php
	@@ -0,0 +1,16 @@
	+<?php
	+
	+class PhabricatorClusterNoHostForRoleException
	+ extends Exception {
	+
	+ private $role;
	+
	+ public function __construct($role) {
	+ $this->role = $role;
	+ }
	+
	+ public function getExceptionTitle() {
	+ pht('Search cluster has no hosts for role "%s"', $this->role);
	+ }
	+
	+}
	diff --git a/src/infrastructure/cluster/PhabricatorClusterStrandedException.php b/src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php
	rename from src/infrastructure/cluster/PhabricatorClusterStrandedException.php
	rename to src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php

File Metadata

Mime Type: text/plain
Expires: Mon, Aug 4, 3:59 PM (1 d, 1 h ago)
Storage Engine: amazon-s3
Storage Format: Encrypted (AES-256-CBC)
Storage Handle: phabricator/secure/vb/tq/tc46fcbjlr736hgy
Default Alt Text: D17384.id41972.diff (63 KB)

D17384.id41972.diffNo OneTemporaryActions

D17384.id41972.diffView Options

File Metadata

Event Timeline

D17384.id41972.diff
No OneTemporary
Actions

D17384.id41972.diff
View Options