diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2255,12 +2255,15 @@ 'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php', 'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php', 'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php', - 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php', - 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php', - 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php', - 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php', - 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php', - 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php', + 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php', + 'PhabricatorClusterException' => 'infrastructure/cluster/exception/PhabricatorClusterException.php', + 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php', + 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php', + 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterNoHostForRoleException' => 'infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php', + 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php', + 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php', + 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/exception/PhabricatorClusterStrandedException.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', 'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php', @@ -2306,6 +2309,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php', 'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php', + 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -2537,7 +2541,6 @@ 'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php', 'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php', 'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php', - 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php', 'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php', 'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php', 'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php', @@ -2642,6 +2645,8 @@ 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', + 'PhabricatorElasticSearchCluster' => 'infrastructure/cluster/PhabricatorElasticSearchCluster.php', + 'PhabricatorElasticSearchQueryBuilder' => 'applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php', 'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php', 'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php', 'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php', @@ -3064,6 +3069,7 @@ 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php', + 'PhabricatorMySQLSearchCluster' => 'infrastructure/cluster/PhabricatorMySQLSearchCluster.php', 'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php', 'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php', 'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php', @@ -3746,6 +3752,7 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'applications/search/applicationpanel/PhabricatorSearchApplicationStorageEnginePanel.php', 'PhabricatorSearchBaseController' => 'applications/search/controller/PhabricatorSearchBaseController.php', 'PhabricatorSearchCheckboxesField' => 'applications/search/field/PhabricatorSearchCheckboxesField.php', + 'PhabricatorSearchCluster' => 'infrastructure/cluster/PhabricatorSearchCluster.php', 'PhabricatorSearchConfigOptions' => 'applications/search/config/PhabricatorSearchConfigOptions.php', 'PhabricatorSearchController' => 'applications/search/controller/PhabricatorSearchController.php', 'PhabricatorSearchCustomFieldProxyField' => 'applications/search/field/PhabricatorSearchCustomFieldProxyField.php', @@ -7280,6 +7287,9 @@ 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterNoHostForRoleException' => 'Exception', + 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType', + 'PhabricatorClusterServiceHealthRecord' => 'Phobject', 'PhabricatorClusterStrandedException' => 'PhabricatorClusterException', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', @@ -7331,6 +7341,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', @@ -7599,7 +7610,6 @@ 'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController', 'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec', 'PhabricatorDataNotAttachedException' => 'Exception', - 'PhabricatorDatabaseHealthRecord' => 'Phobject', 'PhabricatorDatabaseRef' => 'Phobject', 'PhabricatorDatabaseRefParser' => 'Phobject', 'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck', @@ -7707,6 +7717,7 @@ 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorElasticSearchCluster' => 'PhabricatorSearchCluster', 'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailContentSource' => 'PhabricatorContentSource', @@ -8177,6 +8188,7 @@ 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorMySQLSearchCluster' => 'PhabricatorSearchCluster', 'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorNamedQuery' => array( 'PhabricatorSearchDAO', @@ -9036,6 +9048,7 @@ 'PhabricatorSearchApplicationStorageEnginePanel' => 'PhabricatorApplicationConfigurationPanel', 'PhabricatorSearchBaseController' => 'PhabricatorController', 'PhabricatorSearchCheckboxesField' => 'PhabricatorSearchField', + 'PhabricatorSearchCluster' => 'Phobject', 'PhabricatorSearchConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorSearchController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchCustomFieldProxyField' => 'PhabricatorSearchField', diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php --- a/src/applications/config/application/PhabricatorConfigApplication.php +++ b/src/applications/config/application/PhabricatorConfigApplication.php @@ -69,6 +69,7 @@ 'databases/' => 'PhabricatorConfigClusterDatabasesController', 'notifications/' => 'PhabricatorConfigClusterNotificationsController', 'repositories/' => 'PhabricatorConfigClusterRepositoriesController', + 'search/' => 'PhabricatorConfigClusterSearchController', ), ), ); diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php --- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -11,67 +11,94 @@ return; } - $engine = new PhabricatorElasticFulltextStorageEngine(); + $services = PhabricatorSearchCluster::getAllServices(); - $index_exists = null; - $index_sane = null; - try { - $index_exists = $engine->indexExists(); - if ($index_exists) { - $index_sane = $engine->indexIsSane(); + foreach ($services as $service) { + if (!$service instanceof PhabricatorElasticSearchCluster) { + continue; } - } catch (Exception $ex) { - $summary = pht('Elasticsearch is not reachable as configured.'); - $message = pht( - 'Elasticsearch is configured (with the %s setting) but Phabricator '. - 'encountered an exception when trying to test the index.'. - "\n\n". - '%s', - phutil_tag('tt', array(), 'search.elastic.host'), - phutil_tag('pre', array(), $ex->getMessage())); + $host = $service->getHostForRole('read'); + if (!$host) { + $summary = pht('No readable Elasticsearch hosts configured.'); + $message = pht( + 'Elasticsearch is configured (with the %s setting) but none are '. + 'configured with the "read" role.', + phutil_tag('tt', array(), 'cluster.search')); - $this->newIssue('elastic.misconfigured') - ->setName(pht('Elasticsearch Misconfigured')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - return; - } + $this->newIssue('elastic.notreadable') + ->setName(pht('No readable Elasticsearch hosts')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); - if (!$index_exists) { - $summary = pht( - 'You enabled Elasticsearch but the index does not exist.'); + continue; + } + $index_exists = null; + $index_sane = null; + try { + $engine = $host->getEngine(); + $index_exists = $engine->indexExists(); + if ($index_exists) { + $index_sane = $engine->indexIsSane(); + } + } catch (Exception $ex) { + $summary = pht('Elasticsearch is not reachable as configured.'); + $message = pht( + 'Elasticsearch is configured (with the %s setting) but Phabricator '. + 'encountered an exception when trying to test the index.'. + "\n\n". + '%s', + phutil_tag('tt', array(), 'cluster.search'), + phutil_tag('pre', array(), $ex->getMessage())); + + $this->newIssue('elastic.misconfigured') + ->setName(pht('Elasticsearch Misconfigured')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + return; + } - $message = pht( - 'You likely enabled search.elastic.host without creating the '. - 'index. Run `./bin/search init` to correct the index.'); + if (!$index_exists) { + $summary = pht( + 'You enabled Elasticsearch but the index does not exist.'); - $this - ->newIssue('elastic.missing-index') - ->setName(pht('Elasticsearch index Not Found')) - ->setSummary($summary) - ->setMessage($message) - ->addRelatedPhabricatorConfig('search.elastic.host'); - } else if (!$index_sane) { - $summary = pht( - 'Elasticsearch index exists but needs correction.'); + $message = pht( + 'You likely enabled cluster.search without creating the '. + 'index. Run `./bin/search init` to correct the index.'); - $message = pht( - 'Either the Phabricator schema for Elasticsearch has changed '. - 'or Elasticsearch created the index automatically. Run '. - '`./bin/search init` to correct the index.'); + $this + ->newIssue('elastic.missing-index') + ->setName(pht('Elasticsearch index Not Found')) + ->setSummary($summary) + ->setMessage($message) + ->addRelatedPhabricatorConfig('cluster.search'); + } else if (!$index_sane) { + $summary = pht( + 'Elasticsearch index exists but needs correction.'); - $this - ->newIssue('elastic.broken-index') - ->setName(pht('Elasticsearch index Incorrect')) - ->setSummary($summary) - ->setMessage($message); + $message = pht( + 'Either the Phabricator schema for Elasticsearch has changed '. + 'or Elasticsearch created the index automatically. Run '. + '`./bin/search init` to correct the index.'); + + $this + ->newIssue('elastic.broken-index') + ->setName(pht('Elasticsearch index Incorrect')) + ->setSummary($summary) + ->setMessage($message); + } } } protected function shouldUseElasticSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine); + $services = PhabricatorSearchCluster::getAllServices(); + foreach ($services as $service) { + if ($service instanceof PhabricatorElasticSearchCluster) { + return true; + } + } + return false; } } diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php new file mode 100644 --- /dev/null +++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php @@ -0,0 +1,149 @@ +buildSideNavView(); + $nav->selectFilter('cluster/search/'); + + $title = pht('Cluster Search'); + $doc_href = PhabricatorEnv::getDoclink('Cluster: Search'); + + $header = id(new PHUIHeaderView()) + ->setHeader($title) + ->setProfileHeader(true) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb($title) + ->setBorder(true); + + $search_status = $this->buildClusterSearchStatus(); + + $content = id(new PhabricatorConfigPageView()) + ->setHeader($header) + ->setContent($search_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->setNavigation($nav) + ->appendChild($content) + ->addClass('white-background'); + } + + private function buildClusterSearchStatus() { + $viewer = $this->getViewer(); + + $services = PhabricatorSearchCluster::getAllServices(); + Javelin::initBehavior('phabricator-tooltips'); + $status_map = PhabricatorDatabaseRef::getConnectionStatusMap(); + $rows = array(); + foreach ($services as $service) { + + $reachable = false; + try { + $engine = $service->getEngine(); + $reachable = $engine->indexExists(); + } catch (Exception $ex) { + $reachable = false; + } + $service->didHealthCheck($reachable); + + try { + $status = $service->loadServerStatus(); + $status = idx($status_map, $status, array()); + } catch (Exception $ex) { + $status['icon'] = 'fa-times'; + $status['label'] = pht('Connection Error'); + $status['color'] = 'red'; + } + + $type_icon = 'fa-search sky'; + $type_tip = $service->getDisplayName(); + + $type_icon = id(new PHUIIconView()) + ->setIcon($type_icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $type_tip, + )); + + $hosts = $service->getHostRefs(); + if (!count($hosts)) { + $hosts = array($service); + } + foreach ($hosts as $host) { + $status_view = array( + id(new PHUIIconView())->setIcon($status['icon'].' '.$status['color']), + ' ', + $status['label'], + ); + $roles = implode(', ', array_keys($host->getRoles())); + $rows[] = array( + array($type_icon, ' '.$type_tip), + $host->getProtocol(), + $host->getHost(), + $host->getPort(), + $status_view, + $roles, + $engine->getEnginePriority(), + ); + } + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString( + pht('No search servers are configured.')) + ->setHeaders( + array( + pht('Type'), + pht('Protocol'), + pht('Host'), + pht('Port'), + pht('Status'), + pht('Roles'), + pht('Priority'), + null, + )) + ->setColumnClasses( + array( + null, + null, + null, + null, + null, + null, + null, + 'wide', + )); + + return $table; + } + + private function checkIcon($check) { + $icon = $check + ? 'fa-check green' + : 'fa-times red'; + $label = $check + ? pht('Yes') + : pht('No'); + $view = id(new PHUIIconView()) + ->setIcon($icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $label, + )); + + return $view; + } +} diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php --- a/src/applications/config/controller/PhabricatorConfigController.php +++ b/src/applications/config/controller/PhabricatorConfigController.php @@ -42,8 +42,11 @@ pht('Notification Servers'), null, 'fa-bell-o'); $nav->addFilter('cluster/repositories/', pht('Repository Servers'), null, 'fa-code'); + $nav->addFilter('cluster/search/', + pht('Search Servers'), null, 'fa-search'); $nav->addLabel(pht('Modules')); + $modules = PhabricatorConfigModule::getAllModules(); foreach ($modules as $key => $module) { $nav->addFilter('module/'.$key.'/', diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php --- a/src/applications/config/option/PhabricatorClusterConfigOptions.php +++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php @@ -38,6 +38,9 @@ $intro_href = PhabricatorEnv::getDoclink('Clustering Introduction'); $intro_name = pht('Clustering Introduction'); + $search_type = 'custom:PhabricatorClusterSearchConfigOptionType'; + $search_help = 'TODO'; + return array( $this->newOption('cluster.addresses', 'list', array()) ->setLocked(true) @@ -114,6 +117,21 @@ ->setSummary( pht('Configure database read replicas.')) ->setDescription($databases_help), + $this->newOption('cluster.search', $search_type, array()) + ->setLocked(true) + ->setSummary( + pht('Configure full-text search services.')) + ->setDescription($search_help) + ->setDefault( + array( + array( + 'type' => 'mysql', + 'roles' => array( + 'read' => true, + 'write' => true, + ), + ), + )), ); } diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php --- a/src/applications/maniphest/query/ManiphestTaskQuery.php +++ b/src/applications/maniphest/query/ManiphestTaskQuery.php @@ -519,8 +519,8 @@ $fulltext_query->setParameter('types', array(ManiphestTaskPHIDType::TYPECONST)); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $fulltext_results = $engine->executeSearch($fulltext_query); + $fulltext_results = PhabricatorSearchCluster::executeSearch( + $fulltext_query); if (empty($fulltext_results)) { $fulltext_results = array(null); diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php --- a/src/applications/search/config/PhabricatorSearchConfigOptions.php +++ b/src/applications/search/config/PhabricatorSearchConfigOptions.php @@ -20,7 +20,15 @@ } public function getOptions() { + $servers_type = 'custom:PhabricatorClusterSearchConfigOptionType'; + $servers_help = 'TODO'; + return array( + $this->newOption('search.servers', $servers_type, array()) + ->setLocked(true) + ->setSummary( + pht('Configure full-text search servers.')) + ->setDescription($servers_help), $this->newOption('search.elastic.host', 'string', null) ->setLocked(true) ->setDescription(pht('Elastic Search host.')) diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -3,13 +3,31 @@ final class PhabricatorElasticFulltextStorageEngine extends PhabricatorFulltextStorageEngine { - private $uri; + private $ref; private $index; private $timeout; + private $version; + private $timestampFieldKey; + private $textFieldType; + private static $tagCache = array(); + + public function setRef(PhabricatorElasticSearchCluster $ref) { + $this->ref = $ref; + $this->index = str_replace('/', '', $ref->getPath()); + $this->version = (int)$ref->getVersion(); + + $this->timestampFieldKey = $this->version < 2 + ? '_timestamp' + : 'lastModified'; + + $this->textFieldType = $this->version >= 5 + ? 'text' + : 'string'; + return $this; + } - public function __construct() { - $this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host'); - $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace'); + public function isEnabled() { + return true; } public function getEngineIdentifier() { @@ -20,27 +38,13 @@ return 10; } - public function isEnabled() { - return (bool)$this->uri; - } - - public function setURI($uri) { - $this->uri = $uri; - return $this; - } - - public function setIndex($index) { - $this->index = $index; - return $this; - } - public function setTimeout($timeout) { $this->timeout = $timeout; return $this; } - public function getURI() { - return $this->uri; + public function getURI($path = '') { + return $this->ref->getURI($path); } public function getIndex() { @@ -51,6 +55,57 @@ return $this->timeout; } + + protected function resolveTags($tags) { + + $lookup_phids = array(); + foreach ($tags as $phid) { + if (!isset(self::$tagCache[$phid])) { + $lookup_phids[] = $phid; + } + } + if (count($lookup_phids)) { + $projects = id(new PhabricatorProjectQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs($lookup_phids) + ->needSlugs(true) + ->execute(); + + foreach ($projects as $project) { + $phid = $project->getPHID(); + $slugs = $project->getSlugs(); + $slugs = mpull($slugs, 'getSlug'); + $keywords = $project->getDisplayName().' '.implode(' ', $slugs); + $keywords = strtolower($keywords); + $keywords = str_replace('_', ' ', $keywords); + $keywords = explode(' ', $keywords); + $keywords = array_unique($keywords); + self::$tagCache[$phid] = $keywords; + } + } + + $keywords = array(); + foreach ($tags as $phid) { + if (isset(self::$tagCache[$phid])) { + $keywords += self::$tagCache[$phid]; + } + } + $keywords = array_unique($keywords); + // phlog($result); + return implode(' ', $keywords); + } + + public function getTypeConstants($class) { + static $typeconstants = array(); + if (!empty($typeconstants[$class])) { + return $typeconstants[$class]; + } + + $relationship_class = new ReflectionClass($class); + $typeconstants[$class] = $relationship_class->getConstants(); + return array_unique(array_values($typeconstants[$class])); + } + public function reindexAbstractDocument( PhabricatorSearchAbstractDocument $doc) { @@ -61,27 +116,42 @@ ->withPHIDs(array($phid)) ->executeOne(); + $timestamp_key = $this->timestampFieldKey; + // URL is not used internally but it can be useful externally. $spec = array( 'title' => $doc->getDocumentTitle(), 'url' => PhabricatorEnv::getProductionURI($handle->getURI()), 'dateCreated' => $doc->getDocumentCreated(), - '_timestamp' => $doc->getDocumentModified(), - 'field' => array(), - 'relationship' => array(), + $timestamp_key => $doc->getDocumentModified(), ); foreach ($doc->getFieldData() as $field) { - $spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field); + list($field_name, $corpus, $aux) = $field; + if (!isset($spec[$field_name])) { + $spec[$field_name] = $corpus; + } else if (!is_array($spec[$field_name])) { + $spec[$field_name] = array($spec[$field_name], $corpus); + } else { + $spec[$field_name][] = $corpus; + } + if ($aux != null) { + $spec[$field_name.'_aux_phid'] = $aux; + } } + $tags = array(); + foreach ($doc->getRelationshipData() as $relationship) { list($rtype, $to_phid, $to_type, $time) = $relationship; - $spec['relationship'][$rtype][] = array( - 'phid' => $to_phid, - 'phidType' => $to_type, - 'when' => (int)$time, - ); + $spec[$rtype][] = $to_phid; + if ($rtype == PhabricatorSearchRelationship::RELATIONSHIP_PROJECT) { + $tags[] = $to_phid; + } + } + + if (!empty($tags)) { + $spec['tags'] = $this->resolveTags($tags); } $this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT'); @@ -103,10 +173,11 @@ $doc->setDocumentType($response['_type']); $doc->setDocumentTitle($hit['title']); $doc->setDocumentCreated($hit['dateCreated']); - $doc->setDocumentModified($hit['_timestamp']); + $doc->setDocumentModified($hit[$this->timestampFieldKey]); foreach ($hit['field'] as $fdef) { - $doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']); + $field_type = $fdef['type']; + $doc->addField($field_type, $hit[$field_type], $fdef['aux']); } foreach ($hit['relationship'] as $rtype => $rships) { @@ -123,35 +194,45 @@ } private function buildSpec(PhabricatorSavedQuery $query) { - $spec = array(); - $filter = array(); - $title_spec = array(); + $q = new PhabricatorElasticSearchQueryBuilder('bool'); + $query_string = $query->getParameter('query'); + if (strlen($query_string)) { + $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); - if (strlen($query->getParameter('query'))) { - $spec[] = array( + $q->must(array( 'simple_query_string' => array( - 'query' => $query->getParameter('query'), - 'fields' => array('field.corpus'), + 'query' => $query_string, + 'fields' => array( + 'title^4', + 'body^3', + 'cmnt^2', + 'tags', + '_all', + ), + 'default_operator' => 'and', ), - ); + )); - $title_spec = array( + $q->should(array( 'simple_query_string' => array( - 'query' => $query->getParameter('query'), - 'fields' => array('title'), + 'query' => $query_string, + 'fields' => array_values($fields), + 'analyzer' => 'english_exact', + 'default_operator' => 'and', ), - ); + )); + } $exclude = $query->getParameter('exclude'); if ($exclude) { - $filter[] = array( + $q->filter(array( 'not' => array( 'ids' => array( 'values' => array($exclude), ), ), - ); + )); } $relationship_map = array( @@ -176,67 +257,43 @@ $include_closed = !empty($statuses[$rel_closed]); if ($include_open && !$include_closed) { - $relationship_map[$rel_open] = true; + $q->exists($rel_open); } else if (!$include_open && $include_closed) { - $relationship_map[$rel_closed] = true; + $q->exists($rel_closed); } if ($query->getParameter('withUnowned')) { - $relationship_map[$rel_unowned] = true; + $q->exists($rel_unowned); } $rel_owner = PhabricatorSearchRelationship::RELATIONSHIP_OWNER; if ($query->getParameter('withAnyOwner')) { - $relationship_map[$rel_owner] = true; + $q->exists($rel_owner); } else { $owner_phids = $query->getParameter('ownerPHIDs', array()); - $relationship_map[$rel_owner] = $owner_phids; - } - - foreach ($relationship_map as $field => $param) { - if (is_array($param) && $param) { - $should = array(); - foreach ($param as $val) { - $should[] = array( - 'match' => array( - "relationship.{$field}.phid" => array( - 'query' => $val, - 'type' => 'phrase', - ), - ), - ); - } - // We couldn't solve it by minimum_number_should_match because it can - // match multiple owners without matching author. - $spec[] = array('bool' => array('should' => $should)); - } else if ($param) { - $filter[] = array( - 'exists' => array( - 'field' => "relationship.{$field}.phid", - ), - ); + if (count($owner_phids)) { + $q->terms($rel_owner, $owner_phids); } } - if ($spec) { - $spec = array('query' => array('bool' => array('must' => $spec))); - if ($title_spec) { - $spec['query']['bool']['should'] = $title_spec; + foreach ($relationship_map as $field => $phids) { + if (is_array($phids) && !empty($phids)) { + $q->terms($field, $phids); } } - if ($filter) { - $filter = array('filter' => array('and' => $filter)); - if (!$spec) { - $spec = array('query' => array('match_all' => new stdClass())); - } - $spec = array( - 'query' => array( - 'filtered' => $spec + $filter, - ), - ); + if (!$q->clauseCount('must')) { + $q->must(array('match_all' => array('boost' => 1 ))); } + $spec = array( + '_source' => false, + 'query' => array( + 'bool' => $q->toArray(), + ), + ); + + if (!$query->getParameter('query')) { $spec['sort'] = array( array('dateCreated' => 'desc'), @@ -244,8 +301,8 @@ } $spec['from'] = (int)$query->getParameter('offset', 0); - $spec['size'] = (int)$query->getParameter('limit', 25); - + $spec['size'] = min(10000, (int)$query->getParameter('limit', 25)); + // phlog(json_encode($spec)); return $spec; } @@ -261,22 +318,9 @@ // some bigger index). Use '/$types/_search' instead. $uri = '/'.implode(',', $types).'/_search'; - try { - $response = $this->executeRequest($uri, $this->buildSpec($query)); - } catch (HTTPFutureHTTPResponseStatus $ex) { - // elasticsearch probably uses Lucene query syntax: - // http://lucene.apache.org/core/3_6_1/queryparsersyntax.html - // Try literal search if operator search fails. - if (!strlen($query->getParameter('query'))) { - throw $ex; - } - $query = clone $query; - $query->setParameter( - 'query', - addcslashes( - $query->getParameter('query'), '+-&|!(){}[]^"~*?:\\')); - $response = $this->executeRequest($uri, $this->buildSpec($query)); - } + $spec = $this->buildSpec($query); + $response = $this->executeRequest($uri, $spec); + // phlog($response); $phids = ipull($response['hits']['hits'], '_id'); return $phids; @@ -284,7 +328,17 @@ public function indexExists() { try { - return (bool)$this->executeRequest('/_status/', array()); + + if ($this->version >= 5) { + $uri = '/_stats/'; + $res = $this->executeRequest($uri, array()); + return isset($res['indices']['phabricator']); + } else if ($this->version >= 2) { + $uri = ''; + } else { + $uri = '/_status/'; + } + return (bool)$this->executeRequest($uri, array()); } catch (HTTPFutureHTTPResponseStatus $e) { if ($e->getStatusCode() == 404) { return false; @@ -299,43 +353,63 @@ 'index' => array( 'auto_expand_replicas' => '0-2', 'analysis' => array( - 'filter' => array( - 'trigrams_filter' => array( - 'min_gram' => 3, - 'type' => 'ngram', - 'max_gram' => 3, - ), - ), 'analyzer' => array( - 'custom_trigrams' => array( - 'type' => 'custom', - 'filter' => array( - 'lowercase', - 'kstem', - 'trigrams_filter', - ), + 'english_exact' => array( 'tokenizer' => 'standard', + 'filter' => array('lowercase'), ), ), ), ), ); + $fields = $this->getTypeConstants('PhabricatorSearchDocumentFieldType'); + $relationships = $this->getTypeConstants('PhabricatorSearchRelationship'); + $types = array_keys( PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes()); + foreach ($types as $type) { - // Use the custom trigram analyzer for the corpus of text - $data['mappings'][$type]['properties']['field']['properties']['corpus'] = - array('type' => 'string', 'analyzer' => 'custom_trigrams'); + $properties = array(); + foreach ($fields as $field) { + // Use the custom analyzer for the corpus of text + $properties[$field] = array( + 'type' => $this->textFieldType, + 'analyzer' => 'english_exact', + 'search_analyzer' => 'english', + 'search_quote_analyzer' => 'english_exact', + ); + } + + foreach ($relationships as $rel) { + $properties[$rel] = array( + 'type' => $this->textFieldType, + ); + if ($this->version < 5) { + $properties[$rel]['index'] = 'not_analyzed'; + } + } // Ensure we have dateCreated since the default query requires it - $data['mappings'][$type]['properties']['dateCreated']['type'] = 'string'; - } + $properties['dateCreated']['type'] = 'date'; + // Replaces deprecated _timestamp for elasticsearch 2 + if ((int)$this->version >= 2) { + $properties['lastModified']['type'] = 'date'; + } + + $properties['tags'] = array( + 'type' => $this->textFieldType, + 'analyzer' => 'english', + 'store' => true, + ); + $data['mappings'][$type]['properties'] = $properties; + } return $data; } public function indexIsSane() { + if (!$this->indexExists()) { return false; } @@ -345,7 +419,8 @@ $actual = array_merge($cur_settings[$this->index], $cur_mapping[$this->index]); - return $this->check($actual, $this->getIndexConfiguration()); + $res = $this->check($actual, $this->getIndexConfiguration()); + return $res; } /** @@ -410,12 +485,20 @@ $this->executeRequest('/', $data, 'PUT'); } + public function didHealthCheck($reachable) { + static $cache=null; + if ($cache !== null) { + return; + } + + $cache = $reachable; + $this->ref->didHealthCheck($reachable); + return $this; + } + private function executeRequest($path, array $data, $method = 'GET') { - $uri = new PhutilURI($this->uri); - $uri->setPath($this->index); - $uri->appendPath($path); + $uri = $this->ref->getURI($path); $data = json_encode($data); - $future = new HTTPSFuture($uri, $data); if ($method != 'GET') { $future->setMethod($method); @@ -423,19 +506,30 @@ if ($this->getTimeout()) { $future->setTimeout($this->getTimeout()); } - list($body) = $future->resolvex(); + try { + list($body) = $future->resolvex(); + } catch (HTTPFutureResponseStatus $ex) { + if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) { + $this->didHealthCheck(false); + } + throw $ex; + } if ($method != 'GET') { return null; } try { - return phutil_json_decode($body); + $data = phutil_json_decode($body); + $this->didHealthCheck(true); + return $data; } catch (PhutilJSONParserException $ex) { + $this->didHealthCheck(false); throw new PhutilProxyException( pht('ElasticSearch server returned invalid JSON!'), $ex); } + } } diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php new file mode 100644 --- /dev/null +++ b/src/applications/search/fulltextstorage/PhabricatorElasticSearchQueryBuilder.php @@ -0,0 +1,78 @@ +clauses; + if ($termkey == null) { + return $clauses; + } + if (isset($clauses[$termkey])) { + return $clauses[$termkey]; + } + return array(); + } + + public function clauseCount($clausekey) { + if (isset($this->clauses[$clausekey])) { + return count($this->clauses[$clausekey]); + } else { + return 0; + } + } + + public function exists($field) { + return $this->addClause('filter', array( + 'exists' => array( + 'field' => $field, + ), + )); + } + + public function terms($field, $values) { + return $this->addClause('filter', array( + 'terms' => array( + $field => array_values($values), + ), + )); + } + + public function must($clause) { + return $this->addClause('must', $clause); + } + + public function filter($clause) { + return $this->addClause('filter', $clause); + } + + public function should($clause) { + return $this->addClause('should', $clause); + } + + public function mustNot($clause) { + return $this->addClause('must_not', $clause); + } + + public function addClause($clause, $terms) { + $this->clauses[$clause][] = $terms; + return $this; + } + + public function toArray() { + $clauses = $this->getClauses(); + return $clauses; + $cleaned = array(); + foreach ($clauses as $clause => $subclauses) { + if (is_array($subclauses) && count($subclauses) == 1) { + $cleaned[$clause] = array_shift($subclauses); + } else { + $cleaned[$clause] = $subclauses; + } + } + return $cleaned; + } + +} diff --git a/src/applications/search/index/PhabricatorFulltextEngine.php b/src/applications/search/index/PhabricatorFulltextEngine.php --- a/src/applications/search/index/PhabricatorFulltextEngine.php +++ b/src/applications/search/index/PhabricatorFulltextEngine.php @@ -40,8 +40,7 @@ $extension->indexFulltextObject($object, $document); } - $storage_engine = PhabricatorFulltextStorageEngine::loadEngine(); - $storage_engine->reindexAbstractDocument($document); + PhabricatorSearchCluster::reindexAbstractDocument($document); } protected function newAbstractDocument($object) { diff --git a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php --- a/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php +++ b/src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php @@ -13,27 +13,41 @@ public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - $work_done = false; - if (!$engine->indexExists()) { - $console->writeOut( - '%s', - pht('Index does not exist, creating...')); - $engine->initIndex(); - $console->writeOut( - "%s\n", - pht('done.')); - $work_done = true; - } else if (!$engine->indexIsSane()) { - $console->writeOut( - '%s', - pht('Index exists but is incorrect, fixing...')); - $engine->initIndex(); + foreach (PhabricatorSearchCluster::getAllServices() as $service) { $console->writeOut( "%s\n", - pht('done.')); - $work_done = true; + pht('Initializing search service "%s"', $service->getDisplayName())); + + try { + $host = $service->getHostForRole('write'); + } catch (PhabricatorClusterNoHostForRoleException $e) { + // If there are no writable hosts for a given cluster, skip it + $console->writeOut("%s\n", $e->getExceptionTitle()); + continue; + } + + $engine = $host->getEngine(); + + if (!$engine->indexExists()) { + $console->writeOut( + '%s', + pht('Index does not exist, creating...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } else if (!$engine->indexIsSane()) { + $console->writeOut( + '%s', + pht('Index exists but is incorrect, fixing...')); + $engine->initIndex(); + $console->writeOut( + "%s\n", + pht('done.')); + $work_done = true; + } } if ($work_done) { diff --git a/src/applications/search/query/PhabricatorSearchDocumentQuery.php b/src/applications/search/query/PhabricatorSearchDocumentQuery.php --- a/src/applications/search/query/PhabricatorSearchDocumentQuery.php +++ b/src/applications/search/query/PhabricatorSearchDocumentQuery.php @@ -73,10 +73,7 @@ $query = id(clone($this->savedQuery)) ->setParameter('offset', $this->getOffset()) ->setParameter('limit', $this->getRawResultLimit()); - - $engine = PhabricatorFulltextStorageEngine::loadEngine(); - - return $engine->executeSearch($query); + return PhabricatorSearchCluster::executeSearch($query); } public function getQueryApplicationClass() { diff --git a/src/applications/search/worker/PhabricatorSearchWorker.php b/src/applications/search/worker/PhabricatorSearchWorker.php --- a/src/applications/search/worker/PhabricatorSearchWorker.php +++ b/src/applications/search/worker/PhabricatorSearchWorker.php @@ -51,6 +51,7 @@ $lock->unlock(); if (!($ex instanceof PhabricatorWorkerPermanentFailureException)) { + phlog($ex); $ex = new PhabricatorWorkerPermanentFailureException( pht( 'Failed to update search index for document "%s": %s', diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php rename from src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php rename to src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php --- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php +++ b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php @@ -1,20 +1,19 @@ ref = $ref; + public function __construct($cache_key) { + $this->cacheKey = $cache_key; $this->readState(); } - /** * Is the database currently healthy? */ @@ -153,18 +152,13 @@ } } - private function getHealthRecordCacheKey() { - $ref = $this->ref; - - $host = $ref->getHost(); - $port = $ref->getPort(); - - return "cluster.db.health({$host}, {$port})"; + public function getCacheKey() { + return $this->cacheKey; } private function readHealthRecord() { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $health_record = $cache->getKey($cache_key); if (!is_array($health_record)) { @@ -180,7 +174,7 @@ private function writeHealthRecord(array $record) { $cache = PhabricatorCaches::getSetupCache(); - $cache_key = $this->getHealthRecordCacheKey(); + $cache_key = $this->getCacheKey(); $cache->setKey($cache_key, $record); } diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -14,6 +14,7 @@ const REPLICATION_SLOW = 'replica-slow'; const REPLICATION_NOT_REPLICATING = 'not-replicating'; + const KEY_HEALTH = 'cluster.db.health'; const KEY_REFS = 'cluster.db.refs'; const KEY_INDIVIDUAL = 'cluster.db.individual'; @@ -489,9 +490,18 @@ return $this; } + private function getHealthRecordCacheKey() { + $host = $this->getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + public function getHealthRecord() { if (!$this->healthRecord) { - $this->healthRecord = new PhabricatorDatabaseHealthRecord($this); + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); } return $this->healthRecord; } diff --git a/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php b/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorElasticSearchCluster.php @@ -0,0 +1,93 @@ +setRoles(idx($config, 'roles', + array('read' => true, 'write' => true))) + ->setHostRefs(idx($config, 'hosts', array())) + ->setPort(idx($config, 'port', $this->port)) + ->setProtocol(idx($config, 'protocol', $this->protocol)) + ->setPath(idx($config, 'path', $this->path)) + ->setVersion(idx($config, 'version', $this->version)); + return $this; + } + + public function getDisplayName() { + return pht('ElasticSearch'); + } + + public function getEngineIdentifier() { + return 'elasticsearch'; + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setPath($path) { + $this->path = $path; + return $this; + } + + public function getPath() { + return $this->path; + } + + public function setVersion($version) { + $this->version = $version; + return $this; + } + + public function getVersion() { + return $this->version; + } + + public function getURI($to_path = null) { + $uri = id(new PhutilURI('http://'.$this->getHost())) + ->setProtocol($this->getProtocol()) + ->setPort($this->getPort()) + ->setPath($this->getPath()); + + if ($to_path) { + $uri->appendPath($to_path); + } + return $uri; + } + + /** + * @return PhabricatorElasticFulltextStorageEngine + */ + public function getEngine() { + if (!$this->engine) { + $engine = new PhabricatorElasticFulltextStorageEngine(); + $this->engine = $engine->setRef($this); + } + return $this->engine; + } + + public function loadServerStatus() { + $status = $this->getEngine()->indexIsSane() + ? 'okay' + : 'fail'; + $engine = $this->getEngine(); + $status = $engine->indexIsSane() + ? 'okay' + : 'fail'; + return $status; + } + +} diff --git a/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php b/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorMySQLSearchCluster.php @@ -0,0 +1,41 @@ +engine = new PhabricatorMySQLFulltextStorageEngine(); + } + + public function setConfig($config) { + $this->setRoles(idx($config, 'roles', + array('read' => true, 'write' => true))); + return $this; + } + + public function getDisplayName() { + return 'MySQL'; + } + + public function getEngineIdentifier() { + return 'mysql'; + } + + public function getEngine() { + return $this->engine; + } + + public function getProtocol() { + return 'mysql'; + } + + public function loadServerStatus() { + PhabricatorDatabaseRef::queryAll(); + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); + $status = $ref->getConnectionStatus(); + return $status; + } + +} diff --git a/src/infrastructure/cluster/PhabricatorSearchCluster.php b/src/infrastructure/cluster/PhabricatorSearchCluster.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorSearchCluster.php @@ -0,0 +1,253 @@ +disabled = $disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public static function getConnectionStatusMap() { + return array( + self::STATUS_OKAY => array( + 'icon' => 'fa-exchange', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::STATUS_FAIL => array( + 'icon' => 'fa-times', + 'color' => 'red', + 'label' => pht('Failed'), + ), + ); + } + + public function isWritable() { + return $this->hasRole('write'); + } + + public function isReadable() { + return $this->hasRole('read'); + } + + public function hasRole($role) { + return isset($this->roles[$role]) && $this->roles[$role] === true; + } + + public function setRoles(array $roles) { + foreach ($roles as $role => $val) { + $this->roles[$role] = $val; + } + return $this; + } + + public function getRoles() { + return $this->roles; + } + + public function setPort($value) { + $this->port = $value; + return $this; + } + + public function getPort() { + return $this->port; + } + + public function setHost($value) { + $this->host = $value; + return $this; + } + + public function getHost() { + return $this->host; + } + + public function setHostRefs(array $refs) { + $this->hostRefs = $refs; + return $this; + } + + public function getHostRefs() { + return $this->hostRefs; + } + + /** @return PhabricatorSearchCluster */ + public function getHostForRole($role) { + $hosts = $this->getAllHostsForRole($role); + if (empty($hosts)) { + throw new PhabricatorClusterNoHostForRoleException($role); + } + $random = array_rand($hosts); + return $hosts[$random]; + } + + public function getAllHostsForRole($role) { + $hosts = array(); + foreach ($this->hostRefs as $host) { + if ($host->hasRole($role)) { + $hosts[] = $host; + } + } + return $hosts; + } + + public function getHealthRecordCacheKey() { + $host = $this->getHost(); + $port = $this->getPort(); + $key = self::KEY_HEALTH; + + return "{$key}({$host}, {$port})"; + } + +/** + * @return PhabricatorClusterServiceHealthRecord + */ + public function getHealthRecord() { + if (!$this->healthRecord) { + $this->healthRecord = new PhabricatorClusterServiceHealthRecord( + $this->getHealthRecordCacheKey()); + } + return $this->healthRecord; + } + + public function didHealthCheck($reachable) { + $record = $this->getHealthRecord(); + $should_check = $record->getShouldCheck(); + + if ($should_check) { + $record->didHealthCheck($reachable); + } + } + + /** @return PhabricatorSearchCluster[] */ + public static function getAllServices() { + $cache = PhabricatorCaches::getRequestCache(); + + $refs = $cache->getKey(self::KEY_REFS); + if (!$refs) { + $refs = self::newRefs(); + $cache->setKey(self::KEY_REFS, $refs); + } + + return $refs; + } + + /** find one random writable host from each service. + * @return PhabricatorSearchCluster[] writable cluster hosts + */ + public static function getWritableHostForEachService() { + $services = self::getAllServices(); + $all_writable = array(); + foreach ($services as $service) { + $hosts = $service->getAllHostsForRole('write'); + if (count($hosts)) { + $index = array_rand($hosts); + $all_writable[] = $hosts[$index]; + } + } + return $all_writable; + } + + + /** + * @return PhabricatorSearchCluster[] + */ + public static function newRefs() { + $services = PhabricatorEnv::getEnvConfig('cluster.search'); + $types = id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getEngineIdentifier') + ->execute(); + $refs = array(); + + foreach ($services as $config) { + if (!isset($types[$config['type']])) { + throw new Exception(pht('Configured search server type is invalid: %s', + $config['type'])); + } + $type = $types[$config['type']]; + $service = clone($type); + $service->setConfig($config); + if (isset($config['hosts']) && count($config['hosts'])) { + $hosts = array(); + foreach ($config['hosts'] as $host) { + $hostref = clone($service); + $hostref->setConfig($host); + $hostref->setPort(idx($host, 'port', $service->getPort())) + ->setHost(idx($host, 'host')) + ->setRoles(idx($host, 'roles', $service->getRoles())); + $hosts[] = $hostref; + } + $service->setHostRefs($hosts); + } + $refs[] = $service; + } + + return $refs; + } + + /** + * @return PhabricatorFulltextStorageEngine + */ + abstract public function getEngine(); + + abstract public function loadServerStatus(); + + public static function reindexAbstractDocument( + PhabricatorSearchAbstractDocument $doc) { + + $services = self::getAllServices(); + $indexed = 0; + foreach (self::getWritableHostForEachService() as $host) { + $host->getEngine()->reindexAbstractDocument($doc); + $indexed++; + } + if ($indexed == 0) { + throw new PhabricatorClusterNoHostForRoleException('write'); + } + } + + public static function executeSearch(PhabricatorSavedQuery $query) { + $services = self::getAllServices(); + foreach ($services as $service) { + $hosts = $service->getAllHostsForRole('read'); + // try all hosts until one succeeds + foreach ($hosts as $host) { + $last_exception = null; + try { + $res = $host->getEngine()->executeSearch($query); + // return immediately if we get results without an exception + return $res; + } catch (Exception $ex) { + // try each server in turn, only throw if none succeed + $last_exception = $ex; + } + } + } + if ($last_exception) { + throw $last_exception; + } + return $res; + } + +} diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php rename from src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php rename to src/infrastructure/cluster/config/PhabricatorClusterDatabasesConfigOptionType.php diff --git a/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/config/PhabricatorClusterSearchConfigOptionType.php @@ -0,0 +1,54 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Search cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a search service, but '. + 'the value with index "%s" is not a dictionary.', + $index)); + + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'type' => 'string', + 'hosts' => 'optional list>', + 'roles' => 'optional map', + 'port' => 'optional int', + 'protocol' => 'optional string', + 'path' => 'optional string', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search cluster configuration has an invalid service '. + 'specification (at index "%s"): %s.', + $index, + $ex->getMessage())); + } + if (isset($spec['hosts']) && !is_array($spec['hosts'])) { + throw new Exception( + pht( + 'Search cluster configuration has invalid "hosts" specification '. + 'for the service at index "%s": Hosts must be a list with at '. + 'least one dictionary describing a host for this service.'), + $index); + } + } + } + } +} diff --git a/src/infrastructure/cluster/PhabricatorClusterException.php b/src/infrastructure/cluster/exception/PhabricatorClusterException.php rename from src/infrastructure/cluster/PhabricatorClusterException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterException.php diff --git a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php b/src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php rename from src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php rename to src/infrastructure/cluster/exception/PhabricatorClusterExceptionHandler.php diff --git a/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php rename from src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterImpossibleWriteException.php diff --git a/src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php b/src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php rename from src/infrastructure/cluster/PhabricatorClusterImproperWriteException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterImproperWriteException.php diff --git a/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/exception/PhabricatorClusterNoHostForRoleException.php @@ -0,0 +1,16 @@ +role = $role; + } + + public function getExceptionTitle() { + pht('Search cluster has no hosts for role "%s"', $this->role); + } + +} diff --git a/src/infrastructure/cluster/PhabricatorClusterStrandedException.php b/src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php rename from src/infrastructure/cluster/PhabricatorClusterStrandedException.php rename to src/infrastructure/cluster/exception/PhabricatorClusterStrandedException.php