diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2255,6 +2255,9 @@ 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php', 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php', 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterRef' => 'infrastructure/cluster/PhabricatorClusterRef.php', + 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php', + 'PhabricatorClusterSearchRef' => 'infrastructure/cluster/PhabricatorClusterSearchRef.php', 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', @@ -2301,6 +2304,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php', 'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php', 'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php', + 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php', 'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php', 'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php', 'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php', @@ -2630,6 +2634,7 @@ 'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php', 'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php', 'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php', + 'PhabricatorElasticSearchServerRef' => 'infrastructure/cluster/PhabricatorElasticSearchServerRef.php', 'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php', 'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php', 'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php', @@ -3051,6 +3056,7 @@ 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', 'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php', 'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php', + 'PhabricatorMysqlSearchServerRef' => 'infrastructure/cluster/PhabricatorMysqlSearchServerRef.php', 'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php', 'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php', 'PhabricatorNavigationRemarkupRule' => 'infrastructure/markup/rule/PhabricatorNavigationRemarkupRule.php', @@ -7248,6 +7254,9 @@ 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterRef' => 'Phobject', + 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType', + 'PhabricatorClusterSearchRef' => 'PhabricatorClusterRef', 'PhabricatorClusterStrandedException' => 'PhabricatorClusterException', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', @@ -7299,6 +7308,7 @@ 'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController', 'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController', + 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController', 'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule', 'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema', 'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType', @@ -7568,7 +7578,7 @@ 'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec', 'PhabricatorDataNotAttachedException' => 'Exception', 'PhabricatorDatabaseHealthRecord' => 'Phobject', - 'PhabricatorDatabaseRef' => 'Phobject', + 'PhabricatorDatabaseRef' => 'PhabricatorClusterRef', 'PhabricatorDatabaseRefParser' => 'Phobject', 'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorDatasourceEditField' => 'PhabricatorTokenizerEditField', @@ -7670,6 +7680,7 @@ 'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting', 'PhabricatorEditorSetting' => 'PhabricatorStringSetting', 'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', + 'PhabricatorElasticSearchServerRef' => 'PhabricatorClusterSearchRef', 'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorEmailContentSource' => 'PhabricatorContentSource', @@ -8139,6 +8150,7 @@ 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine', 'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck', + 'PhabricatorMysqlSearchServerRef' => 'PhabricatorClusterSearchRef', 'PhabricatorNamedQuery' => array( 'PhabricatorSearchDAO', 'PhabricatorPolicyInterface', diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php --- a/src/applications/config/application/PhabricatorConfigApplication.php +++ b/src/applications/config/application/PhabricatorConfigApplication.php @@ -69,6 +69,7 @@ 'databases/' => 'PhabricatorConfigClusterDatabasesController', 'notifications/' => 'PhabricatorConfigClusterNotificationsController', 'repositories/' => 'PhabricatorConfigClusterRepositoriesController', + 'search/' => 'PhabricatorConfigClusterSearchController', ), ), ); diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php --- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php +++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php @@ -70,8 +70,14 @@ } protected function shouldUseElasticSearchEngine() { - $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); - return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine); + $engines = PhabricatorFulltextStorageEngine::loadAllEngines(); + foreach ($engines as $engine) { + if ($engine instanceof PhabricatorElasticFulltextStorageEngine + && $engine->isEnabled()) { + return true; + } + } + return false; } } diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php new file mode 100644 --- /dev/null +++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php @@ -0,0 +1,143 @@ +buildSideNavView(); + $nav->selectFilter('cluster/search/'); + + $title = pht('Cluster Search'); + $doc_href = PhabricatorEnv::getDoclink('Cluster: Search'); + + $header = id(new PHUIHeaderView()) + ->setHeader($title) + ->setProfileHeader(true) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + $crumbs = $this + ->buildApplicationCrumbs($nav) + ->addTextCrumb($title) + ->setBorder(true); + + $search_status = $this->buildClusterSearchStatus(); + + $content = id(new PhabricatorConfigPageView()) + ->setHeader($header) + ->setContent($search_status); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->setNavigation($nav) + ->appendChild($content) + ->addClass('white-background'); + } + + private function buildClusterSearchStatus() { + $viewer = $this->getViewer(); + + $servers = PhabricatorClusterSearchRef::newRefs(); + Javelin::initBehavior('phabricator-tooltips'); + $status_map = PhabricatorDatabaseRef::getConnectionStatusMap(); + + $rows = array(); + foreach ($servers as $server) { + + try { + $status = $server->loadServerStatus(); + $status = idx($status_map, $status, array()); + } catch (Exception $ex) { + $status['icon'] = 'fa-times'; + $status['label'] = pht('Connection Error'); + $status['color'] = 'red'; + } + + $engine = $server->getEngine(); + + $type_icon = 'fa-search sky'; + $type_tip = $server->getDisplayName(); + + $type_icon = id(new PHUIIconView()) + ->setIcon($type_icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $type_tip, + )); + + $status_view = array( + id(new PHUIIconView())->setIcon("{$status['icon']} {$status['color']}"), + ' ', + $status['label'], + ); + + $rows[] = array( + array($type_icon, ' '.$type_tip), + $server->getProtocol(), + $server->getHost(), + $server->getPort(), + $status_view, + get_class($engine), + $this->checkicon($engine->isEnabled()), + $this->checkicon($server->isWritable()), + $engine->getEnginePriority(), + ); + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString( + pht('No search servers are configured.')) + ->setHeaders( + array( + pht('Type'), + pht('Protocol'), + pht('Host'), + pht('Port'), + pht('Status'), + pht('Engine'), + pht('Enabled'), + pht('Writable'), + pht('Priority'), + null, + )) + ->setColumnClasses( + array( + null, + null, + null, + null, + null, + null, + null, + null, + null, + 'wide', + )); + + return $table; + } + + private function checkicon($check) { + $icon = $check + ? 'fa-check green' + : 'fa-times red'; + $label = $check + ? pht('Yes') + : pht('No'); + $view = id(new PHUIIconView()) + ->setIcon($icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $label, + )); + + return $view; + } +} diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php --- a/src/applications/config/controller/PhabricatorConfigController.php +++ b/src/applications/config/controller/PhabricatorConfigController.php @@ -42,8 +42,11 @@ pht('Notification Servers'), null, 'fa-bell-o'); $nav->addFilter('cluster/repositories/', pht('Repository Servers'), null, 'fa-code'); + $nav->addFilter('cluster/search/', + pht('Search Servers'), null, 'fa-search'); $nav->addLabel(pht('Modules')); + $modules = PhabricatorConfigModule::getAllModules(); foreach ($modules as $key => $module) { $nav->addFilter('module/'.$key.'/', diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php --- a/src/applications/search/config/PhabricatorSearchConfigOptions.php +++ b/src/applications/search/config/PhabricatorSearchConfigOptions.php @@ -20,7 +20,15 @@ } public function getOptions() { + $servers_type = 'custom:PhabricatorClusterSearchConfigOptionType'; + $servers_help = 'TODO'; + return array( + $this->newOption('search.servers', $servers_type, array()) + ->setLocked(true) + ->setSummary( + pht('Configure full-text search servers.')) + ->setDescription($servers_help), $this->newOption('search.elastic.host', 'string', null) ->setLocked(true) ->setDescription(pht('Elastic Search host.')) diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php --- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php @@ -3,13 +3,25 @@ final class PhabricatorElasticFulltextStorageEngine extends PhabricatorFulltextStorageEngine { - private $uri; + private $ref; private $index; + private $uri; private $timeout; - public function __construct() { - $this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host'); - $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace'); + public function setRef(PhabricatorElasticSearchServerRef $ref) { + $this->uri = (string)$ref->getURI(); + $this->index = str_replace('/', '', $ref->getPath()); + $this->version = (int)$ref->getVersion(); + + $this->timestampFieldKey = $this->version < 2 + ? '_timestamp' + : 'lastModified'; + + $this->textFieldType = $this->version >= 5 + ? 'text' + : 'string'; + + $this->enabled = true; } public function getEngineIdentifier() { @@ -336,6 +348,7 @@ } public function indexIsSane() { + if (!$this->indexExists()) { return false; } diff --git a/src/infrastructure/cluster/PhabricatorClusterRef.php b/src/infrastructure/cluster/PhabricatorClusterRef.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorClusterRef.php @@ -0,0 +1,69 @@ +disabled = $is_disabled; + return $this; + } + + public function getDisabled() { + return $this->disabled; + } + + public function setHost($host) { + $this->host = $host; + return $this; + } + + public function getHost() { + return $this->host; + } + + public function setPort($port) { + $this->port = $port; + return $this; + } + + public function getPort() { + return $this->port; + } + + public static function getConnectionStatusMap() { + return array( + self::STATUS_OKAY => array( + 'icon' => 'fa-exchange', + 'color' => 'green', + 'label' => pht('Okay'), + ), + self::STATUS_FAIL => array( + 'icon' => 'fa-times', + 'color' => 'red', + 'label' => pht('Failed'), + ), + self::STATUS_AUTH => array( + 'icon' => 'fa-key', + 'color' => 'red', + 'label' => pht('Invalid Credentials'), + ), + self::STATUS_REPLICATION_CLIENT => array( + 'icon' => 'fa-eye-slash', + 'color' => 'yellow', + 'label' => pht('Missing Permission'), + ), + ); + } + +} diff --git a/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php @@ -0,0 +1,49 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Search cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a search host, but '. + 'the value with index "%s" is not a dictionary.', + $index)); + } + } + + $map = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'type' => 'string', + 'host' => 'string', + 'port' => 'optional int', + 'path' => 'optional string', + 'protocol' => 'optional string', + 'disabled' => 'optional bool', + 'version' => 'optional int', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Search cluster configuration has an invalid host '. + 'specification (at index "%s"): %s.', + $index, + $ex->getMessage())); + } + } + } +} diff --git a/src/infrastructure/cluster/PhabricatorClusterSearchRef.php b/src/infrastructure/cluster/PhabricatorClusterSearchRef.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorClusterSearchRef.php @@ -0,0 +1,91 @@ +writable; + } + public function setWritable($value) { + $this->writable = $value; + return $this; + } + + public static function getLiveServers() { + $cache = PhabricatorCaches::getRequestCache(); + + $refs = $cache->getKey(self::KEY_REFS); + if (!$refs) { + $refs = self::newRefs(); + $cache->setKey(self::KEY_REFS, $refs); + } + + return $refs; + } + + public static function newRefs() { + $builtin_ref = new PhabricatorMysqlSearchServerRef(); + $refs = array($builtin_ref); + // try to load custom fulltext search service configuration. + $configs = PhabricatorEnv::getEnvConfigIfExists('search.servers'); + if (!$configs) { + // if nothing is configured, just return a reference to the built-in + // fulltext search. + return $refs; + } + + $types = id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getEngineIdentifier') + ->execute(); + + foreach ($configs as $config) { + if (!isset($types[$config['type']])) { + throw new Exception(pht('configured search server type is invalid %s', + $config['type'])); + } + $type = $types[$config['type']]; + if ($config['type'] == 'mysql') { + // if there is a config for mysql, then it overrides the built-in ref + $ref = $builtin_ref; + } else { + // otherwise it's a custom type, clone the prototype ref + $ref = @id(clone($type)); + $refs[] = $ref; + } + + foreach ($config as $key => $val) { + $setter = 'set'.ucfirst($key); + if (method_exists($ref, $setter)) { + call_user_func(array($ref, $setter), $val); + } + } + $ref->initEngine(); + } + + return $refs; + } + + public static function getEnabledServers() { + $servers = self::getLiveServers(); + + foreach ($servers as $key => $server) { + if ($server->getDisabled()) { + unset($servers[$key]); + } + } + + return array_values($servers); + } + + public function initEngine() {} + + abstract public function getEngine(); + abstract public function loadServerStatus(); + +} diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php --- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php @@ -1,6 +1,6 @@ getHost(); $port = $ref->getPort(); + $key = $ref::KEY_HEALTH; - return "cluster.db.health({$host}, {$port})"; + return "{$key}({$host}, {$port})"; } private function readHealthRecord() { diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -1,7 +1,7 @@ array( - 'icon' => 'fa-exchange', - 'color' => 'green', - 'label' => pht('Okay'), - ), - self::STATUS_FAIL => array( - 'icon' => 'fa-times', - 'color' => 'red', - 'label' => pht('Failed'), - ), + $map = parent::getConnectionStatusMap(); + return $map + array( self::STATUS_AUTH => array( 'icon' => 'fa-key', 'color' => 'red', diff --git a/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php b/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php @@ -0,0 +1,87 @@ +engine = new PhabricatorElasticFulltextStorageEngine(); + } + + public function getDisplayName() { + return 'ElasticSearch'; + } + + public function setProtocol($protocol) { + $this->protocol = $protocol; + return $this; + } + + public function getProtocol() { + return $this->protocol; + } + + public function setPath($path) { + $this->path = $path; + return $this; + } + + public function getPath() { + return $this->path; + } + + public function setVersion($version) { + $this->version = $version; + return $this; + } + + public function getVersion() { + return $this->version; + } + + public function getPort() { + return $this->port; + } + public function setPort($value) { + $this->port = $value; + return $this; + } + + public function getURI($to_path = null) { + $full_path = rtrim($this->getPath(), '/').'/'.ltrim($to_path, '/'); + + $uri = id(new PhutilURI('http://'.$this->getHost())) + ->setProtocol($this->getProtocol()) + ->setPort($this->getPort()) + ->setPath($full_path); + + return $uri; + } + + public function initEngine() { + $this->engine->setRef($this); + } + + public function getEngineIdentifier() { + return $this->engine->getEngineIdentifier(); + } + + public function getEngine() { + return $this->engine; + } + + public function loadServerStatus() { + $status = $this->getEngine()->indexIsSane() + ? 'okay' + : 'fail'; + return $status; + } + +} diff --git a/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php b/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php @@ -0,0 +1,35 @@ +engine = new PhabricatorMySQLFulltextStorageEngine(); + } + + public function getDisplayName() { + return 'MySQL'; + } + + public function getEngineIdentifier() { + return $this->engine->getEngineIdentifier(); + } + + public function getEngine() { + return $this->engine; + } + + public function getProtocol() { + return 'mysql'; + } + + public function loadServerStatus() { + PhabricatorDatabaseRef::queryAll(); + $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search'); + $status = $ref->getConnectionStatus(); + return $status; + } + +}