Page MenuHomePhabricator
Files F17908698

D17384.id41856.diff
No OneTemporary
Actions

D17384.id41856.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2255,6 +2255,11 @@
'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php',
'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php',
'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php',
+ 'PhabricatorClusterRef' => 'infrastructure/cluster/PhabricatorClusterRef.php',
+ 'PhabricatorClusterSearchConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php',
+ 'PhabricatorClusterSearchRef' => 'infrastructure/cluster/PhabricatorClusterSearchRef.php',
+ 'PhabricatorClusterServiceHealthRecord' => 'infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php',
+ 'PhabricatorClusterServiceInterface' => 'infrastructure/cluster/PhabricatorClusterServiceInterface.php',
'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php',
'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
@@ -2301,6 +2306,7 @@
'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
'PhabricatorConfigClusterNotificationsController' => 'applications/config/controller/PhabricatorConfigClusterNotificationsController.php',
'PhabricatorConfigClusterRepositoriesController' => 'applications/config/controller/PhabricatorConfigClusterRepositoriesController.php',
+ 'PhabricatorConfigClusterSearchController' => 'applications/config/controller/PhabricatorConfigClusterSearchController.php',
'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
@@ -2532,7 +2538,6 @@
'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
- 'PhabricatorDatabaseHealthRecord' => 'infrastructure/cluster/PhabricatorDatabaseHealthRecord.php',
'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
'PhabricatorDatabaseRefParser' => 'infrastructure/cluster/PhabricatorDatabaseRefParser.php',
'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
@@ -2630,6 +2635,7 @@
'PhabricatorEditorMultipleSetting' => 'applications/settings/setting/PhabricatorEditorMultipleSetting.php',
'PhabricatorEditorSetting' => 'applications/settings/setting/PhabricatorEditorSetting.php',
'PhabricatorElasticFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php',
+ 'PhabricatorElasticSearchServerRef' => 'infrastructure/cluster/PhabricatorElasticSearchServerRef.php',
'PhabricatorElasticSearchSetupCheck' => 'applications/config/check/PhabricatorElasticSearchSetupCheck.php',
'PhabricatorEmailAddressesSettingsPanel' => 'applications/settings/panel/PhabricatorEmailAddressesSettingsPanel.php',
'PhabricatorEmailContentSource' => 'applications/metamta/contentsource/PhabricatorEmailContentSource.php',
@@ -3051,6 +3057,7 @@
'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php',
'PhabricatorMySQLFulltextStorageEngine' => 'applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php',
'PhabricatorMySQLSetupCheck' => 'applications/config/check/PhabricatorMySQLSetupCheck.php',
+ 'PhabricatorMysqlSearchServerRef' => 'infrastructure/cluster/PhabricatorMysqlSearchServerRef.php',
'PhabricatorNamedQuery' => 'applications/search/storage/PhabricatorNamedQuery.php',
'PhabricatorNamedQueryQuery' => 'applications/search/query/PhabricatorNamedQueryQuery.php',
'PhabricatorNavigationRemarkupRule' => 'infrastructure/markup/rule/PhabricatorNavigationRemarkupRule.php',
@@ -7248,6 +7255,10 @@
'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler',
'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException',
'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException',
+ 'PhabricatorClusterRef' => 'Phobject',
+ 'PhabricatorClusterSearchConfigOptionType' => 'PhabricatorConfigJSONOptionType',
+ 'PhabricatorClusterSearchRef' => 'PhabricatorClusterRef',
+ 'PhabricatorClusterServiceHealthRecord' => 'Phobject',
'PhabricatorClusterStrandedException' => 'PhabricatorClusterException',
'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
@@ -7299,6 +7310,7 @@
'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterNotificationsController' => 'PhabricatorConfigController',
'PhabricatorConfigClusterRepositoriesController' => 'PhabricatorConfigController',
+ 'PhabricatorConfigClusterSearchController' => 'PhabricatorConfigController',
'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
@@ -7567,7 +7579,6 @@
'PhabricatorDashboardViewController' => 'PhabricatorDashboardProfileController',
'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
'PhabricatorDataNotAttachedException' => 'Exception',
- 'PhabricatorDatabaseHealthRecord' => 'Phobject',
'PhabricatorDatabaseRef' => 'Phobject',
'PhabricatorDatabaseRefParser' => 'Phobject',
'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
@@ -7670,6 +7681,7 @@
'PhabricatorEditorMultipleSetting' => 'PhabricatorSelectSetting',
'PhabricatorEditorSetting' => 'PhabricatorStringSetting',
'PhabricatorElasticFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
+ 'PhabricatorElasticSearchServerRef' => 'PhabricatorClusterSearchRef',
'PhabricatorElasticSearchSetupCheck' => 'PhabricatorSetupCheck',
'PhabricatorEmailAddressesSettingsPanel' => 'PhabricatorSettingsPanel',
'PhabricatorEmailContentSource' => 'PhabricatorContentSource',
@@ -8139,6 +8151,7 @@
'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorMySQLFulltextStorageEngine' => 'PhabricatorFulltextStorageEngine',
'PhabricatorMySQLSetupCheck' => 'PhabricatorSetupCheck',
+ 'PhabricatorMysqlSearchServerRef' => 'PhabricatorClusterSearchRef',
'PhabricatorNamedQuery' => array(
'PhabricatorSearchDAO',
'PhabricatorPolicyInterface',
diff --git a/src/applications/config/application/PhabricatorConfigApplication.php b/src/applications/config/application/PhabricatorConfigApplication.php
--- a/src/applications/config/application/PhabricatorConfigApplication.php
+++ b/src/applications/config/application/PhabricatorConfigApplication.php
@@ -69,6 +69,7 @@
'databases/' => 'PhabricatorConfigClusterDatabasesController',
'notifications/' => 'PhabricatorConfigClusterNotificationsController',
'repositories/' => 'PhabricatorConfigClusterRepositoriesController',
+ 'search/' => 'PhabricatorConfigClusterSearchController',
),
),
);
diff --git a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
--- a/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
+++ b/src/applications/config/check/PhabricatorElasticSearchSetupCheck.php
@@ -70,8 +70,14 @@
}
protected function shouldUseElasticSearchEngine() {
- $search_engine = PhabricatorFulltextStorageEngine::loadEngine();
- return ($search_engine instanceof PhabricatorElasticFulltextStorageEngine);
+ $engines = PhabricatorFulltextStorageEngine::loadAllEngines();
+ foreach ($engines as $engine) {
+ if ($engine instanceof PhabricatorElasticFulltextStorageEngine
+ && $engine->isEnabled()) {
+ return true;
+ }
+ }
+ return false;
}
}
diff --git a/src/applications/config/controller/PhabricatorConfigClusterSearchController.php b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
new file mode 100644
--- /dev/null
+++ b/src/applications/config/controller/PhabricatorConfigClusterSearchController.php
@@ -0,0 +1,152 @@
+<?php
+
+final class PhabricatorConfigClusterSearchController
+ extends PhabricatorConfigController {
+
+ public function handleRequest(AphrontRequest $request) {
+ $nav = $this->buildSideNavView();
+ $nav->selectFilter('cluster/search/');
+
+ $title = pht('Cluster Search');
+ $doc_href = PhabricatorEnv::getDoclink('Cluster: Search');
+
+ $header = id(new PHUIHeaderView())
+ ->setHeader($title)
+ ->setProfileHeader(true)
+ ->addActionLink(
+ id(new PHUIButtonView())
+ ->setIcon('fa-book')
+ ->setHref($doc_href)
+ ->setTag('a')
+ ->setText(pht('Documentation')));
+
+ $crumbs = $this
+ ->buildApplicationCrumbs($nav)
+ ->addTextCrumb($title)
+ ->setBorder(true);
+
+ $search_status = $this->buildClusterSearchStatus();
+
+ $content = id(new PhabricatorConfigPageView())
+ ->setHeader($header)
+ ->setContent($search_status);
+
+ return $this->newPage()
+ ->setTitle($title)
+ ->setCrumbs($crumbs)
+ ->setNavigation($nav)
+ ->appendChild($content)
+ ->addClass('white-background');
+ }
+
+ private function buildClusterSearchStatus() {
+ $viewer = $this->getViewer();
+
+ $servers = PhabricatorClusterSearchRef::newRefs();
+ Javelin::initBehavior('phabricator-tooltips');
+ $status_map = PhabricatorDatabaseRef::getConnectionStatusMap();
+
+ $rows = array();
+ foreach ($servers as $server) {
+
+ $reachable = false;
+ try {
+ $engine = $server->getEngine();
+ $reachable = $engine->indexExists();
+ } catch (Exception $ex) {
+ phlog($ex);
+ // ignore
+ }
+ $server->didHealthCheck($reachable);
+
+ try {
+ $status = $server->loadServerStatus();
+ $status = idx($status_map, $status, array());
+ } catch (Exception $ex) {
+ $status['icon'] = 'fa-times';
+ $status['label'] = pht('Connection Error');
+ $status['color'] = 'red';
+ }
+
+ $type_icon = 'fa-search sky';
+ $type_tip = $server->getDisplayName();
+
+ $type_icon = id(new PHUIIconView())
+ ->setIcon($type_icon)
+ ->addSigil('has-tooltip')
+ ->setMetadata(
+ array(
+ 'tip' => $type_tip,
+ ));
+
+
+ $status_view = array(
+ id(new PHUIIconView())->setIcon("{$status['icon']} {$status['color']}"),
+ ' ',
+ $status['label'],
+ );
+
+ $rows[] = array(
+ array($type_icon, ' '.$type_tip),
+ $server->getProtocol(),
+ $server->getHost(),
+ $server->getPort(),
+ $status_view,
+ get_class($engine),
+ $this->checkicon($engine->isEnabled()),
+ $this->checkicon($server->isWritable()),
+ $engine->getEnginePriority(),
+ );
+ }
+
+ $table = id(new AphrontTableView($rows))
+ ->setNoDataString(
+ pht('No search servers are configured.'))
+ ->setHeaders(
+ array(
+ pht('Type'),
+ pht('Protocol'),
+ pht('Host'),
+ pht('Port'),
+ pht('Status'),
+ pht('Engine'),
+ pht('Enabled'),
+ pht('Writable'),
+ pht('Priority'),
+ null,
+ ))
+ ->setColumnClasses(
+ array(
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ 'wide',
+ ));
+
+ return $table;
+ }
+
+ private function checkicon($check) {
+ $icon = $check
+ ? 'fa-check green'
+ : 'fa-times red';
+ $label = $check
+ ? pht('Yes')
+ : pht('No');
+ $view = id(new PHUIIconView())
+ ->setIcon($icon)
+ ->addSigil('has-tooltip')
+ ->setMetadata(
+ array(
+ 'tip' => $label,
+ ));
+
+ return $view;
+ }
+}
diff --git a/src/applications/config/controller/PhabricatorConfigController.php b/src/applications/config/controller/PhabricatorConfigController.php
--- a/src/applications/config/controller/PhabricatorConfigController.php
+++ b/src/applications/config/controller/PhabricatorConfigController.php
@@ -42,8 +42,11 @@
pht('Notification Servers'), null, 'fa-bell-o');
$nav->addFilter('cluster/repositories/',
pht('Repository Servers'), null, 'fa-code');
+ $nav->addFilter('cluster/search/',
+ pht('Search Servers'), null, 'fa-search');
$nav->addLabel(pht('Modules'));
+
$modules = PhabricatorConfigModule::getAllModules();
foreach ($modules as $key => $module) {
$nav->addFilter('module/'.$key.'/',
diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php
--- a/src/applications/maniphest/query/ManiphestTaskQuery.php
+++ b/src/applications/maniphest/query/ManiphestTaskQuery.php
@@ -506,8 +506,8 @@
$fulltext_query->setParameter('types',
array(ManiphestTaskPHIDType::TYPECONST));
- $engine = PhabricatorFulltextStorageEngine::loadEngine();
- $fulltext_results = $engine->executeSearch($fulltext_query);
+ $fulltext_results = PhabricatorClusterSearchRef::executeSearch(
+ $fulltext_query);
if (empty($fulltext_results)) {
$fulltext_results = array(null);
diff --git a/src/applications/search/config/PhabricatorSearchConfigOptions.php b/src/applications/search/config/PhabricatorSearchConfigOptions.php
--- a/src/applications/search/config/PhabricatorSearchConfigOptions.php
+++ b/src/applications/search/config/PhabricatorSearchConfigOptions.php
@@ -20,7 +20,15 @@
}
public function getOptions() {
+ $servers_type = 'custom:PhabricatorClusterSearchConfigOptionType';
+ $servers_help = 'TODO';
+
return array(
+ $this->newOption('search.servers', $servers_type, array())
+ ->setLocked(true)
+ ->setSummary(
+ pht('Configure full-text search servers.'))
+ ->setDescription($servers_help),
$this->newOption('search.elastic.host', 'string', null)
->setLocked(true)
->setDescription(pht('Elastic Search host.'))
diff --git a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
--- a/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
+++ b/src/applications/search/fulltextstorage/PhabricatorElasticFulltextStorageEngine.php
@@ -3,13 +3,25 @@
final class PhabricatorElasticFulltextStorageEngine
extends PhabricatorFulltextStorageEngine {
- private $uri;
+ private $ref;
private $index;
+ private $uri;
private $timeout;
- public function __construct() {
- $this->uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
- $this->index = PhabricatorEnv::getEnvConfig('search.elastic.namespace');
+ public function setRef(PhabricatorElasticSearchServerRef $ref) {
+ $this->uri = (string)$ref->getURI();
+ $this->index = str_replace('/', '', $ref->getPath());
+ $this->version = (int)$ref->getVersion();
+
+ $this->timestampFieldKey = $this->version < 2
+ ? '_timestamp'
+ : 'lastModified';
+
+ $this->textFieldType = $this->version >= 5
+ ? 'text'
+ : 'string';
+
+ $this->enabled = true;
}
public function getEngineIdentifier() {
@@ -24,16 +36,6 @@
return (bool)$this->uri;
}
- public function setURI($uri) {
- $this->uri = $uri;
- return $this;
- }
-
- public function setIndex($index) {
- $this->index = $index;
- return $this;
- }
-
public function setTimeout($timeout) {
$this->timeout = $timeout;
return $this;
@@ -283,9 +285,21 @@
}
public function indexExists() {
+ phlog("indexExists? version=$this->version");
+ phlog($this);
try {
- return (bool)$this->executeRequest('/_status/', array());
+ if ($this->version >= 5) {
+ $uri = '/_stats/';
+ $res = $this->executeRequest($uri, array());
+ return isset($res['indices']['phabricator']);
+ } else if ($this->version >= 2) {
+ $uri = '';
+ } else {
+ $uri = '/_status/';
+ }
+ return (bool)$this->executeRequest($uri, array());
} catch (HTTPFutureHTTPResponseStatus $e) {
+ phlog($e);
if ($e->getStatusCode() == 404) {
return false;
}
@@ -336,6 +350,7 @@
}
public function indexIsSane() {
+
if (!$this->indexExists()) {
return false;
}
@@ -345,7 +360,8 @@
$actual = array_merge($cur_settings[$this->index],
$cur_mapping[$this->index]);
- return $this->check($actual, $this->getIndexConfiguration());
+ $res = $this->check($actual, $this->getIndexConfiguration());
+ return $res;
}
/**
@@ -410,6 +426,18 @@
$this->executeRequest('/', $data, 'PUT');
}
+ public function didHealthCheck($reachable) {
+ static $cache=null;
+ if ($cache !== null) {
+ return;
+ }
+ $cache = $reachable;
+ phlog("didHealthCheck:$reachable");
+ if ($this->ref) {
+ $this->ref->didHealthCheck($reachable);
+ }
+ }
+
private function executeRequest($path, array $data, $method = 'GET') {
$uri = new PhutilURI($this->uri);
$uri->setPath($this->index);
@@ -423,19 +451,30 @@
if ($this->getTimeout()) {
$future->setTimeout($this->getTimeout());
}
- list($body) = $future->resolvex();
+ try {
+ list($body) = $future->resolvex();
+ } catch (HTTPFutureResponseStatus $ex) {
+ if ($ex->isTimeout() || (int)$ex->getStatusCode() > 499) {
+ $this->didHealthCheck(false);
+ }
+ throw $ex;
+ }
if ($method != 'GET') {
return null;
}
try {
- return phutil_json_decode($body);
+ $data = phutil_json_decode($body);
+ $this->didHealthCheck(true);
+ return $data;
} catch (PhutilJSONParserException $ex) {
+ $this->didHealthCheck(false);
throw new PhutilProxyException(
pht('ElasticSearch server returned invalid JSON!'),
$ex);
}
+
}
}
diff --git a/src/applications/search/index/PhabricatorFulltextEngine.php b/src/applications/search/index/PhabricatorFulltextEngine.php
--- a/src/applications/search/index/PhabricatorFulltextEngine.php
+++ b/src/applications/search/index/PhabricatorFulltextEngine.php
@@ -40,8 +40,7 @@
$extension->indexFulltextObject($object, $document);
}
- $storage_engine = PhabricatorFulltextStorageEngine::loadEngine();
- $storage_engine->reindexAbstractDocument($document);
+ PhabricatorClusterSearchRef::reindexAbstractDocument($document);
}
protected function newAbstractDocument($object) {
diff --git a/src/applications/search/query/PhabricatorSearchDocumentQuery.php b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
--- a/src/applications/search/query/PhabricatorSearchDocumentQuery.php
+++ b/src/applications/search/query/PhabricatorSearchDocumentQuery.php
@@ -73,10 +73,7 @@
$query = id(clone($this->savedQuery))
->setParameter('offset', $this->getOffset())
->setParameter('limit', $this->getRawResultLimit());
-
- $engine = PhabricatorFulltextStorageEngine::loadEngine();
-
- return $engine->executeSearch($query);
+ return PhabricatorClusterSearchRef::executeSearch($query);
}
public function getQueryApplicationClass() {
diff --git a/src/infrastructure/cluster/PhabricatorClusterRef.php b/src/infrastructure/cluster/PhabricatorClusterRef.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorClusterRef.php
@@ -0,0 +1,59 @@
+<?php
+
+abstract class PhabricatorClusterRef
+ extends Phobject {
+
+ protected $disabled;
+ protected $host;
+ protected $port;
+
+ const STATUS_OKAY = 'okay';
+ const STATUS_FAIL = 'fail';
+ const STATUS_AUTH = 'auth';
+ const STATUS_REPLICATION_CLIENT = 'replication-client';
+
+ const KEY_REFS = 'cluster.refs';
+
+ public function setDisabled($is_disabled) {
+ $this->disabled = $is_disabled;
+ return $this;
+ }
+
+ public function getDisabled() {
+ return $this->disabled;
+ }
+
+ public function setHost($host) {
+ $this->host = $host;
+ return $this;
+ }
+
+ public function getHost() {
+ return $this->host;
+ }
+
+ public function setPort($port) {
+ $this->port = $port;
+ return $this;
+ }
+
+ public function getPort() {
+ return $this->port;
+ }
+
+ public static function getConnectionStatusMap() {
+ return array(
+ self::STATUS_OKAY => array(
+ 'icon' => 'fa-exchange',
+ 'color' => 'green',
+ 'label' => pht('Okay'),
+ ),
+ self::STATUS_FAIL => array(
+ 'icon' => 'fa-times',
+ 'color' => 'red',
+ 'label' => pht('Failed'),
+ ),
+ );
+ }
+
+}
diff --git a/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php b/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorClusterSearchConfigOptionType.php
@@ -0,0 +1,49 @@
+<?php
+
+final class PhabricatorClusterSearchConfigOptionType
+ extends PhabricatorConfigJSONOptionType {
+
+ public function validateOption(PhabricatorConfigOption $option, $value) {
+ if (!is_array($value)) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration is not valid: value must be a '.
+ 'list of search hosts.'));
+ }
+
+ foreach ($value as $index => $spec) {
+ if (!is_array($spec)) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration is not valid: each entry in the '.
+ 'list must be a dictionary describing a search host, but '.
+ 'the value with index "%s" is not a dictionary.',
+ $index));
+ }
+ }
+
+ $map = array();
+ foreach ($value as $index => $spec) {
+ try {
+ PhutilTypeSpec::checkMap(
+ $spec,
+ array(
+ 'type' => 'string',
+ 'hosts' => 'optional list<map<string, wild>>',
+ 'roles' => 'optional list<string>',
+ 'port' => 'optional int',
+ 'protocol' => 'optional string',
+ 'path' => 'optional string',
+ 'version' => 'optional int',
+ ));
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ 'Search cluster configuration has an invalid host '.
+ 'specification (at index "%s"): %s.',
+ $index,
+ $ex->getMessage()));
+ }
+ }
+ }
+}
diff --git a/src/infrastructure/cluster/PhabricatorClusterSearchRef.php b/src/infrastructure/cluster/PhabricatorClusterSearchRef.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorClusterSearchRef.php
@@ -0,0 +1,176 @@
+<?php
+
+abstract class PhabricatorClusterSearchRef
+ extends PhabricatorClusterRef {
+
+ const KEY_REFS = 'cluster.search.refs';
+ const KEY_HEALTH = 'cluster.search.health';
+
+ protected $healthRecord;
+ protected $roles = array();
+
+ public function isWritable() {
+ return in_array($this->roles, 'write');
+ }
+
+ public function isReadable() {
+ return in_array($this->roles, 'read');
+ }
+
+ public function setRoles(array $roles) {
+ foreach ($roles as $role) {
+ $this->roles[$role] = $role;
+ }
+ return $this;
+ }
+
+ public function getRoles() {
+ return array_values($this->roles);
+ }
+
+ public function getHealthRecordCacheKey() {
+ $host = $this->getHost();
+ $port = $this->getPort();
+ $key = self::KEY_HEALTH;
+
+ return "{$key}({$host}, {$port})";
+ }
+
+/**
+ * @return PhabricatorClusterServiceHealthRecord
+ */
+ public function getHealthRecord() {
+ if (!$this->healthRecord) {
+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
+ $this->getHealthRecordCacheKey());
+ }
+ return $this->healthRecord;
+ }
+
+ public function didHealthCheck($reachable) {
+ $record = $this->getHealthRecord();
+ $should_check = $record->getShouldCheck();
+
+ if ($should_check) {
+ $record->didHealthCheck($reachable);
+ }
+ }
+
+ public static function getLiveServers() {
+ $cache = PhabricatorCaches::getRequestCache();
+
+ $refs = $cache->getKey(self::KEY_REFS);
+ if (!$refs) {
+ $refs = self::newRefs();
+ $cache->setKey(self::KEY_REFS, $refs);
+ }
+
+ return $refs;
+ }
+
+ public static function newRefs() {
+ $builtin_ref = new PhabricatorMysqlSearchServerRef();
+ $refs = array('all' => array($builtin_ref));
+ // try to load custom fulltext search service configuration.
+ $configs = PhabricatorEnv::getEnvConfigIfExists('search.servers');
+ if (!$configs) {
+ // if nothing is configured, just return a reference to the built-in
+ // fulltext search.
+ return $refs;
+ }
+
+ $types = id(new PhutilClassMapQuery())
+ ->setAncestorClass(__CLASS__)
+ ->setUniqueMethod('getEngineIdentifier')
+ ->execute();
+
+ foreach ($configs as $config) {
+ if (!isset($types[$config['type']])) {
+ throw new Exception(pht('configured search server type is invalid %s',
+ $config['type']));
+ }
+ $type = $types[$config['type']];
+ if ($config['type'] == 'mysql') {
+ // if there is a config for mysql, then it overrides the built-in ref
+ $ref = $builtin_ref;
+ $config['hosts'] = array(
+ array(
+ 'host' => 'local',
+ 'roles' => array('read', 'write'),
+ ),
+ );
+ } else {
+ // otherwise it's a custom type, clone the prototype ref
+ $ref = $type;
+ }
+
+ $hosts = $config['hosts'];
+ unset($config['hosts']);
+ unset($config['type']);
+
+ self::setFromArray($ref, $config);
+
+ foreach ($hosts as $hostconfig) {
+ $hostref = clone($ref);
+ self::setFromArray($hostref, $hostconfig);
+ $refs['all'][] = $hostref;
+ foreach ($hostconfig['roles'] as $role) {
+ $refs[$role][] = $hostref;
+ }
+ }
+ $hostref->initEngine();
+ }
+ return $refs;
+ }
+
+ private static function setFromArray($obj, array $array) {
+ foreach ($array as $key => $val) {
+ $setter = 'set'.ucfirst($key);
+ if (method_exists($obj, $setter)) {
+ call_user_func(array($obj, $setter), $val);
+ }
+ }
+ }
+
+ public static function getServersForRole($role) {
+ $servers = self::getLiveServers();
+ phlog($servers);
+ if (empty($servers[$role])) {
+ throw new Exception(pht('No servers found for role %s', $role));
+ }
+ return $servers[$role];
+ }
+
+ public function initEngine() {}
+
+ abstract public function getEngine();
+ abstract public function loadServerStatus();
+
+ public static function reindexAbstractDocument(
+ PhabricatorSearchAbstractDocument $doc) {
+ $servers = self::getServersForRole('write');
+ foreach ($servers as $server) {
+ $server->getEngine()->reindexAbstractDocument($doc);
+ }
+ }
+
+ public static function executeSearch(PhabricatorSavedQuery $query) {
+ $servers = self::getServersForRole('read');
+ foreach ($servers as $server) {
+ $last_exception = null;
+ try {
+ $res = $server->getEngine()->executeSearch($query);
+ // return immediately if we get results without an exception
+ return $res;
+ } catch (Exception $ex) {
+ // try each server in turn, only throw if none succeed
+ $last_exception = $ex;
+ }
+ }
+ if ($last_exception) {
+ throw $last_exception;
+ }
+ return $res;
+ }
+
+}
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
rename from src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
rename to src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
--- a/src/infrastructure/cluster/PhabricatorDatabaseHealthRecord.php
+++ b/src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php
@@ -1,20 +1,19 @@
<?php
-final class PhabricatorDatabaseHealthRecord
+class PhabricatorClusterServiceHealthRecord
extends Phobject {
- private $ref;
+ private $cacheKey;
private $shouldCheck;
private $isHealthy;
private $upEventCount;
private $downEventCount;
- public function __construct(PhabricatorDatabaseRef $ref) {
- $this->ref = $ref;
+ public function __construct($cache_key) {
+ $this->cacheKey = $cache_key;
$this->readState();
}
-
/**
* Is the database currently healthy?
*/
@@ -153,18 +152,13 @@
}
}
- private function getHealthRecordCacheKey() {
- $ref = $this->ref;
-
- $host = $ref->getHost();
- $port = $ref->getPort();
-
- return "cluster.db.health({$host}, {$port})";
+ public function getCacheKey() {
+ return $this->cacheKey;
}
private function readHealthRecord() {
$cache = PhabricatorCaches::getSetupCache();
- $cache_key = $this->getHealthRecordCacheKey();
+ $cache_key = $this->getCacheKey();
$health_record = $cache->getKey($cache_key);
if (!is_array($health_record)) {
@@ -180,7 +174,7 @@
private function writeHealthRecord(array $record) {
$cache = PhabricatorCaches::getSetupCache();
- $cache_key = $this->getHealthRecordCacheKey();
+ $cache_key = $this->getCacheKey();
$cache->setKey($cache_key, $record);
}
diff --git a/src/infrastructure/cluster/PhabricatorClusterServiceInterface.php b/src/infrastructure/cluster/PhabricatorClusterServiceInterface.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorClusterServiceInterface.php
@@ -0,0 +1,3 @@
+<?php
+
+interface PhabricatorClusterServiceInterface {}
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
@@ -14,6 +14,7 @@
const REPLICATION_SLOW = 'replica-slow';
const REPLICATION_NOT_REPLICATING = 'not-replicating';
+ const KEY_HEALTH = 'cluster.db.health';
const KEY_REFS = 'cluster.db.refs';
const KEY_INDIVIDUAL = 'cluster.db.individual';
@@ -489,9 +490,18 @@
return $this;
}
+ private function getHealthRecordCacheKey() {
+ $host = $this->getHost();
+ $port = $this->getPort();
+ $key = self::KEY_HEALTH;
+
+ return "{$key}({$host}, {$port})";
+ }
+
public function getHealthRecord() {
if (!$this->healthRecord) {
- $this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
+ $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
+ $this->getHealthRecordCacheKey());
}
return $this->healthRecord;
}
diff --git a/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php b/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorElasticSearchServerRef.php
@@ -0,0 +1,93 @@
+<?php
+
+final class PhabricatorElasticSearchServerRef
+ extends PhabricatorClusterSearchRef {
+
+ private $engine;
+ private $version;
+ private $path;
+ private $protocol;
+ protected $port;
+
+ const KEY_REFS = 'search.elastic.refs';
+
+ public function getDisplayName() {
+ return 'ElasticSearch';
+ }
+
+ public function getEngineIdentifier() {
+ return 'elasticsearch';
+ }
+
+ public function setProtocol($protocol) {
+ $this->protocol = $protocol;
+ return $this;
+ }
+
+ public function getProtocol() {
+ return $this->protocol;
+ }
+
+ public function setPath($path) {
+ $this->path = $path;
+ return $this;
+ }
+
+ public function getPath() {
+ return $this->path;
+ }
+
+ public function setVersion($version) {
+ $this->version = $version;
+ return $this;
+ }
+
+ public function getVersion() {
+ return $this->version;
+ }
+
+ public function getPort() {
+ return $this->port;
+ }
+ public function setPort($value) {
+ $this->port = $value;
+ return $this;
+ }
+
+ public function getURI($to_path = null) {
+ $full_path = rtrim($this->getPath(), '/').'/'.ltrim($to_path, '/');
+
+ $uri = id(new PhutilURI('http://'.$this->getHost()))
+ ->setProtocol($this->getProtocol())
+ ->setPort($this->getPort())
+ ->setPath($full_path);
+
+ return $uri;
+ }
+
+ /**
+ * @return PhabricatorElasticFulltextStorageEngine
+ */
+ public function getEngine() {
+ if (!$this->engine) {
+ phlog('getEngine');
+ $engine = new PhabricatorElasticFulltextStorageEngine();
+ $engine->setRef($this);
+ $this->engine = $engine;
+ }
+ return $this->engine;
+ }
+
+ public function loadServerStatus() {
+ $status = $this->getEngine()->indexIsSane()
+ ? 'okay'
+ : 'fail';
+ $engine = $this->getEngine();
+ $status = $engine->indexIsSane()
+ ? 'okay'
+ : 'fail';
+ phlog("indexIsSane: $status");
+ return $status;
+ }
+
+}
diff --git a/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php b/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorMysqlSearchServerRef.php
@@ -0,0 +1,35 @@
+<?php
+
+final class PhabricatorMysqlSearchServerRef
+ extends PhabricatorClusterSearchRef {
+
+ private $engine;
+
+ public function __construct() {
+ $this->engine = new PhabricatorMySQLFulltextStorageEngine();
+ }
+
+ public function getDisplayName() {
+ return 'MySQL';
+ }
+
+ public function getEngineIdentifier() {
+ return $this->engine->getEngineIdentifier();
+ }
+
+ public function getEngine() {
+ return $this->engine;
+ }
+
+ public function getProtocol() {
+ return 'mysql';
+ }
+
+ public function loadServerStatus() {
+ PhabricatorDatabaseRef::queryAll();
+ $ref = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication('search');
+ $status = $ref->getConnectionStatus();
+ return $status;
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 30, 1:56 PM (5 d, 21 h ago)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/sd/xj/vdopv5cqn7cia3jg
Default Alt Text
D17384.id41856.diff (34 KB)

Event Timeline

Phabricator · Built by Phacility