Page MenuHomePhabricator
Files F13327324

D15663.diff
No OneTemporary
Actions

D15663.diff
View Options

diff --git a/resources/celerity/map.php b/resources/celerity/map.php
--- a/resources/celerity/map.php
+++ b/resources/celerity/map.php
@@ -7,8 +7,8 @@
*/
return array(
'names' => array(
- 'core.pkg.css' => '82cefddc',
- 'core.pkg.js' => 'e5484f37',
+ 'core.pkg.css' => '35e4a99a',
+ 'core.pkg.js' => '8a616602',
'darkconsole.pkg.js' => 'e7393ebb',
'differential.pkg.css' => '7ba78475',
'differential.pkg.js' => 'd0cd0df6',
@@ -22,7 +22,7 @@
'rsrc/css/aphront/lightbox-attachment.css' => '7acac05d',
'rsrc/css/aphront/list-filter-view.css' => '5d6f0526',
'rsrc/css/aphront/multi-column.css' => 'fd18389d',
- 'rsrc/css/aphront/notification.css' => '7f684b62',
+ 'rsrc/css/aphront/notification.css' => '3f6c89c9',
'rsrc/css/aphront/panel-view.css' => '8427b78d',
'rsrc/css/aphront/phabricator-nav-view.css' => 'ac79a758',
'rsrc/css/aphront/table-view.css' => '9258e19f',
@@ -494,6 +494,7 @@
'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
'rsrc/js/core/behavior-phabricator-nav.js' => '56a1ca03',
'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => '340c8eff',
+ 'rsrc/js/core/behavior-read-only-warning.js' => 'f8ea359c',
'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b',
'rsrc/js/core/behavior-remarkup-preview.js' => '4b700e9e',
'rsrc/js/core/behavior-reorder-applications.js' => '76b9fc3e',
@@ -666,6 +667,7 @@
'javelin-behavior-project-boards' => '14a1faae',
'javelin-behavior-project-create' => '065227cc',
'javelin-behavior-quicksand-blacklist' => '7927a7d3',
+ 'javelin-behavior-read-only-warning' => 'f8ea359c',
'javelin-behavior-recurring-edit' => '5f1c4d5f',
'javelin-behavior-refresh-csrf' => 'ab2f381b',
'javelin-behavior-releeph-preview-branch' => 'b2b4fbaf',
@@ -766,7 +768,7 @@
'phabricator-main-menu-view' => 'd00a795a',
'phabricator-nav-view-css' => 'ac79a758',
'phabricator-notification' => 'ccf1cbf8',
- 'phabricator-notification-css' => '7f684b62',
+ 'phabricator-notification-css' => '3f6c89c9',
'phabricator-notification-menu-css' => 'f31c0bde',
'phabricator-object-selector-css' => '85ee8ce6',
'phabricator-phtize' => 'd254d646',
@@ -2109,6 +2111,11 @@
'javelin-util',
'phabricator-busy',
),
+ 'f8ea359c' => array(
+ 'javelin-behavior',
+ 'javelin-uri',
+ 'phabricator-notification',
+ ),
'fa0f4fc2' => array(
'javelin-behavior',
'javelin-dom',
@@ -2284,6 +2291,7 @@
'javelin-quicksand',
'javelin-behavior-quicksand-blacklist',
'javelin-behavior-high-security-warning',
+ 'javelin-behavior-read-only-warning',
'javelin-scrollbar',
'javelin-behavior-scrollbar',
'javelin-behavior-durable-column',
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1986,6 +1986,7 @@
'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
+ 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
@@ -6392,6 +6393,7 @@
'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
+ 'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType',
'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
'PhabricatorCommentEditField' => 'PhabricatorEditField',
diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
--- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php
+++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php
@@ -20,6 +20,12 @@
}
protected function executeChecks() {
+ // TODO: These checks should be executed against every reachable replica?
+ // See T10759.
+ if (PhabricatorEnv::isReadOnly()) {
+ return;
+ }
+
$max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
// This primarily supports setting the filesize limit for MySQL to 8MB,
diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php
--- a/src/applications/config/option/PhabricatorClusterConfigOptions.php
+++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php
@@ -20,6 +20,20 @@
}
public function getOptions() {
+ $databases_type = 'custom:PhabricatorClusterDatabasesConfigOptionType';
+ $databases_help = $this->deformat(pht(<<<EOTEXT
+WARNING: This is a prototype option and the description below is currently pure
+fantasy.
+
+This option allows you to make Phabricator aware of database read replicas so
+it can monitor database health, spread load, and degrade gracefully to
+read-only mode in the event of a failure on the primary host. For help with
+configuring cluster databases, see **[[ %s | %s ]]** in the documentation.
+EOTEXT
+ ,
+ PhabricatorEnv::getDoclink('Cluster: Databases'),
+ pht('Cluster: Databases')));
+
return array(
$this->newOption('cluster.addresses', 'list<string>', array())
->setLocked(true)
@@ -88,7 +102,11 @@
'into this mode automatically when it detects that the database '.
'master is unreachable, but you can activate it manually in '.
'order to perform maintenance or test configuration.')),
-
+ $this->newOption('cluster.databases', $databases_type, array())
+ ->setHidden(true)
+ ->setSummary(
+ pht('Configure database read replicas.'))
+ ->setDescription($databases_help),
);
}
diff --git a/src/docs/book/user.book b/src/docs/book/user.book
--- a/src/docs/book/user.book
+++ b/src/docs/book/user.book
@@ -32,6 +32,9 @@
"conduit": {
"name": "API Documentation"
},
+ "cluster": {
+ "name": "Cluster Configuration"
+ },
"fieldmanual": {
"name": "Field Manuals"
},
diff --git a/src/docs/user/cluster/cluster.diviner b/src/docs/user/cluster/cluster.diviner
new file mode 100644
--- /dev/null
+++ b/src/docs/user/cluster/cluster.diviner
@@ -0,0 +1,40 @@
+@title Clustering Introduction
+@group cluster
+
+Guide to configuring Phabricator across multiple hosts for availability and
+performance.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+Phabricator can be configured to run on mulitple hosts with redundant services
+to improve its availability and scalability, and make disaster recovery much
+easier.
+
+Clustering is more complex to setup and maintain than running everything on a
+single host, but greatly reduces the cost of recovering from hardware and
+network failures.
+
+Each Phabricator service has an array of clustering options that can be
+configured independently. Configuring a cluster is inherently complex, and this
+is an advanced feature aimed at installs with large userbases and experienced
+operations personnel who need this high degree of flexibility.
+
+The remainder of this document summarizes how to add redundancy to each
+service and where your efforts are likely to have the greatest impact.
+
+Cluster: Databases
+=================
+
+Configuring multiple database hosts is moderately complex, but normally has the
+highest impact on availability and resistance to data loss. This is usually the
+most important service to make redundant if your focus is on availability and
+disaster recovery.
+
+Configuring replicas allows Phabricator to run in read-only mode if you lose
+the master, and to quickly promote the replica as a replacement.
+
+For details, see @{article:Cluster: Databases}.
diff --git a/src/docs/user/cluster/cluster_databases.diviner b/src/docs/user/cluster/cluster_databases.diviner
new file mode 100644
--- /dev/null
+++ b/src/docs/user/cluster/cluster_databases.diviner
@@ -0,0 +1,161 @@
+@title Cluster: Databases
+@group intro
+
+Configuring Phabricator to use multiple database hosts.
+
+Overview
+========
+
+WARNING: This feature is a very early prototype; the features this document
+describes are mostly speculative fantasy.
+
+You can deploy Phabricator with multiple database hosts, configured as a master
+and a set of replicas. The advantages of doing this are:
+
+ - faster recovery from disasters by promoting a replica;
+ - graceful degradation if the master fails;
+ - reduced load on the master; and
+ - some tools to help monitor and manage replica health.
+
+This configuration is complex, and many installs do not need to pursue it.
+
+Phabricator can not currently be configured into a multi-master mode, nor can
+it be configured to automatically promote a replica to become the new master.
+
+
+Setting up MySQL Replication
+============================
+
+TODO: Write this section.
+
+
+Configuring Replicas
+====================
+
+Once your replicas are in working order, tell Phabricator about them by
+configuring the `cluster.database` option. This option must be configured from
+the command line or in configuration files because Phabricator needs to read
+it //before// it can connect to databases.
+
+This option value will list all of the database hosts that you want Phabricator
+to interact with: your master and all your replicas. Each entry in the list
+should have these keys:
+
+ - `host`: //Required string.// The database host name.
+ - `role`: //Required string.// The cluster role of this host, one of
+ `master` or `replica`.
+ - `port`: //Optional int.// The port to connect to. If omitted, the default
+ port from `mysql.port` will be used.
+ - `user`: //Optional string.// The MySQL username to use to connect to this
+ host. If omitted, the default from `mysql.user` will be used.
+ - `pass`: //Optional string.// The password to use to connect to this host.
+ If omitted, the default from `mysql.pass` will be used.
+ - `disabled`: //Optional bool.// If set to `true`, Phabricator will not
+ connect to this host. You can use this to temporarily take a host out
+ of service.
+
+When `cluster.databases` is configured the `mysql.host` option is not used.
+The other MySQL connection configuration options (`mysql.port`, `mysql.user`,
+`mysql.pass`) are used only to provide defaults.
+
+Once you've configured this option, restart Phabricator for the changes to take
+effect, then continue to "Monitoring and Testing" to verify the configuration.
+
+
+Monitoring and Testing
+======================
+
+TODO: Write this part.
+
+Degradation to Read-Only Mode
+=============================
+
+Phabricator will degrade to read-only mode when any of these conditions occur:
+
+ - you turn it on explicitly;
+ - you configure cluster mode, but don't set up any masters;
+ - the master is misconfigured and unsafe to write to; or
+ - the master is unreachable.
+
+When Phabricator is running in read-only mode, users can still read data and
+browse and clone repositories, but they can not edit, update, or push new
+changes. For example, users can still read disaster recovery information on
+the wiki or emergency contact information on user profiles.
+
+You can enable this mode explicitly by configuring `cluster.read-only`. Some
+reasons you might want to do this include:
+
+ - to test that the mode works like you expect it to;
+ - to make sure that information you need will be available;
+ - to prevent new writes while performing database maintenance; or
+ - to permanently archive a Phabricator install.
+
+You can also enable this mode implicitly by configuring `cluster.databases`
+but disabling the master, or by not specifying any host as a master. This may
+be more convenient than turning it on explicitly during the course of
+operations work.
+
+Before writing to a master, Phabricator will verify that the host is not
+configured as a replica. This is a safety feature to prevent data loss if your
+MySQL and Phabricator configurations disagree about replica configuration. If
+your `master` is currently replicating from another host, Phabricator will
+treat it as a `replica` instead and implicitly degrade into read-only mode.
+
+Finally, if Phabricator is unable to reach the master, it will degrade into
+read-only mode. For details on how Phabricator determines that a master is
+unreachable, see "Unreachable Masters" below.
+
+If a master becomes unreachable, this normally corresponds to loss of the
+master host, a severed network link, or some other sort of disaster.
+Phabricator will degrade and continue operating in read-only mode until the
+master recovers or operations personnel can assess the situation and intervene.
+
+If you end up in a situation where you have lost the master and can not get it
+back online (or can not restore it quickly) you can promote a replica to become
+the new master. See the next section, "Promoting a Replica", for details.
+
+
+Promoting a Replica
+===================
+
+TODO: Write this, too.
+
+
+Unreachable Masters
+===================
+
+This section describes how Phabricator determines that a master has been lost,
+marks it unreachable, and degrades into read-only mode.
+
+TODO: For now, it doesn't.
+
+
+Backups
+======
+
+Even if you configure replication, you should still retain separate backup
+snapshots. Replicas protect you from data loss if you lose a host, but they do
+not let you recover from data mutation mistakes.
+
+If something issues `DELETE` or `UPDATE` statements and destroys data on the
+master, the mutation will propagate to the replicas almost immediately and the
+data will be gone forever. Normally, the only way to recover this data is from
+backup snapshots.
+
+Although you should still have a backup process, your backup process can
+safely pull dumps from a replica instead of the master. This operation can
+be slow, so offloading it to a replica can make the perforance of the master
+more consistent.
+
+To dump from a replica, wait for this TODO to be resolved and then do whatever
+it says to do:
+
+TODO: Make `bin/storage dump` replica-aware. See T10758.
+
+
+Next Steps
+==========
+
+Continue by:
+
+ - returning to @{article:Clustering Introduction}.
diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
@@ -0,0 +1,98 @@
+<?php
+
+final class PhabricatorClusterDatabasesConfigOptionType
+ extends PhabricatorConfigJSONOptionType {
+
+ public function validateOption(PhabricatorConfigOption $option, $value) {
+ if (!is_array($value)) {
+ throw new Exception(
+ pht(
+ 'Database cluster configuration is not valid: value must be a '.
+ 'list of database hosts.'));
+ }
+
+ foreach ($value as $index => $spec) {
+ if (!is_array($spec)) {
+ throw new Exception(
+ pht(
+ 'Database cluster configuration is not valid: each entry in the '.
+ 'list must be a dictionary describing a database host, but '.
+ 'the value with index "%s" is not a dictionary.',
+ $index));
+ }
+ }
+
+ $masters = array();
+ $map = array();
+ foreach ($value as $index => $spec) {
+ try {
+ PhutilTypeSpec::checkMap(
+ $spec,
+ array(
+ 'host' => 'string',
+ 'role' => 'string',
+ 'port' => 'optional int',
+ 'user' => 'optional string',
+ 'pass' => 'optional string',
+ 'disabled' => 'optional bool',
+ ));
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ 'Database cluster configuration has an invalid host '.
+ 'specification (at index "%s"): %s.',
+ $index,
+ $ex->getMessage()));
+ }
+
+ $role = $spec['role'];
+ $host = $spec['host'];
+ $port = idx($spec, 'port');
+
+ switch ($role) {
+ case 'master':
+ case 'replica':
+ break;
+ default:
+ throw new Exception(
+ pht(
+ 'Database cluster configuration describes an invalid '.
+ 'host ("%s", at index "%s") with an unrecognized role ("%s"). '.
+ 'Valid roles are "%s" or "%s".',
+ $spec['host'],
+ $index,
+ $spec['role'],
+ 'master',
+ 'replica'));
+ }
+
+ if ($role === 'master') {
+ $masters[] = $host;
+ }
+
+ // We can't guarantee that you didn't just give the same host two
+ // different names in DNS, but this check can catch silly copy/paste
+ // mistakes.
+ $key = "{$host}:{$port}";
+ if (isset($map[$key])) {
+ throw new Exception(
+ pht(
+ 'Database cluster configuration is invalid: it describes the '.
+ 'same host ("%s") multiple times. Each host should appear only '.
+ 'once in the list.',
+ $host));
+ }
+ $map[$key] = true;
+ }
+
+ if (count($masters) > 1) {
+ throw new Exception(
+ pht(
+ 'Database cluster configuration is invalid: it describes multiple '.
+ 'masters. No more than one host may be a master. Hosts currently '.
+ 'configured as masters: %s.',
+ implode(', ', $masters)));
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/PhabricatorMarkupEngine.php b/src/infrastructure/markup/PhabricatorMarkupEngine.php
--- a/src/infrastructure/markup/PhabricatorMarkupEngine.php
+++ b/src/infrastructure/markup/PhabricatorMarkupEngine.php
@@ -254,6 +254,8 @@
}
}
+ $is_readonly = PhabricatorEnv::isReadOnly();
+
foreach ($objects as $key => $info) {
// False check in case MySQL doesn't support unicode characters
// in the string (T1191), resulting in unserialize returning false.
@@ -279,7 +281,7 @@
->setCacheData($data)
->setMetadata($metadata);
- if (isset($use_cache[$key])) {
+ if (isset($use_cache[$key]) && !$is_readonly) {
// This is just filling a cache and always safe, even on a read pathway.
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
$blocks[$key]->replace();
diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php
--- a/src/view/page/PhabricatorStandardPageView.php
+++ b/src/view/page/PhabricatorStandardPageView.php
@@ -276,7 +276,7 @@
Javelin::initBehavior(
'read-only-warning',
array(
- 'message' => pht('This install is currently in read-only mode.'),
+ 'message' => pht('Phabricator is currently in read-only mode.'),
));
}

File Metadata

Mime Type
text/plain
Expires
Sun, Jun 16, 5:00 PM (1 w, 18 h ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
6274627
Default Alt Text
D15663.diff (19 KB)

Event Timeline

Phabricator · Built by Phacility