diff --git a/resources/celerity/map.php b/resources/celerity/map.php --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -7,8 +7,8 @@ */ return array( 'names' => array( - 'core.pkg.css' => '82cefddc', - 'core.pkg.js' => 'e5484f37', + 'core.pkg.css' => '35e4a99a', + 'core.pkg.js' => '8a616602', 'darkconsole.pkg.js' => 'e7393ebb', 'differential.pkg.css' => '7ba78475', 'differential.pkg.js' => 'd0cd0df6', @@ -22,7 +22,7 @@ 'rsrc/css/aphront/lightbox-attachment.css' => '7acac05d', 'rsrc/css/aphront/list-filter-view.css' => '5d6f0526', 'rsrc/css/aphront/multi-column.css' => 'fd18389d', - 'rsrc/css/aphront/notification.css' => '7f684b62', + 'rsrc/css/aphront/notification.css' => '3f6c89c9', 'rsrc/css/aphront/panel-view.css' => '8427b78d', 'rsrc/css/aphront/phabricator-nav-view.css' => 'ac79a758', 'rsrc/css/aphront/table-view.css' => '9258e19f', @@ -494,6 +494,7 @@ 'rsrc/js/core/behavior-oncopy.js' => '2926fff2', 'rsrc/js/core/behavior-phabricator-nav.js' => '56a1ca03', 'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => '340c8eff', + 'rsrc/js/core/behavior-read-only-warning.js' => 'f8ea359c', 'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b', 'rsrc/js/core/behavior-remarkup-preview.js' => '4b700e9e', 'rsrc/js/core/behavior-reorder-applications.js' => '76b9fc3e', @@ -666,6 +667,7 @@ 'javelin-behavior-project-boards' => '14a1faae', 'javelin-behavior-project-create' => '065227cc', 'javelin-behavior-quicksand-blacklist' => '7927a7d3', + 'javelin-behavior-read-only-warning' => 'f8ea359c', 'javelin-behavior-recurring-edit' => '5f1c4d5f', 'javelin-behavior-refresh-csrf' => 'ab2f381b', 'javelin-behavior-releeph-preview-branch' => 'b2b4fbaf', @@ -766,7 +768,7 @@ 'phabricator-main-menu-view' => 'd00a795a', 'phabricator-nav-view-css' => 'ac79a758', 'phabricator-notification' => 'ccf1cbf8', - 'phabricator-notification-css' => '7f684b62', + 'phabricator-notification-css' => '3f6c89c9', 'phabricator-notification-menu-css' => 'f31c0bde', 'phabricator-object-selector-css' => '85ee8ce6', 'phabricator-phtize' => 'd254d646', @@ -2109,6 +2111,11 @@ 'javelin-util', 'phabricator-busy', ), + 'f8ea359c' => array( + 'javelin-behavior', + 'javelin-uri', + 'phabricator-notification', + ), 'fa0f4fc2' => array( 'javelin-behavior', 'javelin-dom', @@ -2284,6 +2291,7 @@ 'javelin-quicksand', 'javelin-behavior-quicksand-blacklist', 'javelin-behavior-high-security-warning', + 'javelin-behavior-read-only-warning', 'javelin-scrollbar', 'javelin-behavior-scrollbar', 'javelin-behavior-durable-column', diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1986,6 +1986,7 @@ 'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php', 'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php', 'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php', + 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', 'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php', @@ -6392,6 +6393,7 @@ 'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine', 'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions', + 'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', 'PhabricatorCommentEditField' => 'PhabricatorEditField', diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php --- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php +++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php @@ -20,6 +20,12 @@ } protected function executeChecks() { + // TODO: These checks should be executed against every reachable replica? + // See T10759. + if (PhabricatorEnv::isReadOnly()) { + return; + } + $max_allowed_packet = self::loadRawConfigValue('max_allowed_packet'); // This primarily supports setting the filesize limit for MySQL to 8MB, diff --git a/src/applications/config/option/PhabricatorClusterConfigOptions.php b/src/applications/config/option/PhabricatorClusterConfigOptions.php --- a/src/applications/config/option/PhabricatorClusterConfigOptions.php +++ b/src/applications/config/option/PhabricatorClusterConfigOptions.php @@ -20,6 +20,20 @@ } public function getOptions() { + $databases_type = 'custom:PhabricatorClusterDatabasesConfigOptionType'; + $databases_help = $this->deformat(pht(<<newOption('cluster.addresses', 'list', array()) ->setLocked(true) @@ -88,7 +102,11 @@ 'into this mode automatically when it detects that the database '. 'master is unreachable, but you can activate it manually in '. 'order to perform maintenance or test configuration.')), - + $this->newOption('cluster.databases', $databases_type, array()) + ->setHidden(true) + ->setSummary( + pht('Configure database read replicas.')) + ->setDescription($databases_help), ); } diff --git a/src/docs/book/user.book b/src/docs/book/user.book --- a/src/docs/book/user.book +++ b/src/docs/book/user.book @@ -32,6 +32,9 @@ "conduit": { "name": "API Documentation" }, + "cluster": { + "name": "Cluster Configuration" + }, "fieldmanual": { "name": "Field Manuals" }, diff --git a/src/docs/user/cluster/cluster.diviner b/src/docs/user/cluster/cluster.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/cluster/cluster.diviner @@ -0,0 +1,40 @@ +@title Clustering Introduction +@group cluster + +Guide to configuring Phabricator across multiple hosts for availability and +performance. + +Overview +======== + +WARNING: This feature is a very early prototype; the features this document +describes are mostly speculative fantasy. + +Phabricator can be configured to run on mulitple hosts with redundant services +to improve its availability and scalability, and make disaster recovery much +easier. + +Clustering is more complex to setup and maintain than running everything on a +single host, but greatly reduces the cost of recovering from hardware and +network failures. + +Each Phabricator service has an array of clustering options that can be +configured independently. Configuring a cluster is inherently complex, and this +is an advanced feature aimed at installs with large userbases and experienced +operations personnel who need this high degree of flexibility. + +The remainder of this document summarizes how to add redundancy to each +service and where your efforts are likely to have the greatest impact. + +Cluster: Databases +================= + +Configuring multiple database hosts is moderately complex, but normally has the +highest impact on availability and resistance to data loss. This is usually the +most important service to make redundant if your focus is on availability and +disaster recovery. + +Configuring replicas allows Phabricator to run in read-only mode if you lose +the master, and to quickly promote the replica as a replacement. + +For details, see @{article:Cluster: Databases}. diff --git a/src/docs/user/cluster/cluster_databases.diviner b/src/docs/user/cluster/cluster_databases.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/cluster/cluster_databases.diviner @@ -0,0 +1,161 @@ +@title Cluster: Databases +@group intro + +Configuring Phabricator to use multiple database hosts. + +Overview +======== + +WARNING: This feature is a very early prototype; the features this document +describes are mostly speculative fantasy. + +You can deploy Phabricator with multiple database hosts, configured as a master +and a set of replicas. The advantages of doing this are: + + - faster recovery from disasters by promoting a replica; + - graceful degradation if the master fails; + - reduced load on the master; and + - some tools to help monitor and manage replica health. + +This configuration is complex, and many installs do not need to pursue it. + +Phabricator can not currently be configured into a multi-master mode, nor can +it be configured to automatically promote a replica to become the new master. + + +Setting up MySQL Replication +============================ + +TODO: Write this section. + + +Configuring Replicas +==================== + +Once your replicas are in working order, tell Phabricator about them by +configuring the `cluster.database` option. This option must be configured from +the command line or in configuration files because Phabricator needs to read +it //before// it can connect to databases. + +This option value will list all of the database hosts that you want Phabricator +to interact with: your master and all your replicas. Each entry in the list +should have these keys: + + - `host`: //Required string.// The database host name. + - `role`: //Required string.// The cluster role of this host, one of + `master` or `replica`. + - `port`: //Optional int.// The port to connect to. If omitted, the default + port from `mysql.port` will be used. + - `user`: //Optional string.// The MySQL username to use to connect to this + host. If omitted, the default from `mysql.user` will be used. + - `pass`: //Optional string.// The password to use to connect to this host. + If omitted, the default from `mysql.pass` will be used. + - `disabled`: //Optional bool.// If set to `true`, Phabricator will not + connect to this host. You can use this to temporarily take a host out + of service. + +When `cluster.databases` is configured the `mysql.host` option is not used. +The other MySQL connection configuration options (`mysql.port`, `mysql.user`, +`mysql.pass`) are used only to provide defaults. + +Once you've configured this option, restart Phabricator for the changes to take +effect, then continue to "Monitoring and Testing" to verify the configuration. + + +Monitoring and Testing +====================== + +TODO: Write this part. + +Degradation to Read-Only Mode +============================= + +Phabricator will degrade to read-only mode when any of these conditions occur: + + - you turn it on explicitly; + - you configure cluster mode, but don't set up any masters; + - the master is misconfigured and unsafe to write to; or + - the master is unreachable. + +When Phabricator is running in read-only mode, users can still read data and +browse and clone repositories, but they can not edit, update, or push new +changes. For example, users can still read disaster recovery information on +the wiki or emergency contact information on user profiles. + +You can enable this mode explicitly by configuring `cluster.read-only`. Some +reasons you might want to do this include: + + - to test that the mode works like you expect it to; + - to make sure that information you need will be available; + - to prevent new writes while performing database maintenance; or + - to permanently archive a Phabricator install. + +You can also enable this mode implicitly by configuring `cluster.databases` +but disabling the master, or by not specifying any host as a master. This may +be more convenient than turning it on explicitly during the course of +operations work. + +Before writing to a master, Phabricator will verify that the host is not +configured as a replica. This is a safety feature to prevent data loss if your +MySQL and Phabricator configurations disagree about replica configuration. If +your `master` is currently replicating from another host, Phabricator will +treat it as a `replica` instead and implicitly degrade into read-only mode. + +Finally, if Phabricator is unable to reach the master, it will degrade into +read-only mode. For details on how Phabricator determines that a master is +unreachable, see "Unreachable Masters" below. + +If a master becomes unreachable, this normally corresponds to loss of the +master host, a severed network link, or some other sort of disaster. +Phabricator will degrade and continue operating in read-only mode until the +master recovers or operations personnel can assess the situation and intervene. + +If you end up in a situation where you have lost the master and can not get it +back online (or can not restore it quickly) you can promote a replica to become +the new master. See the next section, "Promoting a Replica", for details. + + +Promoting a Replica +=================== + +TODO: Write this, too. + + +Unreachable Masters +=================== + +This section describes how Phabricator determines that a master has been lost, +marks it unreachable, and degrades into read-only mode. + +TODO: For now, it doesn't. + + +Backups +====== + +Even if you configure replication, you should still retain separate backup +snapshots. Replicas protect you from data loss if you lose a host, but they do +not let you recover from data mutation mistakes. + +If something issues `DELETE` or `UPDATE` statements and destroys data on the +master, the mutation will propagate to the replicas almost immediately and the +data will be gone forever. Normally, the only way to recover this data is from +backup snapshots. + +Although you should still have a backup process, your backup process can +safely pull dumps from a replica instead of the master. This operation can +be slow, so offloading it to a replica can make the perforance of the master +more consistent. + +To dump from a replica, wait for this TODO to be resolved and then do whatever +it says to do: + +TODO: Make `bin/storage dump` replica-aware. See T10758. + + +Next Steps +========== + +Continue by: + + - returning to @{article:Clustering Introduction}. diff --git a/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php @@ -0,0 +1,98 @@ + $spec) { + if (!is_array($spec)) { + throw new Exception( + pht( + 'Database cluster configuration is not valid: each entry in the '. + 'list must be a dictionary describing a database host, but '. + 'the value with index "%s" is not a dictionary.', + $index)); + } + } + + $masters = array(); + $map = array(); + foreach ($value as $index => $spec) { + try { + PhutilTypeSpec::checkMap( + $spec, + array( + 'host' => 'string', + 'role' => 'string', + 'port' => 'optional int', + 'user' => 'optional string', + 'pass' => 'optional string', + 'disabled' => 'optional bool', + )); + } catch (Exception $ex) { + throw new Exception( + pht( + 'Database cluster configuration has an invalid host '. + 'specification (at index "%s"): %s.', + $index, + $ex->getMessage())); + } + + $role = $spec['role']; + $host = $spec['host']; + $port = idx($spec, 'port'); + + switch ($role) { + case 'master': + case 'replica': + break; + default: + throw new Exception( + pht( + 'Database cluster configuration describes an invalid '. + 'host ("%s", at index "%s") with an unrecognized role ("%s"). '. + 'Valid roles are "%s" or "%s".', + $spec['host'], + $index, + $spec['role'], + 'master', + 'replica')); + } + + if ($role === 'master') { + $masters[] = $host; + } + + // We can't guarantee that you didn't just give the same host two + // different names in DNS, but this check can catch silly copy/paste + // mistakes. + $key = "{$host}:{$port}"; + if (isset($map[$key])) { + throw new Exception( + pht( + 'Database cluster configuration is invalid: it describes the '. + 'same host ("%s") multiple times. Each host should appear only '. + 'once in the list.', + $host)); + } + $map[$key] = true; + } + + if (count($masters) > 1) { + throw new Exception( + pht( + 'Database cluster configuration is invalid: it describes multiple '. + 'masters. No more than one host may be a master. Hosts currently '. + 'configured as masters: %s.', + implode(', ', $masters))); + } + } + +} diff --git a/src/infrastructure/markup/PhabricatorMarkupEngine.php b/src/infrastructure/markup/PhabricatorMarkupEngine.php --- a/src/infrastructure/markup/PhabricatorMarkupEngine.php +++ b/src/infrastructure/markup/PhabricatorMarkupEngine.php @@ -254,6 +254,8 @@ } } + $is_readonly = PhabricatorEnv::isReadOnly(); + foreach ($objects as $key => $info) { // False check in case MySQL doesn't support unicode characters // in the string (T1191), resulting in unserialize returning false. @@ -279,7 +281,7 @@ ->setCacheData($data) ->setMetadata($metadata); - if (isset($use_cache[$key])) { + if (isset($use_cache[$key]) && !$is_readonly) { // This is just filling a cache and always safe, even on a read pathway. $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); $blocks[$key]->replace(); diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php --- a/src/view/page/PhabricatorStandardPageView.php +++ b/src/view/page/PhabricatorStandardPageView.php @@ -276,7 +276,7 @@ Javelin::initBehavior( 'read-only-warning', array( - 'message' => pht('This install is currently in read-only mode.'), + 'message' => pht('Phabricator is currently in read-only mode.'), )); }