diff --git a/resources/sql/autopatches/20161121.cluster.01.hoststate.sql b/resources/sql/autopatches/20161121.cluster.01.hoststate.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20161121.cluster.01.hoststate.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_meta_data.hoststate ( + stateKey VARCHAR(128) NOT NULL COLLATE {$COLLATE_TEXT}, + stateValue LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}, + PRIMARY KEY (stateKey) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3795,6 +3795,7 @@ 'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php', 'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php', 'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php', + 'PhabricatorStorageManagementPartitionWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php', 'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php', 'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php', 'PhabricatorStorageManagementRenamespaceWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php', @@ -8975,6 +8976,7 @@ 'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow', + 'PhabricatorStorageManagementPartitionWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow', 'PhabricatorStorageManagementRenamespaceWorkflow' => 'PhabricatorStorageManagementWorkflow', diff --git a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php --- a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php +++ b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php @@ -205,6 +205,38 @@ break; } + // If we have more than one master, we require that the cluster database + // configuration written to each database node is exactly the same as the + // one we are running with. + $masters = PhabricatorDatabaseRef::getAllMasterDatabaseRefs(); + if (count($masters) > 1) { + $state_actual = queryfx_one( + $conn_meta, + 'SELECT stateValue FROM %T WHERE stateKey = %s', + PhabricatorStorageManagementAPI::TABLE_HOSTSTATE, + 'cluster.databases'); + if ($state_actual) { + $state_actual = $state_actual['stateValue']; + } + + $state_expect = $ref->getPartitionStateForCommit(); + if ($state_expect !== $state_actual) { + $message = pht( + 'Database host "%s" has a configured cluster state which disagrees '. + 'with the state on this host ("%s"). Run `bin/storage partition` '. + 'to commit local state to the cluster. This host may have started '. + 'with an out-of-date configuration.', + $ref->getRefKey(), + php_uname('n')); + + $this->newIssue('db.state.desync') + ->setName(pht('Cluster Configuration Out of Sync')) + ->setMessage($message) + ->setIsFatal(true); + return true; + } + } } + } diff --git a/src/docs/user/cluster/cluster_partitioning.diviner b/src/docs/user/cluster/cluster_partitioning.diviner --- a/src/docs/user/cluster/cluster_partitioning.diviner +++ b/src/docs/user/cluster/cluster_partitioning.diviner @@ -123,6 +123,21 @@ names. You can get a list of databases with `bin/storage databases` to identify the correct database names. +After you have configured partitioning, it needs to be committed to the +databases. This writes a copy of the configuration to tables on the databases, +preventing errors if a webserver accidentally starts with an old or invalid +configuration. + +To commit the configuration, run this command: + +``` +phabricator/ $ ./bin/storage partition +``` + +Run this command after making any partition or clustering changes. Webservers +will not serve traffic if their configuration and the database configuration +differ. + Launching a new Partition ========================= @@ -135,6 +150,7 @@ are partitioning, you will need to configure your existing master as the new "default". This will let Phabricator interact with it, but won't send any traffic to it yet. + - Run `bin/storage partition`. - Run `bin/storage upgrade` to initialize the schemata on the new hosts. - Stop writes to the applications you want to move by putting Phabricator in read-only mode, or shutting down the webserver and daemons, or telling @@ -143,6 +159,7 @@ - Load the data into the application databases on the new master. - Reconfigure the "partition" setup so that Phabricator knows the databases have moved. + - Run `bin/storage partition`. - While still in read-only mode, check that all the data appears to be intact. - Resume writes. diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -180,6 +180,18 @@ return $this->applicationMap; } + public function getPartitionStateForCommit() { + $state = PhabricatorEnv::getEnvConfig('cluster.databases'); + foreach ($state as $key => $value) { + // Don't store passwords, since we don't care if they differ and + // users may find it surprising. + unset($state[$key]['pass']); + } + ksort($state); + + return phutil_json_encode($state); + } + public function setMasterRef(PhabricatorDatabaseRef $master_ref) { $this->masterRef = $master_ref; return $this; @@ -498,9 +510,6 @@ $masters = array(); foreach ($refs as $ref) { - if ($ref->getDisabled()) { - continue; - } if ($ref->getIsMaster()) { $masters[] = $ref; } diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php --- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php +++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php @@ -19,6 +19,7 @@ const COLLATE_FULLTEXT = 'COLLATE_FULLTEXT'; const TABLE_STATUS = 'patch_status'; + const TABLE_HOSTSTATE = 'hoststate'; public function setDisableUTF8MB4($disable_utf8_mb4) { $this->disableUTF8MB4 = $disable_utf8_mb4; diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php @@ -0,0 +1,44 @@ +setName('partition') + ->setExamples('**partition** [__options__]') + ->setSynopsis(pht('Commit partition configuration to databases.')) + ->setArguments(array()); + } + + public function didExecute(PhutilArgumentParser $args) { + echo tsprintf( + "%s\n", + pht('Committing configured partition map to databases...')); + + foreach ($this->getMasterAPIs() as $api) { + $ref = $api->getRef(); + $conn = $ref->newManagementConnection(); + + $state = $ref->getPartitionStateForCommit(); + + queryfx( + $conn, + 'INSERT INTO %T.%T (stateKey, stateValue) VALUES (%s, %s) + ON DUPLICATE KEY UPDATE stateValue = VALUES(stateValue)', + $api->getDatabaseName('meta_data'), + PhabricatorStorageManagementAPI::TABLE_HOSTSTATE, + 'cluster.databases', + $state); + + echo tsprintf( + "%s\n", + pht( + 'Wrote configuration on database host "%s".', + $ref->getRefKey())); + } + + return 0; + } + +} diff --git a/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php b/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php --- a/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php +++ b/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php @@ -18,6 +18,20 @@ 'unique' => true, ), )); + + $this->buildRawSchema( + 'meta_data', + PhabricatorStorageManagementAPI::TABLE_HOSTSTATE, + array( + 'stateKey' => 'text128', + 'stateValue' => 'text', + ), + array( + 'PRIMARY' => array( + 'columns' => array('stateKey'), + 'unique' => true, + ), + )); } }