diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1989,7 +1989,9 @@ 'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php', 'PhabricatorClusterException' => 'infrastructure/cluster/PhabricatorClusterException.php', 'PhabricatorClusterExceptionHandler' => 'infrastructure/cluster/PhabricatorClusterExceptionHandler.php', + 'PhabricatorClusterImpossibleWriteException' => 'infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php', 'PhabricatorClusterImproperWriteException' => 'infrastructure/cluster/PhabricatorClusterImproperWriteException.php', + 'PhabricatorClusterStrandedException' => 'infrastructure/cluster/PhabricatorClusterStrandedException.php', 'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php', 'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php', 'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php', @@ -6402,7 +6404,9 @@ 'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType', 'PhabricatorClusterException' => 'Exception', 'PhabricatorClusterExceptionHandler' => 'PhabricatorRequestExceptionHandler', + 'PhabricatorClusterImpossibleWriteException' => 'PhabricatorClusterException', 'PhabricatorClusterImproperWriteException' => 'PhabricatorClusterException', + 'PhabricatorClusterStrandedException' => 'PhabricatorClusterException', 'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField', 'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension', 'PhabricatorCommentEditField' => 'PhabricatorEditField', diff --git a/src/applications/system/controller/PhabricatorSystemReadOnlyController.php b/src/applications/system/controller/PhabricatorSystemReadOnlyController.php --- a/src/applications/system/controller/PhabricatorSystemReadOnlyController.php +++ b/src/applications/system/controller/PhabricatorSystemReadOnlyController.php @@ -8,6 +8,7 @@ } public function handleRequest(AphrontRequest $request) { + $viewer = $this->getViewer(); $reason = $request->getURIData('reason'); $body = array(); @@ -48,15 +49,77 @@ phutil_tag('tt', array(), 'cluster.databases')); $button = pht('Wait Patiently'); break; + case PhabricatorEnv::READONLY_UNREACHABLE: + $title = pht('Unable to Reach Master'); + $body[] = pht( + 'Phabricator was unable to connect to the writable ("master") '. + 'database while handling this request, and automatically degraded '. + 'into read-only mode.'); + $body[] = pht( + 'This may happen if there is a temporary network anomaly on the '. + 'server side, like cosmic radiation or spooky ghosts. If this '. + 'failure was caused by a transient service interruption, '. + 'Phabricator will recover momentarily.'); + $body[] = pht( + 'This may also indicate that a more serious failure has occurred. '. + 'If this interruption does not resolve on its own, Phabricator '. + 'will soon detect the persistent disruption and degrade into '. + 'read-only mode until the issue is resolved.'); + $button = pht('Quite Unsettling'); + break; + case PhabricatorEnv::READONLY_SEVERED: + $title = pht('Severed From Master'); + $body[] = pht( + 'Phabricator has consistently been unable to reach the writable '. + '("master") database while processing recent requests.'); + $body[] = pht( + 'This likely indicates a severe misconfiguration or major service '. + 'interruption.'); + $body[] = pht( + 'Phabricator will periodically retry the connection and recover '. + 'once service is restored. Most causes of persistent service '. + 'interruption will require administrative intervention in order '. + 'to restore service.'); + $body[] = pht( + 'Although this may be the result of a misconfiguration or '. + 'operational error, this is also the state you reach if a '. + 'meteor recently obliterated a datacenter.'); + $button = pht('Panic!'); + break; default: return new Aphront404Response(); } + switch ($reason) { + case PhabricatorEnv::READONLY_UNREACHABLE: + case PhabricatorEnv::READONLY_SEVERED: + $body[] = pht( + 'This request was served from a replica database. Replica '. + 'databases may lag behind the master, so very recent activity '. + 'may not be reflected in the UI. This data will be restored if '. + 'the master database is restored, but may have been lost if the '. + 'master database has been reduced to a pile of ash.'); + break; + } + $body[] = pht( 'In read-only mode you can read existing information, but you will not '. 'be able to edit objects or create new objects until this mode is '. 'disabled.'); + if ($viewer->getIsAdmin()) { + $body[] = pht( + 'As an administrator, you can review status information from the '. + '%s control panel. This may provide more information about the '. + 'current state of affairs.', + phutil_tag( + 'a', + array( + 'href' => '/config/cluster/databases/', + ), + pht('Cluster Database Status'))); + } + $dialog = $this->newDialog() ->setTitle($title) ->setWidth(AphrontDialogView::WIDTH_FORM) diff --git a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php b/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php --- a/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php +++ b/src/infrastructure/cluster/PhabricatorClusterExceptionHandler.php @@ -25,11 +25,15 @@ $title = $ex->getExceptionTitle(); - return id(new AphrontDialogView()) + $dialog = id(new AphrontDialogView()) ->setTitle($title) ->setUser($viewer) ->appendParagraph($ex->getMessage()) ->addCancelButton('/', pht('Proceed With Caution')); + + return id(new AphrontDialogResponse()) + ->setDialog($dialog) + ->setHTTPResponseCode(500); } } diff --git a/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php b/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cluster/PhabricatorClusterImpossibleWriteException.php @@ -0,0 +1,10 @@ +host = $host; return $this; @@ -190,7 +194,19 @@ ); } - public static function loadAll() { + public static function getLiveRefs() { + $cache = PhabricatorCaches::getRequestCache(); + + $refs = $cache->getKey(self::KEY_REFS); + if (!$refs) { + $refs = self::newRefs(); + $cache->setKey(self::KEY_REFS, $refs); + } + + return $refs; + } + + public static function newRefs() { $refs = array(); $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); @@ -232,7 +248,7 @@ } public static function queryAll() { - $refs = self::loadAll(); + $refs = self::newRefs(); foreach ($refs as $ref) { if ($ref->getDisabled()) { @@ -242,6 +258,7 @@ $conn = $ref->newManagementConnection(); $t_start = microtime(true); + $replica_status = false; try { $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS'); $ref->setConnectionStatus(self::STATUS_OKAY); @@ -269,33 +286,35 @@ $t_end = microtime(true); $ref->setConnectionLatency($t_end - $t_start); - $is_replica = (bool)$replica_status; - if ($ref->getIsMaster() && $is_replica) { - $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA); - $ref->setReplicaMessage( - pht( - 'This host has a "master" role, but is replicating data from '. - 'another host ("%s")!', - idx($replica_status, 'Master_Host'))); - } else if (!$ref->getIsMaster() && !$is_replica) { - $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE); - $ref->setReplicaMessage( - pht( - 'This host has a "replica" role, but is not replicating data '. - 'from a master (no output from "SHOW SLAVE STATUS").')); - } else { - $ref->setReplicaStatus(self::REPLICATION_OKAY); - } - - if ($is_replica) { - $latency = (int)idx($replica_status, 'Seconds_Behind_Master'); - $ref->setReplicaDelay($latency); - if ($latency > 30) { - $ref->setReplicaStatus(self::REPLICATION_SLOW); + if ($replica_status !== false) { + $is_replica = (bool)$replica_status; + if ($ref->getIsMaster() && $is_replica) { + $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA); $ref->setReplicaMessage( pht( - 'This replica is lagging far behind the master. Data is at '. - 'risk!')); + 'This host has a "master" role, but is replicating data from '. + 'another host ("%s")!', + idx($replica_status, 'Master_Host'))); + } else if (!$ref->getIsMaster() && !$is_replica) { + $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE); + $ref->setReplicaMessage( + pht( + 'This host has a "replica" role, but is not replicating data '. + 'from a master (no output from "SHOW SLAVE STATUS").')); + } else { + $ref->setReplicaStatus(self::REPLICATION_OKAY); + } + + if ($is_replica) { + $latency = (int)idx($replica_status, 'Seconds_Behind_Master'); + $ref->setReplicaDelay($latency); + if ($latency > 30) { + $ref->setReplicaStatus(self::REPLICATION_SLOW); + $ref->setReplicaMessage( + pht( + 'This replica is lagging far behind the master. Data is at '. + 'risk!')); + } } } } @@ -318,8 +337,31 @@ )); } + public function isSevered() { + return $this->didFailToConnect; + } + + public function isReachable(AphrontDatabaseConnection $connection) { + if ($this->isSevered()) { + return false; + } + + try { + $connection->openConnection(); + $reachable = true; + } catch (Exception $ex) { + $reachable = false; + } + + if (!$reachable) { + $this->didFailToConnect = true; + } + + return $reachable; + } + public static function getMasterDatabaseRef() { - $refs = self::loadAll(); + $refs = self::getLiveRefs(); if (!$refs) { $conf = PhabricatorEnv::newObjectFromConfig( @@ -348,7 +390,7 @@ } public static function getReplicaDatabaseRef() { - $refs = self::loadAll(); + $refs = self::getLiveRefs(); if (!$refs) { return null; diff --git a/src/infrastructure/env/PhabricatorEnv.php b/src/infrastructure/env/PhabricatorEnv.php --- a/src/infrastructure/env/PhabricatorEnv.php +++ b/src/infrastructure/env/PhabricatorEnv.php @@ -60,6 +60,8 @@ private static $readOnlyReason; const READONLY_CONFIG = 'config'; + const READONLY_UNREACHABLE = 'unreachable'; + const READONLY_SEVERED = 'severed'; const READONLY_MASTERLESS = 'masterless'; /** @@ -217,6 +219,8 @@ $master = PhabricatorDatabaseRef::getMasterDatabaseRef(); if (!$master) { self::setReadOnly(true, self::READONLY_MASTERLESS); + } else if ($master->isSevered()) { + self::setReadOnly(true, self::READONLY_SEVERED); } try { @@ -468,6 +472,12 @@ return pht( 'Phabricator is in read-only mode (no writable database '. 'is configured).'); + case self::READONLY_UNREACHABLE: + return pht( + 'Phabricator is in read-only mode (unreachable master).'); + case self::READONLY_SEVERED: + return pht( + 'Phabricator is in read-only mode (major interruption).'); } return pht('Phabricator is in read-only mode.'); diff --git a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php --- a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php +++ b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php @@ -60,8 +60,8 @@ $this->raiseImproperWrite($database); } - $refs = PhabricatorDatabaseRef::loadAll(); - if ($refs) { + $is_cluster = (bool)PhabricatorEnv::getEnvConfig('cluster.databases'); + if ($is_cluster) { $connection = $this->newClusterConnection($database, $mode); } else { $connection = $this->newBasicConnection($database, $mode, $namespace); @@ -99,8 +99,19 @@ private function newClusterConnection($database, $mode) { $master = PhabricatorDatabaseRef::getMasterDatabaseRef(); - if ($master) { - return $master->newApplicationConnection($database); + + if ($master && !$master->isSevered()) { + $connection = $master->newApplicationConnection($database); + if ($master->isReachable($connection)) { + return $connection; + } else { + if ($mode == 'w') { + $this->raiseImpossibleWrite($database); + } + PhabricatorEnv::setReadOnly( + true, + PhabricatorEnv::READONLY_UNREACHABLE); + } } $replica = PhabricatorDatabaseRef::getReplicaDatabaseRef(); @@ -111,8 +122,11 @@ $connection = $replica->newApplicationConnection($database); $connection->setReadOnly(true); + if ($replica->isReachable($connection)) { + return $connection; + } - return $connection; + $this->raiseUnreachable($database); } private function raiseImproperWrite($database) { @@ -124,6 +138,23 @@ $database)); } + private function raiseImpossibleWrite($database) { + throw new PhabricatorClusterImpossibleWriteException( + pht( + 'Unable to connect to master database ("%s"). This is a severe '. + 'failure; your request did not complete.', + $database)); + } + + private function raiseUnreachable($database) { + throw new PhabricatorClusterStrandedException( + pht( + 'Unable to establish a connection to ANY database host '. + '(while trying "%s"). All masters and replicas are completely '. + 'unreachable.', + $database)); + } + /** * @task config