diff --git a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php index 8937c1f205..4bb9ca6255 100644 --- a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php +++ b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php @@ -1,765 +1,764 @@ repository = $repository; return $this; } public function getRepository() { return $this->repository; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setLog(DiffusionRepositoryClusterEngineLogInterface $log) { $this->logger = $log; return $this; } /* -( Cluster Synchronization )-------------------------------------------- */ /** * Synchronize repository version information after creating a repository. * * This initializes working copy versions for all currently bound devices to * 0, so that we don't get stuck making an ambiguous choice about which * devices are leaders when we later synchronize before a read. * * @task sync */ public function synchronizeWorkingCopyAfterCreation() { if (!$this->shouldEnableSynchronization()) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $bindings = $service->getActiveBindings(); foreach ($bindings as $binding) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $binding->getDevicePHID(), 0); } return $this; } /** * @task sync */ public function synchronizeWorkingCopyAfterHostingChange() { if (!$this->shouldEnableSynchronization()) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); // After converting a hosted repository to observed, or vice versa, we // need to reset version numbers because the clocks for observed and hosted // repositories run on different units. // We identify all the cluster leaders and reset their version to 0. // We identify all the cluster followers and demote them. // This allows the cluster to start over again at version 0 but keep the // same leaders. if ($versions) { $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); foreach ($versions as $version) { $device_phid = $version->getDevicePHID(); if ($version->getRepositoryVersion() == $max_version) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, 0); } else { PhabricatorRepositoryWorkingCopyVersion::demoteDevice( $repository_phid, $device_phid); } } } return $this; } /** * @task sync */ public function synchronizeWorkingCopyBeforeRead() { if (!$this->shouldEnableSynchronization()) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); $read_lock = PhabricatorRepositoryWorkingCopyVersion::getReadLock( $repository_phid, $device_phid); $lock_wait = phutil_units('2 minutes in seconds'); $this->logLine( pht( 'Waiting up to %s second(s) for a cluster read lock on "%s"...', new PhutilNumber($lock_wait), $device->getName())); try { $start = PhabricatorTime::getNow(); $read_lock->lock($lock_wait); $waited = (PhabricatorTime::getNow() - $start); if ($waited) { $this->logLine( pht( 'Acquired read lock after %s second(s).', new PhutilNumber($waited))); } else { $this->logLine( pht( 'Acquired read lock immediately.')); } } catch (Exception $ex) { throw new PhutilProxyException( pht( 'Failed to acquire read lock after waiting %s second(s). You '. 'may be able to retry later.', new PhutilNumber($lock_wait)), $ex); } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); $this_version = idx($versions, $device_phid); if ($this_version) { $this_version = (int)$this_version->getRepositoryVersion(); } else { $this_version = -1; } if ($versions) { // This is the normal case, where we have some version information and // can identify which nodes are leaders. If the current node is not a // leader, we want to fetch from a leader and then update our version. $max_version = (int)max(mpull($versions, 'getRepositoryVersion')); if ($max_version > $this_version) { if ($repository->isHosted()) { $fetchable = array(); foreach ($versions as $version) { if ($version->getRepositoryVersion() == $max_version) { $fetchable[] = $version->getDevicePHID(); } } $this->synchronizeWorkingCopyFromDevices($fetchable); } else { $this->synchornizeWorkingCopyFromRemote(); } PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, $max_version); } else { $this->logLine( pht( 'Device "%s" is already a cluster leader and does not need '. 'to be synchronized.', $device->getName())); } $result_version = $max_version; } else { // If no version records exist yet, we need to be careful, because we // can not tell which nodes are leaders. // There might be several nodes with arbitrary existing data, and we have // no way to tell which one has the "right" data. If we pick wrong, we // might erase some or all of the data in the repository. // Since this is dangeorus, we refuse to guess unless there is only one // device. If we're the only device in the group, we obviously must be // a leader. $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $bindings = $service->getActiveBindings(); $device_map = array(); foreach ($bindings as $binding) { $device_map[$binding->getDevicePHID()] = true; } if (count($device_map) > 1) { throw new Exception( pht( 'Repository "%s" exists on more than one device, but no device '. 'has any repository version information. Phabricator can not '. - 'guess which copy of the existing data is authoritative. Remove '. - 'all but one device from service to mark the remaining device '. - 'as the authority.', + 'guess which copy of the existing data is authoritative. Promote '. + 'a device or see "Ambigous Leaders" in the documentation.', $repository->getDisplayName())); } if (empty($device_map[$device->getPHID()])) { throw new Exception( pht( 'Repository "%s" is being synchronized on device "%s", but '. 'this device is not bound to the corresponding cluster '. 'service ("%s").', $repository->getDisplayName(), $device->getName(), $service->getName())); } // The current device is the only device in service, so it must be a // leader. We can safely have any future nodes which come online read // from it. PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, 0); $result_version = 0; } $read_lock->unlock(); return $result_version; } /** * @task sync */ public function synchronizeWorkingCopyBeforeWrite() { if (!$this->shouldEnableSynchronization()) { return; } $repository = $this->getRepository(); $viewer = $this->getViewer(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); $table = new PhabricatorRepositoryWorkingCopyVersion(); $locked_connection = $table->establishConnection('w'); $write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock( $repository_phid); $write_lock->useSpecificConnection($locked_connection); $lock_wait = phutil_units('2 minutes in seconds'); $this->logLine( pht( 'Waiting up to %s second(s) for a cluster write lock...', new PhutilNumber($lock_wait))); try { $start = PhabricatorTime::getNow(); $write_lock->lock($lock_wait); $waited = (PhabricatorTime::getNow() - $start); if ($waited) { $this->logLine( pht( 'Acquired write lock after %s second(s).', new PhutilNumber($waited))); } else { $this->logLine( pht( 'Acquired write lock immediately.')); } } catch (Exception $ex) { throw new PhutilProxyException( pht( 'Failed to acquire write lock after waiting %s second(s). You '. 'may be able to retry later.', new PhutilNumber($lock_wait))); } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); foreach ($versions as $version) { if (!$version->getIsWriting()) { continue; } throw new Exception( pht( 'An previous write to this repository was interrupted; refusing '. 'new writes. This issue requires operator intervention to resolve, '. 'see "Write Interruptions" in the "Cluster: Repositories" in the '. 'documentation for instructions.')); } try { $max_version = $this->synchronizeWorkingCopyBeforeRead(); } catch (Exception $ex) { $write_lock->unlock(); throw $ex; } $pid = getmypid(); $hash = Filesystem::readRandomCharacters(12); $this->clusterWriteOwner = "{$pid}.{$hash}"; PhabricatorRepositoryWorkingCopyVersion::willWrite( $locked_connection, $repository_phid, $device_phid, array( 'userPHID' => $viewer->getPHID(), 'epoch' => PhabricatorTime::getNow(), 'devicePHID' => $device_phid, ), $this->clusterWriteOwner); $this->clusterWriteVersion = $max_version; $this->clusterWriteLock = $write_lock; } public function synchronizeWorkingCopyAfterDiscovery($new_version) { if (!$this->shouldEnableSynchronization()) { return; } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); if ($repository->isHosted()) { return; } $viewer = $this->getViewer(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); // NOTE: We are not holding a lock here because this method is only called // from PhabricatorRepositoryDiscoveryEngine, which already holds a device // lock. Even if we do race here and record an older version, the // consequences are mild: we only do extra work to correct it later. $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); $versions = mpull($versions, null, 'getDevicePHID'); $this_version = idx($versions, $device_phid); if ($this_version) { $this_version = (int)$this_version->getRepositoryVersion(); } else { $this_version = -1; } if ($new_version > $this_version) { PhabricatorRepositoryWorkingCopyVersion::updateVersion( $repository_phid, $device_phid, $new_version); } } /** * @task sync */ public function synchronizeWorkingCopyAfterWrite() { if (!$this->shouldEnableSynchronization()) { return; } if (!$this->clusterWriteLock) { throw new Exception( pht( 'Trying to synchronize after write, but not holding a write '. 'lock!')); } $repository = $this->getRepository(); $repository_phid = $repository->getPHID(); $device = AlmanacKeys::getLiveDevice(); $device_phid = $device->getPHID(); // It is possible that we've lost the global lock while receiving the push. // For example, the master database may have been restarted between the // time we acquired the global lock and now, when the push has finished. // We wrote a durable lock while we were holding the the global lock, // essentially upgrading our lock. We can still safely release this upgraded // lock even if we're no longer holding the global lock. // If we fail to release the lock, the repository will be frozen until // an operator can figure out what happened, so we try pretty hard to // reconnect to the database and release the lock. $now = PhabricatorTime::getNow(); $duration = phutil_units('5 minutes in seconds'); $try_until = $now + $duration; $did_release = false; $already_failed = false; while (PhabricatorTime::getNow() <= $try_until) { try { // NOTE: This means we're still bumping the version when pushes fail. We // could select only un-rejected events instead to bump a little less // often. $new_log = id(new PhabricatorRepositoryPushEventQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withRepositoryPHIDs(array($repository_phid)) ->setLimit(1) ->executeOne(); $old_version = $this->clusterWriteVersion; if ($new_log) { $new_version = $new_log->getID(); } else { $new_version = $old_version; } PhabricatorRepositoryWorkingCopyVersion::didWrite( $repository_phid, $device_phid, $this->clusterWriteVersion, $new_version, $this->clusterWriteOwner); $did_release = true; break; } catch (AphrontConnectionQueryException $ex) { $connection_exception = $ex; } catch (AphrontConnectionLostQueryException $ex) { $connection_exception = $ex; } if (!$already_failed) { $already_failed = true; $this->logLine( pht('CRITICAL. Failed to release cluster write lock!')); $this->logLine( pht( 'The connection to the master database was lost while receiving '. 'the write.')); $this->logLine( pht( 'This process will spend %s more second(s) attempting to '. 'recover, then give up.', new PhutilNumber($duration))); } sleep(1); } if ($did_release) { if ($already_failed) { $this->logLine( pht('RECOVERED. Link to master database was restored.')); } $this->logLine(pht('Released cluster write lock.')); } else { throw new Exception( pht( 'Failed to reconnect to master database and release held write '. 'lock ("%s") on device "%s" for repository "%s" after trying '. 'for %s seconds(s). This repository will be frozen.', $this->clusterWriteOwner, $device->getName(), $this->getDisplayName(), new PhutilNumber($duration))); } // We can continue even if we've lost this lock, everything is still // consistent. try { $this->clusterWriteLock->unlock(); } catch (Exception $ex) { // Ignore. } $this->clusterWriteLock = null; $this->clusterWriteOwner = null; } /* -( Internals )---------------------------------------------------------- */ /** * @task internal */ private function shouldEnableSynchronization() { $repository = $this->getRepository(); $service_phid = $repository->getAlmanacServicePHID(); if (!$service_phid) { return false; } // TODO: For now, this is only supported for Git. if (!$repository->isGit()) { return false; } $device = AlmanacKeys::getLiveDevice(); if (!$device) { return false; } return true; } /** * @task internal */ private function synchornizeWorkingCopyFromRemote() { $repository = $this->getRepository(); $device = AlmanacKeys::getLiveDevice(); $local_path = $repository->getLocalPath(); $fetch_uri = $repository->getRemoteURIEnvelope(); if ($repository->isGit()) { $this->requireWorkingCopy(); $argv = array( 'fetch --prune -- %P %s', $fetch_uri, '+refs/*:refs/*', ); } else { throw new Exception(pht('Remote sync only supported for git!')); } $future = DiffusionCommandEngine::newCommandEngine($repository) ->setArgv($argv) ->setSudoAsDaemon(true) ->setCredentialPHID($repository->getCredentialPHID()) ->setURI($repository->getRemoteURIObject()) ->newFuture(); $future->setCWD($local_path); try { $future->resolvex(); } catch (Exception $ex) { $this->logLine( pht( 'Synchronization of "%s" from remote failed: %s', $device->getName(), $ex->getMessage())); throw $ex; } } /** * @task internal */ private function synchronizeWorkingCopyFromDevices(array $device_phids) { $repository = $this->getRepository(); $service = $repository->loadAlmanacService(); if (!$service) { throw new Exception(pht('Failed to load repository cluster service.')); } $device_map = array_fuse($device_phids); $bindings = $service->getActiveBindings(); $fetchable = array(); foreach ($bindings as $binding) { // We can't fetch from nodes which don't have the newest version. $device_phid = $binding->getDevicePHID(); if (empty($device_map[$device_phid])) { continue; } // TODO: For now, only fetch over SSH. We could support fetching over // HTTP eventually. if ($binding->getAlmanacPropertyValue('protocol') != 'ssh') { continue; } $fetchable[] = $binding; } if (!$fetchable) { throw new Exception( pht( 'Leader lost: no up-to-date nodes in repository cluster are '. 'fetchable.')); } $caught = null; foreach ($fetchable as $binding) { try { $this->synchronizeWorkingCopyFromBinding($binding); $caught = null; break; } catch (Exception $ex) { $caught = $ex; } } if ($caught) { throw $caught; } } /** * @task internal */ private function synchronizeWorkingCopyFromBinding($binding) { $repository = $this->getRepository(); $device = AlmanacKeys::getLiveDevice(); $this->logLine( pht( 'Synchronizing this device ("%s") from cluster leader ("%s") before '. 'read.', $device->getName(), $binding->getDevice()->getName())); $fetch_uri = $repository->getClusterRepositoryURIFromBinding($binding); $local_path = $repository->getLocalPath(); if ($repository->isGit()) { $this->requireWorkingCopy(); $argv = array( 'fetch --prune -- %s %s', $fetch_uri, '+refs/*:refs/*', ); } else { throw new Exception(pht('Binding sync only supported for git!')); } $future = DiffusionCommandEngine::newCommandEngine($repository) ->setArgv($argv) ->setConnectAsDevice(true) ->setSudoAsDaemon(true) ->setURI($fetch_uri) ->newFuture(); $future->setCWD($local_path); try { $future->resolvex(); } catch (Exception $ex) { $this->logLine( pht( 'Synchronization of "%s" from leader "%s" failed: %s', $device->getName(), $binding->getDevice()->getName(), $ex->getMessage())); throw $ex; } } /** * @task internal */ private function logLine($message) { return $this->logText("# {$message}\n"); } /** * @task internal */ private function logText($message) { $log = $this->logger; if ($log) { $log->writeClusterEngineLogMessage($message); } return $this; } private function requireWorkingCopy() { $repository = $this->getRepository(); $local_path = $repository->getLocalPath(); if (!Filesystem::pathExists($local_path)) { $device = AlmanacKeys::getLiveDevice(); throw new Exception( pht( 'Repository "%s" does not have a working copy on this device '. 'yet, so it can not be synchronized. Wait for the daemons to '. 'construct one or run `bin/repository update %s` on this host '. '("%s") to build it explicitly.', $repository->getDisplayName(), $repository->getMonogram(), $device->getName())); } } } diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementClusterizeWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementClusterizeWorkflow.php index 7424b84eae..7178564067 100644 --- a/src/applications/repository/management/PhabricatorRepositoryManagementClusterizeWorkflow.php +++ b/src/applications/repository/management/PhabricatorRepositoryManagementClusterizeWorkflow.php @@ -1,117 +1,150 @@ setName('clusterize') ->setExamples('**clusterize** [options] __repository__ ...') ->setSynopsis( pht('Convert existing repositories into cluster repositories.')) ->setArguments( array( array( 'name' => 'service', 'param' => 'service', 'help' => pht( 'Cluster repository service in Almanac to move repositories '. 'into.'), ), array( 'name' => 'remove-service', 'help' => pht('Take repositories out of a cluster.'), ), array( 'name' => 'repositories', 'wildcard' => true, ), )); } public function execute(PhutilArgumentParser $args) { $viewer = $this->getViewer(); $repositories = $this->loadRepositories($args, 'repositories'); if (!$repositories) { throw new PhutilArgumentUsageException( pht('Specify one or more repositories to clusterize.')); } $service_name = $args->getArg('service'); $remove_service = $args->getArg('remove-service'); if ($remove_service && $service_name) { throw new PhutilArgumentUsageException( pht('Specify --service or --remove-service, but not both.')); } if (!$service_name && !$remove_service) { throw new PhutilArgumentUsageException( pht('Specify --service or --remove-service.')); } if ($remove_service) { $service = null; } else { $service = id(new AlmanacServiceQuery()) ->setViewer($viewer) ->withNames(array($service_name)) ->withServiceTypes( array( AlmanacClusterRepositoryServiceType::SERVICETYPE, )) + ->needBindings(true) ->executeOne(); if (!$service) { throw new PhutilArgumentUsageException( pht( 'No repository service "%s" exists.', $service_name)); } } - if ($service) { $service_phid = $service->getPHID(); + + $bindings = $service->getActiveBindings(); + + $unique_devices = array(); + foreach ($bindings as $binding) { + $unique_devices[$binding->getDevicePHID()] = $binding->getDevice(); + } + + if (count($unique_devices) > 1) { + $device_names = mpull($unique_devices, 'getName'); + + echo id(new PhutilConsoleBlock()) + ->addParagraph( + pht( + 'Service "%s" is actively bound to more than one device (%s).', + $service_name, + implode(', ', $device_names))) + ->addParagraph( + pht( + 'If you clusterize a repository onto this service it may be '. + 'unclear which devices have up-to-date copies of the '. + 'repository. If so, leader/follower ambiguity will freeze the '. + 'repository. You may need to manually promote a device to '. + 'unfreeze it. See "Ambiguous Leaders" in the documentation '. + 'for discussion.')) + ->drawConsoleString(); + + $prompt = pht('Continue anyway?'); + if (!phutil_console_confirm($prompt)) { + throw new PhutilArgumentUsageException( + pht('User aborted the workflow.')); + } + } } else { $service_phid = null; } $content_source = $this->newContentSource(); $diffusion_phid = id(new PhabricatorDiffusionApplication())->getPHID(); foreach ($repositories as $repository) { $xactions = array(); $xactions[] = id(new PhabricatorRepositoryTransaction()) ->setTransactionType(PhabricatorRepositoryTransaction::TYPE_SERVICE) ->setNewValue($service_phid); id(new PhabricatorRepositoryEditor()) ->setActor($viewer) ->setActingAsPHID($diffusion_phid) ->setContentSource($content_source) ->setContinueOnNoEffect(true) ->setContinueOnMissingFields(true) ->applyTransactions($repository, $xactions); if ($service) { echo tsprintf( "%s\n", pht( 'Moved repository "%s" to cluster service "%s".', $repository->getDisplayName(), $service->getName())); } else { echo tsprintf( "%s\n", pht( 'Removed repository "%s" from cluster service.', $repository->getDisplayName())); } } return 0; } } diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner index 463814c890..21aba731cc 100644 --- a/src/docs/user/cluster/cluster_repositories.diviner +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -1,487 +1,492 @@ @title Cluster: Repositories @group cluster Configuring Phabricator to use multiple repository hosts. Overview ======== If you use Git, you can deploy Phabricator with multiple repository hosts, configured so that each host is readable and writable. The advantages of doing this are: - you can completely survive the loss of repository hosts; - reads and writes can scale across multiple machines; and - read and write performance across multiple geographic regions may improve. This configuration is complex, and many installs do not need to pursue it. This configuration is not currently supported with Subversion or Mercurial. How Reads and Writes Work ========================= Phabricator repository replicas are multi-master: every node is readable and writable, and a cluster of nodes can (almost always) survive the loss of any arbitrary subset of nodes so long as at least one node is still alive. Phabricator maintains an internal version for each repository, and increments it when the repository is mutated. Before responding to a read, replicas make sure their version of the repository is up to date (no node in the cluster has a newer version of the repository). If it isn't, they block the read until they can complete a fetch. Before responding to a write, replicas obtain a global lock, perform the same version check and fetch if necessary, then allow the write to continue. Additionally, repositories passively check other nodes for updates and replicate changes in the background. After you push a change to a repository, it will usually spread passively to all other repository nodes within a few minutes. Even if passive replication is slow, the active replication makes acknowledged changes sequential to all observers: after a write is acknowledged, all subsequent reads are guaranteed to see it. The system does not permit stale reads, and you do not need to wait for a replication delay to see a consistent view of the repository no matter which node you ask. HTTP vs HTTPS ============= Intracluster requests (from the daemons to repository servers, or from webservers to repository servers) are permitted to use HTTP, even if you have set `security.require-https` in your configuration. It is common to terminate SSL at a load balancer and use plain HTTP beyond that, and the `security.require-https` feature is primarily focused on making client browser behavior more convenient for users, so it does not apply to intracluster traffic. Using HTTP within the cluster leaves you vulnerable to attackers who can observe traffic within a datacenter, or observe traffic between datacenters. This is normally very difficult, but within reach for state-level adversaries like the NSA. If you are concerned about these attackers, you can terminate HTTPS on repository hosts and bind to them with the "https" protocol. Just be aware that the `security.require-https` setting won't prevent you from making configuration mistakes, as it doesn't cover intracluster traffic. Other mitigations are possible, but securing a network against the NSA and similar agents of other rogue nations is beyond the scope of this document. Repository Hosts ================ Repository hosts must run a complete, fully configured copy of Phabricator, including a webserver. They must also run a properly configured `sshd`. If you are converting existing hosts into cluster hosts, you may need to revisit @{article:Diffusion User Guide: Repository Hosting} and make sure the system user accounts have all the necessary `sudo` permissions. In particular, cluster devices need `sudo` access to `ssh` so they can read device keys. Generally, these hosts will run the same set of services and configuration that web hosts run. If you prefer, you can overlay these services and put web and repository services on the same hosts. See @{article:Clustering Introduction} for some guidance on overlaying services. When a user requests information about a repository that can only be satisfied by examining a repository working copy, the webserver receiving the request will make an HTTP service call to a repository server which hosts the repository to retrieve the data it needs. It will use the result of this query to respond to the user. Setting up a Cluster Services ============================= To set up clustering, first register the devices that you want to use as part of the cluster with Almanac. For details, see @{article:Cluster: Devices}. NOTE: Once you create a service, new repositories will immediately allocate on it. You may want to disable repository creation during initial setup. Once the hosts are registered as devices, you can create a new service in Almanac: - First, register at least one device according to the device clustering instructions. - Create a new service of type **Phabricator Cluster: Repository** in Almanac. - Bind this service to all the interfaces on the device or devices. - For each binding, add a `protocol` key with one of these values: `ssh`, `http`, `https`. For example, a service might look like this: - Service: `repos001.mycompany.net` - Binding: `repo001.mycompany.net:80`, `protocol=http` - Binding: `repo001.mycompany.net:2222`, `protocol=ssh` The service itself has a `closed` property. You can set this to `true` to disable new repository allocations on this service (for example, if it is reaching capacity). Migrating to Clustered Services =============================== To convert existing repositories on an install into cluster repositories, you will generally perform these steps: - Register the existing host as a cluster device. - Configure a single host repository service using //only// that host. This puts you in a transitional state where repositories on the host can work as either on-host repositories or cluster repositories. You can move forward from here slowly and make sure services still work, with a quick path back to safety if you run into trouble. To move forward, migrate one repository to the service and make sure things work correctly. If you run into issues, you can back out by migrating the repository off the service. To migrate a repository onto a cluster service, use this command: ``` $ ./bin/repository clusterize --service ``` To migrate a repository back off a service, use this command: ``` $ ./bin/repository clusterize --remove-service ``` This command only changes how Phabricator connects to the repository; it does not move any data or make any complex structural changes. When Phabricator needs information about a non-clustered repository, it just runs a command like `git log` directly on disk. When Phabricator needs information about a clustered repository, it instead makes a service call to another server, asking that server to run `git log` instead. In a single-host cluster the server will make this service call to itself, so nothing will really change. But this //is// an effective test for most possible configuration mistakes. If your canary repository works well, you can migrate the rest of your repositories when ready (you can use `bin/repository list` to quickly get a list of all repository monograms). Once all repositories are migrated, you've reached a stable state and can remain here as long as you want. This state is sufficient to convert daemons, SSH, and web services into clustered versions and spread them across multiple machines if those goals are more interesting. Obviously, your single-device "cluster" will not be able to survive the loss of the single repository host, but you can take as long as you want to expand the cluster and add redundancy. After creating a service, you do not need to `clusterize` new repositories: they will automatically allocate onto an open service. When you're ready to expand the cluster, continue below. Expanding a Cluster =================== To expand an existing cluster, follow these general steps: - Register new devices in Almanac. - Add bindings to the new devices to the repository service, also in Almanac. - Start the daemons on the new devices. For instructions on configuring and registering devices, see @{article:Cluster: Devices}. As soon as you add active bindings to a service, Phabricator will begin synchronizing repositories and sending traffic to the new device. You do not need to copy any repository data to the device: Phabricator will automatically synchronize it. If you have a large amount of repository data, you may want to help this process along by copying the repository directory from an existing cluster device before bringing the new host online. This is optional, but can reduce the amount of time required to fully synchronize the cluster. You do not need to synchronize the most up-to-date data or stop writes during this process. For example, loading the most recent backup snapshot onto the new device will substantially reduce the amount of data that needs to be synchronized. Contracting a Cluster ===================== To reduce the size of an existing cluster, follow these general steps: - Disable the bindings from the service to the dead device in Almanac. If you are removing a device because it failed abruptly (or removing several devices at once) it is possible that some repositories will have lost all their leaders. See "Loss of Leaders" below to understand and resolve this. Monitoring Services =================== You can get an overview of repository cluster status from the {nav Config > Repository Servers} screen. This table shows a high-level overview of all active repository services. **Repos**: The number of repositories hosted on this service. **Sync**: Synchronization status of repositories on this service. This is an at-a-glance view of service health, and can show these values: - **Synchronized**: All nodes are fully synchronized and have the latest version of all repositories. - **Partial**: All repositories either have at least two leaders, or have a very recent write which is not expected to have propagated yet. - **Unsynchronized**: At least one repository has changes which are only available on one node and were not pushed very recently. Data may be at risk. - **No Repositories**: This service has no repositories. - **Ambiguous Leader**: At least one repository has an ambiguous leader. If this screen identifies problems, you can drill down into repository details to get more information about them. See the next section for details. Monitoring Repositories ======================= You can get a more detailed view the current status of a specific repository on cluster devices in {nav Diffusion > (Repository) > Manage Repository > Cluster Configuration}. This screen shows all the configured devices which are hosting the repository and the available version on that device. **Version**: When a repository is mutated by a push, Phabricator increases an internal version number for the repository. This column shows which version is on disk on the corresponding device. After a change is pushed, the device which received the change will have a larger version number than the other devices. The change should be passively replicated to the remaining devices after a brief period of time, although this can take a while if the change was large or the network connection between devices is slow or unreliable. You can click the version number to see the corresponding push logs for that change. The logs contain details about what was changed, and can help you identify if replication is slow because a change is large or for some other reason. **Writing**: This shows that the device is currently holding a write lock. This normally means that it is actively receiving a push, but can also mean that there was a write interruption. See "Write Interruptions" below for details. **Last Writer**: This column identifies the user who most recently pushed a change to this device. If the write lock is currently held, this user is the user whose change is holding the lock. **Last Write At**: When the most recent write started. If the write lock is currently held, this shows when the lock was acquired. Cluster Failure Modes ===================== There are three major cluster failure modes: - **Write Interruptions**: A write started but did not complete, leaving the disk state and cluster state out of sync. - **Loss of Leaders**: None of the devices with the most up-to-date data are reachable. - **Ambiguous Leaders**: The internal state of the repository is unclear. Phabricator can detect these issues, and responds by freezing the repository (usually preventing all reads and writes) until the issue is resolved. These conditions are normally rare and very little data is at risk, but Phabricator errs on the side of caution and requires decisions which may result in data loss to be confirmed by a human. The next sections cover these failure modes and appropriate responses in more detail. In general, you will respond to these issues by assessing the situation and then possibly choosing to discard some data. Write Interruptions =================== A repository cluster can be put into an inconsistent state by an interruption in a brief window during and immediately after a write. This looks like this: - A change is pushed to a server. - The server acquires a write lock and begins writing the change. - During or immediately after the write, lightning strikes the server and destroys it. Phabricator can not commit changes to a working copy (stored on disk) and to the global state (stored in a database) atomically, so there is necessarily a narrow window between committing these two different states when some tragedy can befall a server, leaving the global and local views of the repository state possibly divergent. In these cases, Phabricator fails into a frozen state where further writes are not permitted until the failure is investigated and resolved. When a repository is frozen in this way it remains readable. You can use the monitoring console to review the state of a frozen repository with a held write lock. The **Writing** column will show which device is holding the lock, and whoever is named in the **Last Writer** column may be able to help you figure out what happened by providing more information about what they were doing and what they observed. Because the push was not acknowledged, it is normally safe to resolve this issue by demoting the device. Demoting the device will undo any changes committed by the push, and they will be lost forever. However, the user should have received an error anyway, and should not expect their push to have worked. Still, data is technically at risk and you may want to investigate further and try to understand the issue in more detail before continuing. There is no way to explicitly keep the write, but if it was committed to disk you can recover it manually from the working copy on the device (for example, by using `git format-patch`) and then push it again after recovering. If you demote the device, the in-process write will be thrown away, even if it was complete on disk. To demote the device and release the write lock, run this command: ``` phabricator/ $ ./bin/repository thaw --demote ``` {icon exclamation-triangle, color="yellow"} Any committed but unacknowledged data on the device will be lost. Loss of Leaders =============== A more straightforward failure condition is the loss of all servers in a cluster which have the most up-to-date copy of a repository. This looks like this: - There is a cluster setup with two devices, X and Y. - A new change is pushed to server X. - Before the change can propagate to server Y, lightning strikes server X and destroys it. Here, all of the "leader" devices with the most up-to-date copy of the repository have been lost. Phabricator will freeze the repository refuse to serve requests because it can not serve reads consistently and can not accept new writes without data loss. The most straightforward way to resolve this issue is to restore any leader to service. The change will be able to replicate to other devices once a leader comes back online. If you are unable to restore a leader or unsure that you can restore one quickly, you can use the monitoring console to review which changes are present on the leaders but not present on the followers by examining the push logs. If you are comfortable discarding these changes, you can instruct Phabricator that it can forget about the leaders in two ways: disable the service bindings to all of the leader devices so they are no longer part of the cluster, or use `bin/repository thaw` to `--demote` the leaders explicitly. If you do this, **you will lose data**. Either action will discard any changes on the affected leaders which have not replicated to other devices in the cluster. To remove a device from the cluster, disable all of the bindings to it in Almanac, using the web UI. {icon exclamation-triangle, color="red"} Any data which is only present on the disabled device will be lost. To demote a device without removing it from the cluster, run this command: ``` phabricator/ $ ./bin/repository thaw rXYZ --demote repo002.corp.net ``` {icon exclamation-triangle, color="red"} Any data which is only present on **this** device will be lost. Ambiguous Leaders ================= Repository clusters can also freeze if the leader devices are ambiguous. This -can happen if you replace an entire cluster with new devices suddenly, or -make a mistake with the `--demote` flag. This generally arises from some kind -of operator error, like this: +can happen if you replace an entire cluster with new devices suddenly, or make +a mistake with the `--demote` flag. This may arise from some kind of operator +error, like these: - Someone accidentally uses `bin/repository thaw ... --demote` to demote every device in a cluster. - Someone accidentally deletes all the version information for a repository from the database by making a mistake with a `DELETE` or `UPDATE` query. - - Someone accidentally disable all of the devices in a cluster, then add + - Someone accidentally disables all of the devices in a cluster, then adds entirely new ones before repositories can propagate. +If you are moving repositories into cluster services, you can also reach this +state if you use `clusterize` to associate a repository with a service that is +bound to multiple active devices. In this case, Phabricator will not know which +device or devices have up-to-date information. + When Phabricator can not tell which device in a cluster is a leader, it freezes the cluster because it is possible that some devices have less data and others have more, and if it choses a leader arbitrarily it may destroy some data which you would prefer to retain. To resolve this, you need to tell Phabricator which device has the most up-to-date data and promote that device to become a leader. If you know all devices have the same data, you are free to promote any device. If you promote a device, **you may lose data** if you promote the wrong device and some other device really had more up-to-date data. If you want to double check, you can examine the working copies on disk before promoting by connecting to the machines and using commands like `git log` to inspect state. Once you have identified a device which has data you're happy with, use `bin/repository thaw` to `--promote` the device. The data on the chosen device will become authoritative: ``` phabricator/ $ ./bin/repository thaw rXYZ --promote repo002.corp.net ``` {icon exclamation-triangle, color="red"} Any data which is only present on **other** devices will be lost. Backups ====== Even if you configure clustering, you should still consider retaining separate backup snapshots. Replicas protect you from data loss if you lose a host, but they do not let you rewind time to recover from data mutation mistakes. If something issues a `--force` push that destroys branch heads, the mutation will propagate to the replicas. You may be able to manually restore the branches by using tools like the Phabricator push log or the Git reflog so it is less important to retain repository snapshots than database snapshots, but it is still possible for data to be lost permanently, especially if you don't notice the problem for some time. Retaining separate backup snapshots will improve your ability to recover more data more easily in a wider range of disaster situations. Next Steps ========== Continue by: - returning to @{article:Clustering Introduction}.