diff --git a/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php index 1b3da10cae..c60b24b089 100644 --- a/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php +++ b/src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php @@ -1,213 +1,213 @@ buildSideNavView(); $nav->selectFilter('cluster/databases/'); $title = pht('Cluster Databases'); $crumbs = $this ->buildApplicationCrumbs($nav) ->addTextCrumb(pht('Cluster Databases')); $database_status = $this->buildClusterDatabaseStatus(); $view = id(new PHUITwoColumnView()) ->setNavigation($nav) ->setMainColumn($database_status); return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) ->appendChild($view); } private function buildClusterDatabaseStatus() { $viewer = $this->getViewer(); $databases = PhabricatorDatabaseRef::queryAll(); $connection_map = PhabricatorDatabaseRef::getConnectionStatusMap(); $replica_map = PhabricatorDatabaseRef::getReplicaStatusMap(); Javelin::initBehavior('phabricator-tooltips'); $rows = array(); foreach ($databases as $database) { $messages = array(); if ($database->getIsMaster()) { $role_icon = id(new PHUIIconView()) ->setIcon('fa-database sky') ->addSigil('has-tooltip') ->setMetadata( array( 'tip' => pht('Master'), )); } else { $role_icon = id(new PHUIIconView()) ->setIcon('fa-download') ->addSigil('has-tooltip') ->setMetadata( array( 'tip' => pht('Replica'), )); } if ($database->getDisabled()) { $conn_icon = 'fa-times'; $conn_color = 'grey'; $conn_label = pht('Disabled'); } else { $status = $database->getConnectionStatus(); $info = idx($connection_map, $status, array()); $conn_icon = idx($info, 'icon'); $conn_color = idx($info, 'color'); $conn_label = idx($info, 'label'); if ($status === PhabricatorDatabaseRef::STATUS_OKAY) { $latency = $database->getConnectionLatency(); $latency = (int)(1000000 * $latency); $conn_label = pht('%s us', new PhutilNumber($latency)); } } $connection = array( id(new PHUIIconView())->setIcon("{$conn_icon} {$conn_color}"), ' ', $conn_label, ); if ($database->getDisabled()) { $replica_icon = 'fa-times'; $replica_color = 'grey'; $replica_label = pht('Disabled'); } else { $status = $database->getReplicaStatus(); $info = idx($replica_map, $status, array()); $replica_icon = idx($info, 'icon'); $replica_color = idx($info, 'color'); $replica_label = idx($info, 'label'); if ($database->getIsMaster()) { if ($status === PhabricatorDatabaseRef::REPLICATION_OKAY) { $replica_icon = 'fa-database'; } } else { switch ($status) { case PhabricatorDatabaseRef::REPLICATION_OKAY: case PhabricatorDatabaseRef::REPLICATION_SLOW: $delay = $database->getReplicaDelay(); if ($delay) { $replica_label = pht('%ss Behind', new PhutilNumber($delay)); } else { $replica_label = pht('Up to Date'); } break; } } } $replication = array( id(new PHUIIconView())->setIcon("{$replica_icon} {$replica_color}"), ' ', $replica_label, ); $health = $database->getHealthRecord(); $health_up = $health->getUpEventCount(); $health_down = $health->getDownEventCount(); if ($health->getIsHealthy()) { $health_icon = id(new PHUIIconView()) ->setIcon('fa-plus green'); } else { $health_icon = id(new PHUIIconView()) ->setIcon('fa-times red'); $messages[] = pht( 'UNHEALTHY: This database has failed recent health checks. Traffic '. 'will not be sent to it until it recovers.'); } $health_count = pht( '%s / %s', new PhutilNumber($health_up), new PhutilNumber($health_up + $health_down)); $health_status = array( $health_icon, ' ', $health_count, ); $conn_message = $database->getConnectionMessage(); if ($conn_message) { $messages[] = $conn_message; } $replica_message = $database->getReplicaMessage(); if ($replica_message) { $messages[] = $replica_message; } $messages = phutil_implode_html(phutil_tag('br'), $messages); $rows[] = array( $role_icon, $database->getHost(), $database->getPort(), $database->getUser(), $connection, $replication, $health_status, $messages, ); } $table = id(new AphrontTableView($rows)) ->setNoDataString( pht('Phabricator is not configured in cluster mode.')) ->setHeaders( array( null, pht('Host'), pht('Port'), pht('User'), pht('Connection'), pht('Replication'), pht('Health'), pht('Messages'), )) ->setColumnClasses( array( null, null, null, null, null, null, null, 'wide', )); $doc_href = PhabricatorEnv::getDoclink('Cluster: Databases'); $header = id(new PHUIHeaderView()) ->setHeader(pht('Cluster Database Status')) ->addActionLink( id(new PHUIButtonView()) ->setIcon('fa-book') ->setHref($doc_href) ->setTag('a') - ->setText(pht('Database Clustering Documentation'))); + ->setText(pht('Documentation'))); return id(new PHUIObjectBoxView()) ->setHeader($header) ->setTable($table); } } diff --git a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php index 5993f74355..2a6841ebdb 100644 --- a/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php +++ b/src/applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php @@ -1,20 +1,129 @@ getRepository(); + $viewer = $this->getViewer(); + + $service_phid = $repository->getAlmanacServicePHID(); + if ($service_phid) { + $service = id(new AlmanacServiceQuery()) + ->setViewer($viewer) + ->withServiceTypes( + array( + AlmanacClusterRepositoryServiceType::SERVICETYPE, + )) + ->withPHIDs(array($service_phid)) + ->needBindings(true) + ->executeOne(); + if (!$service) { + // TODO: Viewer may not have permission to see the service, or it may + // be invalid? Raise some more useful error here? + throw new Exception(pht('Unable to load cluster service.')); + } + } else { + $service = null; + } + + Javelin::initBehavior('phabricator-tooltips'); + + $rows = array(); + if ($service) { + $bindings = $service->getBindings(); + $bindings = mgroup($bindings, 'getDevicePHID'); + + foreach ($bindings as $binding_group) { + $all_disabled = true; + foreach ($binding_group as $binding) { + if (!$binding->getIsDisabled()) { + $all_disabled = false; + break; + } + } + + $any_binding = head($binding_group); + + if ($all_disabled) { + $binding_icon = 'fa-times grey'; + $binding_tip = pht('Disabled'); + } else { + $binding_icon = 'fa-folder-open green'; + $binding_tip = pht('Active'); + } + + $binding_icon = id(new PHUIIconView()) + ->setIcon($binding_icon) + ->addSigil('has-tooltip') + ->setMetadata( + array( + 'tip' => $binding_tip, + )); + + $device = $any_binding->getDevice(); + + $rows[] = array( + $binding_icon, + phutil_tag( + 'a', + array( + 'href' => $device->getURI(), + ), + $device->getName()), + ); + } + } + + $table = id(new AphrontTableView($rows)) + ->setNoDataString(pht('This is not a cluster repository.')) + ->setHeaders( + array( + null, + pht('Device'), + )) + ->setColumnClasses( + array( + null, + 'wide', + )); + + $doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories'); + + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Cluster Status')) + ->addActionLink( + id(new PHUIButtonView()) + ->setIcon('fa-book') + ->setHref($doc_href) + ->setTag('a') + ->setText(pht('Documentation'))); + + if ($service) { + $header->setSubheader( + pht( + 'This repository is hosted on %s.', + phutil_tag( + 'a', + array( + 'href' => $service->getURI(), + ), + $service->getName()))); + } + + return id(new PHUIObjectBoxView()) + ->setHeader($header) + ->setTable($table); } } diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php index 1eb3eda929..da8d9336d2 100644 --- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php +++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php @@ -1,552 +1,564 @@ getArgv(); array_unshift($argv, __CLASS__); $args = new PhutilArgumentParser($argv); $args->parse( array( array( 'name' => 'no-discovery', 'help' => pht('Pull only, without discovering commits.'), ), array( 'name' => 'not', 'param' => 'repository', 'repeat' => true, 'help' => pht('Do not pull __repository__.'), ), array( 'name' => 'repositories', 'wildcard' => true, 'help' => pht('Pull specific __repositories__ instead of all.'), ), )); $no_discovery = $args->getArg('no-discovery'); $include = $args->getArg('repositories'); $exclude = $args->getArg('not'); // Each repository has an individual pull frequency; after we pull it, // wait that long to pull it again. When we start up, try to pull everything // serially. $retry_after = array(); $min_sleep = 15; $max_futures = 4; $futures = array(); $queue = array(); while (!$this->shouldExit()) { PhabricatorCaches::destroyRequestCache(); $device = AlmanacKeys::getLiveDevice(); $pullable = $this->loadPullableRepositories($include, $exclude, $device); // If any repositories have the NEEDS_UPDATE flag set, pull them // as soon as possible. $need_update_messages = $this->loadRepositoryUpdateMessages(true); foreach ($need_update_messages as $message) { $repo = idx($pullable, $message->getRepositoryID()); if (!$repo) { continue; } $this->log( pht( 'Got an update message for repository "%s"!', $repo->getMonogram())); $retry_after[$message->getRepositoryID()] = time(); } // If any repositories were deleted, remove them from the retry timer map // so we don't end up with a retry timer that never gets updated and // causes us to sleep for the minimum amount of time. $retry_after = array_select_keys( $retry_after, array_keys($pullable)); // Figure out which repositories we need to queue for an update. foreach ($pullable as $id => $repository) { $monogram = $repository->getMonogram(); if (isset($futures[$id])) { $this->log(pht('Repository "%s" is currently updating.', $monogram)); continue; } if (isset($queue[$id])) { $this->log(pht('Repository "%s" is already queued.', $monogram)); continue; } $after = idx($retry_after, $id, 0); if ($after > time()) { $this->log( pht( 'Repository "%s" is not due for an update for %s second(s).', $monogram, new PhutilNumber($after - time()))); continue; } if (!$after) { $this->log( pht( 'Scheduling repository "%s" for an initial update.', $monogram)); } else { $this->log( pht( 'Scheduling repository "%s" for an update (%s seconds overdue).', $monogram, new PhutilNumber(time() - $after))); } $queue[$id] = $after; } // Process repositories in the order they became candidates for updates. asort($queue); // Dequeue repositories until we hit maximum parallelism. while ($queue && (count($futures) < $max_futures)) { foreach ($queue as $id => $time) { $repository = idx($pullable, $id); if (!$repository) { $this->log( pht('Repository %s is no longer pullable; skipping.', $id)); unset($queue[$id]); continue; } $monogram = $repository->getMonogram(); $this->log(pht('Starting update for repository "%s".', $monogram)); unset($queue[$id]); $futures[$id] = $this->buildUpdateFuture( $repository, $no_discovery); break; } } if ($queue) { $this->log( pht( 'Not enough process slots to schedule the other %s '. 'repository(s) for updates yet.', phutil_count($queue))); } if ($futures) { $iterator = id(new FutureIterator($futures)) ->setUpdateInterval($min_sleep); foreach ($iterator as $id => $future) { $this->stillWorking(); if ($future === null) { $this->log(pht('Waiting for updates to complete...')); $this->stillWorking(); if ($this->loadRepositoryUpdateMessages()) { $this->log(pht('Interrupted by pending updates!')); break; } continue; } unset($futures[$id]); $retry_after[$id] = $this->resolveUpdateFuture( $pullable[$id], $future, $min_sleep); // We have a free slot now, so go try to fill it. break; } // Jump back into prioritization if we had any futures to deal with. continue; } $this->waitForUpdates($min_sleep, $retry_after); } } /** * @task pull */ private function buildUpdateFuture( PhabricatorRepository $repository, $no_discovery) { $bin = dirname(phutil_get_library_root('phabricator')).'/bin/repository'; $flags = array(); if ($no_discovery) { $flags[] = '--no-discovery'; } $monogram = $repository->getMonogram(); $future = new ExecFuture('%s update %Ls -- %s', $bin, $flags, $monogram); // Sometimes, the underlying VCS commands will hang indefinitely. We've // observed this occasionally with GitHub, and other users have observed // it with other VCS servers. // To limit the damage this can cause, kill the update out after a // reasonable amount of time, under the assumption that it has hung. // Since it's hard to know what a "reasonable" amount of time is given that // users may be downloading a repository full of pirated movies over a // potato, these limits are fairly generous. Repositories exceeding these // limits can be manually pulled with `bin/repository update X`, which can // just run for as long as it wants. if ($repository->isImporting()) { $timeout = phutil_units('4 hours in seconds'); } else { $timeout = phutil_units('15 minutes in seconds'); } $future->setTimeout($timeout); return $future; } /** * Check for repositories that should be updated immediately. * * With the `$consume` flag, an internal cursor will also be incremented so * that these messages are not returned by subsequent calls. * * @param bool Pass `true` to consume these messages, so the process will * not see them again. * @return list Pending update messages. * * @task pull */ private function loadRepositoryUpdateMessages($consume = false) { $type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE; $messages = id(new PhabricatorRepositoryStatusMessage())->loadAllWhere( 'statusType = %s AND id > %d', $type_need_update, $this->statusMessageCursor); // Keep track of messages we've seen so that we don't load them again. // If we reload messages, we can get stuck a loop if we have a failing // repository: we update immediately in response to the message, but do // not clear the message because the update does not succeed. We then // immediately retry. Instead, messages are only permitted to trigger // an immediate update once. if ($consume) { foreach ($messages as $message) { $this->statusMessageCursor = max( $this->statusMessageCursor, $message->getID()); } } return $messages; } /** * @task pull */ private function loadPullableRepositories( array $include, array $exclude, AlmanacDevice $device = null) { $query = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()); if ($include) { $query->withIdentifiers($include); } $repositories = $query->execute(); $repositories = mpull($repositories, null, 'getPHID'); if ($include) { $map = $query->getIdentifierMap(); foreach ($include as $identifier) { if (empty($map[$identifier])) { throw new Exception( pht( 'No repository "%s" exists!', $identifier)); } } } if ($exclude) { $xquery = id(new PhabricatorRepositoryQuery()) ->setViewer($this->getViewer()) ->withIdentifiers($exclude); $excluded_repos = $xquery->execute(); $xmap = $xquery->getIdentifierMap(); foreach ($exclude as $identifier) { if (empty($xmap[$identifier])) { throw new Exception( pht( 'No repository "%s" exists!', $identifier)); } } foreach ($excluded_repos as $excluded_repo) { unset($repositories[$excluded_repo->getPHID()]); } } foreach ($repositories as $key => $repository) { if (!$repository->isTracked()) { unset($repositories[$key]); } } $service_phids = array(); foreach ($repositories as $key => $repository) { $service_phid = $repository->getAlmanacServicePHID(); // If the repository is bound to a service but this host is not a // recognized device, or vice versa, don't pull the repository. $is_cluster_repo = (bool)$service_phid; $is_cluster_device = (bool)$device; if ($is_cluster_repo != $is_cluster_device) { if ($is_cluster_device) { $this->log( pht( 'Repository "%s" is not a cluster repository, but the current '. 'host is a cluster device ("%s"), so the repository will not '. 'be updated on this host.', $repository->getDisplayName(), $device->getName())); } else { $this->log( pht( 'Repository "%s" is a cluster repository, but the current '. 'host is not a cluster device (it has no device ID), so the '. 'repository will not be updated on this host.', $repository->getDisplayName())); } unset($repositories[$key]); continue; } if ($service_phid) { $service_phids[] = $service_phid; } } if ($device) { $device_phid = $device->getPHID(); if ($service_phids) { // We could include `withDevicePHIDs()` here to pull a smaller result // set, but we can provide more helpful diagnostic messages below if // we fetch a little more data. $services = id(new AlmanacServiceQuery()) ->setViewer($this->getViewer()) ->withPHIDs($service_phids) + ->withServiceTypes( + array( + AlmanacClusterRepositoryServiceType::SERVICETYPE, + )) ->needBindings(true) ->execute(); $services = mpull($services, null, 'getPHID'); } else { $services = array(); } foreach ($repositories as $key => $repository) { $service_phid = $repository->getAlmanacServicePHID(); $service = idx($services, $service_phid); if (!$service) { $this->log( pht( 'Repository "%s" is on cluster service "%s", but that service '. 'could not be loaded, so the repository will not be updated '. 'on this host.', $repository->getDisplayName(), $service_phid)); unset($repositories[$key]); continue; } $bindings = $service->getBindings(); - $bindings = mpull($bindings, null, 'getDevicePHID'); - $binding = idx($bindings, $device_phid); - if (!$binding) { + $bindings = mgroup($bindings, 'getDevicePHID'); + $bindings = idx($bindings, $device_phid); + if (!$bindings) { $this->log( pht( 'Repository "%s" is on cluster service "%s", but that service '. 'is not bound to this device ("%s"), so the repository will '. 'not be updated on this host.', $repository->getDisplayName(), $service->getName(), $device->getName())); unset($repositories[$key]); continue; } - if ($binding->getIsDisabled()) { + $all_disabled = true; + foreach ($bindings as $binding) { + if (!$binding->getIsDisabled()) { + $all_disabled = false; + break; + } + } + + if ($all_disabled) { $this->log( pht( 'Repository "%s" is on cluster service "%s", but the binding '. 'between that service and this device ("%s") is disabled, so '. 'the not be updated on this host.', $repository->getDisplayName(), $service->getName(), $device->getName())); unset($repositories[$key]); continue; } // We have a valid service that is actively bound to the current host // device, so we're good to go. } } // Shuffle the repositories, then re-key the array since shuffle() // discards keys. This is mostly for startup, we'll use soft priorities // later. shuffle($repositories); $repositories = mpull($repositories, null, 'getID'); return $repositories; } /** * @task pull */ private function resolveUpdateFuture( PhabricatorRepository $repository, ExecFuture $future, $min_sleep) { $monogram = $repository->getMonogram(); $this->log(pht('Resolving update for "%s".', $monogram)); try { list($stdout, $stderr) = $future->resolvex(); } catch (Exception $ex) { $proxy = new PhutilProxyException( pht( 'Error while updating the "%s" repository.', $repository->getMonogram()), $ex); phlog($proxy); return time() + $min_sleep; } if (strlen($stderr)) { $stderr_msg = pht( 'Unexpected output while updating repository "%s": %s', $monogram, $stderr); phlog($stderr_msg); } $smart_wait = $repository->loadUpdateInterval($min_sleep); $this->log( pht( 'Based on activity in repository "%s", considering a wait of %s '. 'seconds before update.', $repository->getMonogram(), new PhutilNumber($smart_wait))); return time() + $smart_wait; } /** * Sleep for a short period of time, waiting for update messages from the * * * @task pull */ private function waitForUpdates($min_sleep, array $retry_after) { $this->log( pht('No repositories need updates right now, sleeping...')); $sleep_until = time() + $min_sleep; if ($retry_after) { $sleep_until = min($sleep_until, min($retry_after)); } while (($sleep_until - time()) > 0) { $sleep_duration = ($sleep_until - time()); $this->log( pht( 'Sleeping for %s more second(s)...', new PhutilNumber($sleep_duration))); $this->sleep(1); if ($this->shouldExit()) { $this->log(pht('Awakened from sleep by graceful shutdown!')); return; } if ($this->loadRepositoryUpdateMessages()) { $this->log(pht('Awakened from sleep by pending updates!')); break; } } } } diff --git a/src/docs/user/cluster/cluster.diviner b/src/docs/user/cluster/cluster.diviner index 2b6376d49e..a05381af92 100644 --- a/src/docs/user/cluster/cluster.diviner +++ b/src/docs/user/cluster/cluster.diviner @@ -1,40 +1,57 @@ @title Clustering Introduction @group cluster Guide to configuring Phabricator across multiple hosts for availability and performance. Overview ======== WARNING: This feature is a very early prototype; the features this document describes are mostly speculative fantasy. Phabricator can be configured to run on mulitple hosts with redundant services to improve its availability and scalability, and make disaster recovery much easier. Clustering is more complex to setup and maintain than running everything on a single host, but greatly reduces the cost of recovering from hardware and network failures. Each Phabricator service has an array of clustering options that can be configured independently. Configuring a cluster is inherently complex, and this is an advanced feature aimed at installs with large userbases and experienced operations personnel who need this high degree of flexibility. The remainder of this document summarizes how to add redundancy to each service and where your efforts are likely to have the greatest impact. + Cluster: Databases ================= Configuring multiple database hosts is moderately complex, but normally has the highest impact on availability and resistance to data loss. This is usually the most important service to make redundant if your focus is on availability and disaster recovery. Configuring replicas allows Phabricator to run in read-only mode if you lose the master, and to quickly promote the replica as a replacement. For details, see @{article:Cluster: Databases}. + + +Cluster: Repositories +===================== + +Configuring multiple repository hosts is complex. + +Repository replicas are important for availability if you host repositories +on Phabricator, but less important if you host repositories elsewhere +(instead, you should focus on making that service more available). + +The distributed nature of Git and Mercurial tend to mean that they are +naturally somewhat resistant to data loss: every clone of a repository includes +the entire history. + +For details, see @{article:Cluster: Repositories}. diff --git a/src/docs/user/cluster/cluster_repositories.diviner b/src/docs/user/cluster/cluster_repositories.diviner new file mode 100644 index 0000000000..d9e859fd42 --- /dev/null +++ b/src/docs/user/cluster/cluster_repositories.diviner @@ -0,0 +1,86 @@ +@title Cluster: Repositories +@group intro + +Configuring Phabricator to use multiple repository hosts. + +Overview +======== + +WARNING: This feature is a very early prototype; the features this document +describes are mostly speculative fantasy. + +If you use Git or Mercurial, you can deploy Phabricator with multiple +repository hosts, configured so that each host is readable and writable. The +advantages of doing this are: + + - you can completely survive the loss of repository hosts; + - reads and writes can scale across multiple machines; and + - read and write performance across multiple geographic regions may improve. + +This configuration is complex, and many installs do not need to pursue it. + +This configuration is not currently supported with Subversion. + + +Repository Hosts +================ + +Repository hosts must run a complete, fully configured copy of Phabricator, +including a webserver. If you make repositories available over SSH, they must +also run a properly configured `sshd`. + +Generally, these hosts will run the same set of services and configuration that +web hosts run. If you prefer, you can overlay these services and put web and +repository services on the same hosts. + +When a user requests information about a repository that can only be satisfied +by examining a repository working copy, the webserver receiving the reqeust +will make an HTTP service call to a repository server which hosts the +repository to retrieve the data it needs. It will use the result of this query +to respond to the user. + + +How Reads and Writes Work +========================= + +Phabricator repository replicas are multi-master: every node is readable and +writable, and a cluster of nodes can (almost always) survive the loss of any +arbitrary subset of nodes so long as at least one node is still alive. + +Phabricator maintains an internal version for each repository, and increments +it when the repository is mutated. + +Before responding to a read, replicas make sure their version of the repository +is up to date (no node in the cluster has a newer version of the repository). +If it isn't, they block the read until they can complete a fetch. + +Before responding to a write, replicas obtain a global lock, perform the same +version check and fetch if necessary, then allow the write to continue. + + +Backups +====== + +Even if you configure clustering, you should still consider retaining separate +backup snapshots. Replicas protect you from data loss if you lose a host, but +they do not let you rewind time to recover from data mutation mistakes. + +If something issues a `--force` push that destroys branch heads, the mutation +will propagate to the replicas. + +You may be able to manually restore the branches by using tools like the +Phabricator push log or the Git reflog so it is less important to retain +repository snapshots than database snapshots, but it is still possible for +data to be lost permanently, especially if you don't notice the problem for +some time. + +Retaining separate backup snapshots will improve your ability to recover more +data more easily in a wider range of disaster situations. + + +Next Steps +========== + +Continue by: + + - returning to @{article:Clustering Introduction}.