diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -746,6 +746,7 @@ 'DiffusionRepositoryBasicsManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryBasicsManagementPanel.php', 'DiffusionRepositoryByIDRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryByIDRemarkupRule.php', 'DiffusionRepositoryClusterEngine' => 'applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php', + 'DiffusionRepositoryClusterEngineLogInterface' => 'applications/diffusion/protocol/DiffusionRepositoryClusterEngineLogInterface.php', 'DiffusionRepositoryClusterManagementPanel' => 'applications/diffusion/management/DiffusionRepositoryClusterManagementPanel.php', 'DiffusionRepositoryController' => 'applications/diffusion/controller/DiffusionRepositoryController.php', 'DiffusionRepositoryCreateController' => 'applications/diffusion/controller/DiffusionRepositoryCreateController.php', @@ -4854,7 +4855,10 @@ 'DiffusionGitReceivePackSSHWorkflow' => 'DiffusionGitSSHWorkflow', 'DiffusionGitRequest' => 'DiffusionRequest', 'DiffusionGitResponse' => 'AphrontResponse', - 'DiffusionGitSSHWorkflow' => 'DiffusionSSHWorkflow', + 'DiffusionGitSSHWorkflow' => array( + 'DiffusionSSHWorkflow', + 'DiffusionRepositoryClusterEngineLogInterface', + ), 'DiffusionGitUploadPackSSHWorkflow' => 'DiffusionGitSSHWorkflow', 'DiffusionHistoryController' => 'DiffusionController', 'DiffusionHistoryQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', diff --git a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php --- a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php +++ b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php @@ -13,6 +13,7 @@ private $viewer; private $clusterWriteLock; private $clusterWriteVersion; + private $logger; /* -( Configuring Synchronization )---------------------------------------- */ @@ -36,6 +37,11 @@ return $this->viewer; } + public function setLog(DiffusionRepositoryClusterEngineLogInterface $log) { + $this->logger = $log; + return $this; + } + /* -( Cluster Synchronization )-------------------------------------------- */ @@ -92,8 +98,36 @@ $repository_phid, $device_phid); - // TODO: Raise a more useful exception if we fail to grab this lock. - $read_lock->lock(phutil_units('2 minutes in seconds')); + $lock_wait = phutil_units('2 minutes in seconds'); + + $this->logLine( + pht( + 'Waiting up to %s second(s) for a cluster read lock on "%s"...', + new PhutilNumber($lock_wait), + $device->getName())); + + try { + $start = PhabricatorTime::getNow(); + $read_lock->lock($lock_wait); + $waited = (PhabricatorTime::getNow() - $start); + + if ($waited) { + $this->logLine( + pht( + 'Acquired read lock after %s second(s).', + new PhutilNumber($waited))); + } else { + $this->logLine( + pht( + 'Acquired read lock immediately.')); + } + } catch (Exception $ex) { + throw new PhutilProxyException( + pht( + 'Failed to acquire read lock after waiting %s second(s). You '. + 'may be able to retry later.', + new PhutilNumber($lock_wait))); + } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); @@ -126,6 +160,12 @@ $repository_phid, $device_phid, $max_version); + } else { + $this->logLine( + pht( + 'Device "%s" is already a cluster leader and does not need '. + 'to be synchronized.', + $device->getName())); } $result_version = $max_version; @@ -210,8 +250,35 @@ $write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock( $repository_phid); - // TODO: Raise a more useful exception if we fail to grab this lock. - $write_lock->lock(phutil_units('2 minutes in seconds')); + $lock_wait = phutil_units('2 minutes in seconds'); + + $this->logLine( + pht( + 'Waiting up to %s second(s) for a cluster write lock...', + new PhutilNumber($lock_wait))); + + try { + $start = PhabricatorTime::getNow(); + $write_lock->lock($lock_wait); + $waited = (PhabricatorTime::getNow() - $start); + + if ($waited) { + $this->logLine( + pht( + 'Acquired write lock after %s second(s).', + new PhutilNumber($waited))); + } else { + $this->logLine( + pht( + 'Acquired write lock immediately.')); + } + } catch (Exception $ex) { + throw new PhutilProxyException( + pht( + 'Failed to acquire write lock after waiting %s second(s). You '. + 'may be able to retry later.', + new PhutilNumber($lock_wait))); + } $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions( $repository_phid); @@ -393,13 +460,20 @@ */ private function synchronizeWorkingCopyFromBinding($binding) { $repository = $this->getRepository(); + $device = AlmanacKeys::getLiveDevice(); + + $this->logLine( + pht( + 'Synchronizing this device ("%s") from cluster leader ("%s") before '. + 'read.', + $device->getName(), + $binding->getDevice()->getName())); $fetch_uri = $repository->getClusterRepositoryURIFromBinding($binding); $local_path = $repository->getLocalPath(); if ($repository->isGit()) { if (!Filesystem::pathExists($local_path)) { - $device = AlmanacKeys::getLiveDevice(); throw new Exception( pht( 'Repository "%s" does not have a working copy on this device '. @@ -429,7 +503,36 @@ $future->setCWD($local_path); - $future->resolvex(); + try { + $future->resolvex(); + } catch (Exception $ex) { + $this->logLine( + pht( + 'Synchronization of "%s" from leader "%s" failed: %s', + $device->getName(), + $binding->getDevice()->getName(), + $ex->getMessage())); + throw $ex; + } + } + + + /** + * @task internal + */ + private function logLine($message) { + return $this->logText("# {$message}\n"); } + + /** + * @task internal + */ + private function logText($message) { + $log = $this->logger; + if ($log) { + $log->writeClusterEngineLogMessage($message); + } + return $this; + } } diff --git a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngineLogInterface.php b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngineLogInterface.php new file mode 100644 --- /dev/null +++ b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngineLogInterface.php @@ -0,0 +1,7 @@ +getRepository(); $viewer = $this->getViewer(); + $device = AlmanacKeys::getLiveDevice(); // This is a write, and must have write access. $this->requireWriteAccess(); $cluster_engine = id(new DiffusionRepositoryClusterEngine()) ->setViewer($viewer) - ->setRepository($repository); + ->setRepository($repository) + ->setLog($this); if ($this->shouldProxy()) { $command = $this->getProxyCommand(); $did_synchronize = false; + + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Push received by \"%s\", forwarding to cluster host.\n", + $device->getName())); + } } else { $command = csprintf('git-receive-pack %s', $repository->getLocalPath()); $did_synchronize = true; $cluster_engine->synchronizeWorkingCopyBeforeWrite(); + + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Ready to receive on cluster host \"%s\".\n", + $device->getName())); + } } $caught = null; diff --git a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php --- a/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitSSHWorkflow.php @@ -1,12 +1,18 @@ getArgs(); $path = head($args->getArg('dir')); diff --git a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php --- a/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php +++ b/src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php @@ -16,18 +16,34 @@ protected function executeRepositoryOperations() { $repository = $this->getRepository(); $viewer = $this->getUser(); + $device = AlmanacKeys::getLiveDevice(); $skip_sync = $this->shouldSkipReadSynchronization(); if ($this->shouldProxy()) { $command = $this->getProxyCommand(); + + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Fetch received by \"%s\", forwarding to cluster host.\n", + $device->getName())); + } } else { $command = csprintf('git-upload-pack -- %s', $repository->getLocalPath()); if (!$skip_sync) { $cluster_engine = id(new DiffusionRepositoryClusterEngine()) ->setViewer($viewer) ->setRepository($repository) + ->setLog($this) ->synchronizeWorkingCopyBeforeRead(); + + if ($device) { + $this->writeClusterEngineLogMessage( + pht( + "# Cleared to fetch on cluster host \"%s\".\n", + $device->getName())); + } } } $command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);