Page MenuHomePhabricator
Files F15451432

D14211.diff
No OneTemporary
Actions

D14211.diff
View Options

diff --git a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
--- a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
+++ b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
@@ -163,7 +163,6 @@
->withPHIDs(array($binding_phid))
->executeOne();
if (!$binding) {
- // TODO: This is probably a permanent failure, destroy this resource?
throw new Exception(
pht(
'Unable to load binding "%s" to create command interface.',
diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
--- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
@@ -297,7 +297,6 @@
foreach ($phids as $phid) {
if (empty($repositories[$phid])) {
- // TODO: Permanent failure.
throw new Exception(
pht(
'Repository PHID "%s" does not exist.',
@@ -306,12 +305,16 @@
}
foreach ($repositories as $repository) {
- switch ($repository->getVersionControlSystem()) {
+ $repository_vcs = $repository->getVersionControlSystem();
+ switch ($repository_vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
break;
default:
- // TODO: Permanent failure.
- throw new Exception(pht('Unsupported VCS!'));
+ throw new Exception(
+ pht(
+ 'Repository ("%s") has unsupported VCS ("%s").',
+ $repository->getPHID(),
+ $repository_vcs));
}
}
@@ -328,8 +331,10 @@
->withPHIDs(array($lease_phid))
->executeOne();
if (!$lease) {
- // TODO: Permanent failure.
- throw new Exception(pht('Unable to load lease "%s".', $lease_phid));
+ throw new Exception(
+ pht(
+ 'Unable to load lease ("%s").',
+ $lease_phid));
}
return $lease;
diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
--- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
+++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php
@@ -1,9 +1,11 @@
<?php
/**
+ * @task update Updating Resources
* @task command Processing Commands
* @task activate Activating Resources
* @task release Releasing Resources
+ * @task break Breaking Resources
* @task destroy Destroying Resources
*/
final class DrydockResourceUpdateWorker extends DrydockWorker {
@@ -19,7 +21,7 @@
try {
$resource = $this->loadResource($resource_phid);
- $this->updateResource($resource);
+ $this->handleUpdate($resource);
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
@@ -28,6 +30,37 @@
$lock->unlock();
}
+
+/* -( Updating Resources )------------------------------------------------- */
+
+
+ /**
+ * Update a resource, handling exceptions thrown during the update.
+ *
+ * @param DrydockReosource Resource to update.
+ * @return void
+ * @task update
+ */
+ private function handleUpdate(DrydockResource $resource) {
+ try {
+ $this->updateResource($resource);
+ } catch (Exception $ex) {
+ if ($this->isTemporaryException($ex)) {
+ $this->yieldResource($resource, $ex);
+ } else {
+ $this->breakResource($resource, $ex);
+ }
+ }
+ }
+
+
+ /**
+ * Update a resource.
+ *
+ * @param DrydockResource Resource to update.
+ * @return void
+ * @task update
+ */
private function updateResource(DrydockResource $resource) {
$this->processResourceCommands($resource);
@@ -52,6 +85,26 @@
}
+ /**
+ * Convert a temporary exception into a yield.
+ *
+ * @param DrydockResource Resource to yield.
+ * @param Exception Temporary exception worker encountered.
+ * @task update
+ */
+ private function yieldResource(DrydockResource $resource, Exception $ex) {
+ $duration = $this->getYieldDurationFromException($ex);
+
+ $resource->logEvent(
+ DrydockResourceActivationYieldLogType::LOGCONST,
+ array(
+ 'duration' => $duration,
+ ));
+
+ throw new PhabricatorWorkerYieldException($duration);
+ }
+
+
/* -( Processing Commands )------------------------------------------------ */
@@ -138,14 +191,9 @@
$viewer = $this->getViewer();
$drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
- $resource->openTransaction();
- $resource
- ->setStatus(DrydockResourceStatus::STATUS_RELEASED)
- ->save();
-
- // TODO: Hold slot locks until destruction?
- DrydockSlotLock::releaseLocks($resource->getPHID());
- $resource->saveTransaction();
+ $resource
+ ->setStatus(DrydockResourceStatus::STATUS_RELEASED)
+ ->save();
$statuses = array(
DrydockLeaseStatus::STATUS_PENDING,
@@ -173,6 +221,47 @@
}
+/* -( Breaking Resources )------------------------------------------------- */
+
+
+ /**
+ * @task break
+ */
+ private function breakResource(DrydockResource $resource, Exception $ex) {
+ switch ($resource->getStatus()) {
+ case DrydockResourceStatus::STATUS_BROKEN:
+ case DrydockResourceStatus::STATUS_RELEASED:
+ case DrydockResourceStatus::STATUS_DESTROYED:
+ // If the resource was already broken, just throw a normal exception.
+ // This will retry the task eventually.
+ throw new PhutilProxyException(
+ pht(
+ 'Unexpected failure while destroying resource ("%s").',
+ $resource->getPHID()),
+ $ex);
+ }
+
+ $resource
+ ->setStatus(DrydockResourceStatus::STATUS_BROKEN)
+ ->save();
+
+ $resource->scheduleUpdate();
+
+ $resource->logEvent(
+ DrydockResourceActivationFailureLogType::LOGCONST,
+ array(
+ 'class' => get_class($ex),
+ 'message' => $ex->getMessage(),
+ ));
+
+ throw new PhabricatorWorkerPermanentFailureException(
+ pht(
+ 'Permanent failure while activating resource ("%s"): %s',
+ $resource->getPHID(),
+ $ex->getMessage()));
+ }
+
+
/* -( Destroying Resources )----------------------------------------------- */
@@ -183,6 +272,8 @@
$blueprint = $resource->getBlueprint();
$blueprint->destroyResource($resource);
+ DrydockSlotLock::releaseLocks($resource->getPHID());
+
$resource
->setStatus(DrydockResourceStatus::STATUS_DESTROYED)
->save();
diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php
--- a/src/applications/drydock/worker/DrydockWorker.php
+++ b/src/applications/drydock/worker/DrydockWorker.php
@@ -114,4 +114,45 @@
throw new PhabricatorWorkerYieldException($expires - $now);
}
+ protected function isTemporaryException(Exception $ex) {
+ if ($ex instanceof PhabricatorWorkerYieldException) {
+ return true;
+ }
+
+ if ($ex instanceof DrydockSlotLockException) {
+ return true;
+ }
+
+ if ($ex instanceof PhutilAggregateException) {
+ $any_temporary = false;
+ foreach ($ex->getExceptions() as $sub) {
+ if ($this->isTemporaryException($sub)) {
+ $any_temporary = true;
+ break;
+ }
+ }
+ if ($any_temporary) {
+ return true;
+ }
+ }
+
+ if ($ex instanceof PhutilProxyException) {
+ return $this->isTemporaryException($ex->getPreviousException());
+ }
+
+ return false;
+ }
+
+ protected function getYieldDurationFromException(Exception $ex) {
+ if ($ex instanceof PhabricatorWorkerYieldException) {
+ return $ex->getDuration();
+ }
+
+ if ($ex instanceof DrydockSlotLockException) {
+ return 5;
+ }
+
+ return 15;
+ }
+
}

File Metadata

Mime Type
text/plain
Expires
Sat, Mar 29, 10:03 PM (3 w, 3 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7347504
Default Alt Text
D14211.diff (7 KB)

Event Timeline

Phabricator · Built by Phacility