Differential D21807 Diff 51981 src/applications/drydock/worker/DrydockLeaseUpdateWorker.php

Changeset View

Standalone View

src/applications/drydock/worker/DrydockLeaseUpdateWorker.php

Show First 20 Lines • Show All 182 Lines • ▼ Show 20 Lines	if (!$blueprints) {
pht(		pht(
'No active Drydock blueprint exists which can ever allocate a '.		'No active Drydock blueprint exists which can ever allocate a '.
'resource for lease "%s".',		'resource for lease "%s".',
$lease->getPHID()));		$lease->getPHID()));
}		}

// First, try to find a suitable open resource which we can acquire a new		// First, try to find a suitable open resource which we can acquire a new
// lease on.		// lease on.
$resources = $this->loadResourcesForAllocatingLease($blueprints, $lease);		$resources = $this->loadAcquirableResourcesForLease($blueprints, $lease);

// If no resources exist yet, see if we can build one.		list($free_resources, $used_resources) = $this->partitionResources(
if (!$resources) {		$lease,
$usable_blueprints = $this->removeOverallocatedBlueprints(		$resources);
$blueprints,
$lease);		$resource = $this->leaseAnyResource($lease, $free_resources);
		if ($resource) {
		return $resource;
		}

		// We're about to try creating a resource. If we're already creating
		// something, just yield until that resolves.

		$this->yieldForPendingResources($lease);

		// We haven't been able to lease an existing resource yet, so now we try to
		// create one. We may still have some less-desirable "used" resources that
		// we'll sometimes try to lease later if we fail to allocate a new resource.

		$resource = $this->newLeasedResource($lease, $blueprints);
		if ($resource) {
		return $resource;
		}

		// We haven't been able to lease a desirable "free" resource or create a
		// new resource. Try to lease a "used" resource.

		$resource = $this->leaseAnyResource($lease, $used_resources);
		if ($resource) {
		return $resource;
		}

		// If this lease has already triggered a reclaim, just yield and wait for
		// it to resolve.
		$this->yieldForReclaimingResources($lease);

		// Try to reclaim a resource. This will yield if it reclaims something.
		$this->reclaimAnyResource($lease, $blueprints);

		// We weren't able to lease, create, or reclaim any resources. We just have
		// to wait for resources to become available.

		$lease->logEvent(
		DrydockLeaseWaitingForResourcesLogType::LOGCONST,
		array(
		'blueprintPHIDs' => mpull($blueprints, 'getPHID'),
		));

		throw new PhabricatorWorkerYieldException(15);
		}

		private function reclaimAnyResource(DrydockLease $lease, array $blueprints) {
		assert_instances_of($blueprints, 'DrydockBlueprint');

// If we get nothing back here, some blueprint claims it can eventually
// satisfy the lease, just not right now. This is a temporary failure,
// and we expect allocation to succeed eventually.
if (!$usable_blueprints) {
$blueprints = $this->rankBlueprints($blueprints, $lease);		$blueprints = $this->rankBlueprints($blueprints, $lease);

// Try to actively reclaim unused resources. If we succeed, jump back		// Try to actively reclaim unused resources. If we succeed, jump back
// into the queue in an effort to claim it.		// into the queue in an effort to claim it.

foreach ($blueprints as $blueprint) {		foreach ($blueprints as $blueprint) {
$reclaimed = $this->reclaimResources($blueprint, $lease);		$reclaimed = $this->reclaimResources($blueprint, $lease);
if ($reclaimed) {		if ($reclaimed) {

$lease->logEvent(		$lease->logEvent(
DrydockLeaseReclaimLogType::LOGCONST,		DrydockLeaseReclaimLogType::LOGCONST,
array(		array(
'resourcePHIDs' => array($reclaimed->getPHID()),		'resourcePHIDs' => array($reclaimed->getPHID()),
));		));

		// Yield explicitly here: we'll be awakened when the resource is
		// reclaimed.

throw new PhabricatorWorkerYieldException(15);		throw new PhabricatorWorkerYieldException(15);
}		}
}		}
		}

		private function yieldForPendingResources(DrydockLease $lease) {
		// See T13677. If this lease has already triggered the allocation of
		// one or more resources and they are still pending, just yield and
		// wait for them.

		$viewer = $this->getViewer();

		$phids = $lease->getAllocatedResourcePHIDs();
		if (!$phids) {
		return null;
		}

		$resources = id(new DrydockResourceQuery())
		->setViewer($viewer)
		->withPHIDs($phids)
		->withStatuses(
		array(
		DrydockResourceStatus::STATUS_PENDING,
		))
		->setLimit(1)
		->execute();
		if (!$resources) {
		return;
		}

$lease->logEvent(		$lease->logEvent(
DrydockLeaseWaitingForResourcesLogType::LOGCONST,		DrydockLeaseWaitingForActivationLogType::LOGCONST,
array(		array(
'blueprintPHIDs' => mpull($blueprints, 'getPHID'),		'resourcePHIDs' => mpull($resources, 'getPHID'),
));		));

throw new PhabricatorWorkerYieldException(15);		throw new PhabricatorWorkerYieldException(15);
}		}

		private function yieldForReclaimingResources(DrydockLease $lease) {
		$viewer = $this->getViewer();

		$phids = $lease->getReclaimedResourcePHIDs();
		if (!$phids) {
		return;
		}

		$resources = id(new DrydockResourceQuery())
		->setViewer($viewer)
		->withPHIDs($phids)
		->withStatuses(
		array(
		DrydockResourceStatus::STATUS_ACTIVE,
		))
		->setLimit(1)
		->execute();
		if (!$resources) {
		return;
		}

		$lease->logEvent(
		DrydockLeaseWaitingForReclamationLogType::LOGCONST,
		array(
		'resourcePHIDs' => mpull($resources, 'getPHID'),
		));

		throw new PhabricatorWorkerYieldException(15);
		}

		private function newLeasedResource(
		DrydockLease $lease,
		array $blueprints) {
		assert_instances_of($blueprints, 'DrydockBlueprint');

		$usable_blueprints = $this->removeOverallocatedBlueprints(
		$blueprints,
		$lease);

		// If we get nothing back here, some blueprint claims it can eventually
		// satisfy the lease, just not right now. This is a temporary failure,
		// and we expect allocation to succeed eventually.

		// Return, try to lease a "used" resource, and continue from there.

		if (!$usable_blueprints) {
		return null;
		}

$usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);		$usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);

		$new_resources = $this->newResources($lease, $usable_blueprints);
		if (!$new_resources) {
		// If we were unable to create any new resources, return and
		// try to lease a "used" resource.
		return null;
		}

		$new_resources = $this->removeUnacquirableResources(
		$new_resources,
		$lease);
		if (!$new_resources) {
		// If we make it here, we just built a resource but aren't allowed
		// to acquire it. We expect this to happen if the resource prevents
		// acquisition until it activates, which is common when a resource
		// needs to perform setup steps.

		// Explicitly yield and wait for activation, since we don't want to
		// lease a "used" resource.

		throw new PhabricatorWorkerYieldException(15);
		}

		$resource = $this->leaseAnyResource($lease, $new_resources);
		if ($resource) {
		return $resource;
		}

		// We may not be able to lease a resource even if we just built it:
		// another process may snatch it up before we can lease it. This should
		// be rare, but is not concerning. Just try to build another resource.

		// We likely could try to build the next resource immediately, but err on
		// the side of caution and yield for now, at least until this code is
		// better vetted.

		throw new PhabricatorWorkerYieldException(15);
		}

		private function partitionResources(
		DrydockLease $lease,
		array $resources) {

		assert_instances_of($resources, 'DrydockResource');
		$viewer = $this->getViewer();

		$lease_statuses = array(
		DrydockLeaseStatus::STATUS_PENDING,
		DrydockLeaseStatus::STATUS_ACQUIRED,
		DrydockLeaseStatus::STATUS_ACTIVE,
		);

		// Partition resources into "free" resources (which we can try to lease
		// immediately) and "used" resources, which we can only to lease after we
		// fail to allocate a new resource.

		// "Free" resources are unleased and/or prefer reuse over allocation.
		// "Used" resources are leased and prefer allocation over reuse.

		$free_resources = array();
		$used_resources = array();

		foreach ($resources as $resource) {
		$blueprint = $resource->getBlueprint();

		if (!$blueprint->shouldAllocateSupplementalResource($resource, $lease)) {
		$free_resources[] = $resource;
		continue;
		}

		$leases = id(new DrydockLeaseQuery())
		->setViewer($viewer)
		->withResourcePHIDs(array($resource->getPHID()))
		->withStatuses($lease_statuses)
		->setLimit(1)
		->execute();
		if (!$leases) {
		$free_resources[] = $resource;
		continue;
		}

		$used_resources[] = $resource;
		}

		return array($free_resources, $used_resources);
		}

		private function newResources(
		DrydockLease $lease,
		array $blueprints) {
		assert_instances_of($blueprints, 'DrydockBlueprint');

		$resources = array();
$exceptions = array();		$exceptions = array();
foreach ($usable_blueprints as $blueprint) {		foreach ($blueprints as $blueprint) {
		$caught = null;
try {		try {
$resources[] = $this->allocateResource($blueprint, $lease);		$resources[] = $this->allocateResource($blueprint, $lease);

// Bail after allocating one resource, we don't need any more than		// Bail after allocating one resource, we don't need any more than
// this.		// this.
break;		break;
} catch (Exception $ex) {		} catch (Exception $ex) {
		$caught = $ex;
		} catch (Throwable $ex) {
		$caught = $ex;
		}

		if ($caught) {
// This failure is not normally expected, so log it. It can be		// This failure is not normally expected, so log it. It can be
// caused by something mundane and recoverable, however (see below		// caused by something mundane and recoverable, however (see below
// for discussion).		// for discussion).

// We log to the blueprint separately from the log to the lease:		// We log to the blueprint separately from the log to the lease:
// the lease is not attached to a blueprint yet so the lease log		// the lease is not attached to a blueprint yet so the lease log
// will not show up on the blueprint; more than one blueprint may		// will not show up on the blueprint; more than one blueprint may
// fail; and the lease is not really impacted (and won't log) if at		// fail; and the lease is not really impacted (and won't log) if at
// least one blueprint actually works.		// least one blueprint actually works.

$blueprint->logEvent(		$blueprint->logEvent(
DrydockResourceAllocationFailureLogType::LOGCONST,		DrydockResourceAllocationFailureLogType::LOGCONST,
array(		array(
'class' => get_class($ex),		'class' => get_class($caught),
'message' => $ex->getMessage(),		'message' => $caught->getMessage(),
));		));

$exceptions[] = $ex;		$exceptions[] = $caught;
}		}
}		}

if (!$resources) {		if (!$resources) {
// If one or more blueprints claimed that they would be able to		// If one or more blueprints claimed that they would be able to allocate
// allocate resources but none are actually able to allocate resources,		// resources but none are actually able to allocate resources, log the
// log the failure and yield so we try again soon.		// failure and yield so we try again soon.

// This can happen if some unexpected issue occurs during allocation		// This can happen if some unexpected issue occurs during allocation
// (for example, a call to build a VM fails for some reason) or if we		// (for example, a call to build a VM fails for some reason) or if we
// raced another allocator and the blueprint is now full.		// raced another allocator and the blueprint is now full.

$ex = new PhutilAggregateException(		$ex = new PhutilAggregateException(
pht(		pht(
'All blueprints failed to allocate a suitable new resource when '.		'All blueprints failed to allocate a suitable new resource when '.
'trying to allocate lease ("%s").',		'trying to allocate lease ("%s").',
$lease->getPHID()),		$lease->getPHID()),
$exceptions);		$exceptions);

$lease->logEvent(		$lease->logEvent(
DrydockLeaseAllocationFailureLogType::LOGCONST,		DrydockLeaseAllocationFailureLogType::LOGCONST,
array(		array(
'class' => get_class($ex),		'class' => get_class($ex),
'message' => $ex->getMessage(),		'message' => $ex->getMessage(),
));		));

throw new PhabricatorWorkerYieldException(15);		return null;
}		}

$resources = $this->removeUnacquirableResources($resources, $lease);		return $resources;
if (!$resources) {
// If we make it here, we just built a resource but aren't allowed
// to acquire it. We expect this during routine operation if the
// resource prevents acquisition until it activates. Yield and wait
// for activation.
throw new PhabricatorWorkerYieldException(15);
}		}

// NOTE: We have not acquired the lease yet, so it is possible that the
// resource we just built will be snatched up by some other lease before		private function leaseAnyResource(
// we can acquire it. This is not problematic: we'll retry a little later		DrydockLease $lease,
// and should succeed eventually.		array $resources) {
		assert_instances_of($resources, 'DrydockResource');

		if (!$resources) {
		return null;
}		}

$resources = $this->rankResources($resources, $lease);		$resources = $this->rankResources($resources, $lease);

$exceptions = array();		$exceptions = array();
$yields = array();		$yields = array();
$allocated = false;
		$allocated = null;
foreach ($resources as $resource) {		foreach ($resources as $resource) {
try {		try {
$resource = $this->newResourceForAcquisition($resource, $lease);
$this->acquireLease($resource, $lease);		$this->acquireLease($resource, $lease);
$allocated = true;		$allocated = $resource;
break;		break;
} catch (DrydockResourceLockException $ex) {		} catch (DrydockResourceLockException $ex) {
// We need to lock the resource to actually acquire it. If we aren't		// We need to lock the resource to actually acquire it. If we aren't
// able to acquire the lock quickly enough, we can yield and try again		// able to acquire the lock quickly enough, we can yield and try again
// later.		// later.
$yields[] = $ex;		$yields[] = $ex;
		} catch (DrydockSlotLockException $ex) {
		// This also just indicates we ran into some kind of contention,
		// probably from another lease. Just yield.
		$yields[] = $ex;
} catch (DrydockAcquiredBrokenResourceException $ex) {		} catch (DrydockAcquiredBrokenResourceException $ex) {
// If a resource was reclaimed or destroyed by the time we actually		// If a resource was reclaimed or destroyed by the time we actually
// got around to acquiring it, we just got unlucky. We can yield and		// got around to acquiring it, we just got unlucky.
// try again later.
$yields[] = $ex;		$yields[] = $ex;
} catch (PhabricatorWorkerYieldException $ex) {		} catch (PhabricatorWorkerYieldException $ex) {
// We can be told to yield, particularly by the supplemental allocator		// We can be told to yield, particularly by the supplemental allocator
// trying to give us a supplemental resource.		// trying to give us a supplemental resource.
$yields[] = $ex;		$yields[] = $ex;
} catch (Exception $ex) {		} catch (Exception $ex) {
$exceptions[] = $ex;		$exceptions[] = $ex;
}		}
}		}

if (!$allocated) {		if ($allocated) {
		return $allocated;
		}

if ($yields) {		if ($yields) {
throw new PhabricatorWorkerYieldException(15);		throw new PhabricatorWorkerYieldException(15);
} else {		}

throw new PhutilAggregateException(		throw new PhutilAggregateException(
pht(		pht(
'Unable to acquire lease "%s" on any resource.',		'Unable to acquire lease "%s" on any resource.',
$lease->getPHID()),		$lease->getPHID()),
$exceptions);		$exceptions);
}		}
}
}


/**		/**
* Get all the concrete @{class:DrydockBlueprint}s which can possibly		* Get all the concrete @{class:DrydockBlueprint}s which can possibly
* build a resource to satisfy a lease.		* build a resource to satisfy a lease.
*		*
* @param DrydockLease Requested lease.		* @param DrydockLease Requested lease.
* @return list<DrydockBlueprint> List of qualifying blueprints.		* @return list<DrydockBlueprint> List of qualifying blueprints.
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines	/* -( Drydock Allocator )-------------------------------------------------- */
*		*
* @param list<DrydockBlueprint> Blueprints which may produce suitable		* @param list<DrydockBlueprint> Blueprints which may produce suitable
* resources.		* resources.
* @param DrydockLease Requested lease.		* @param DrydockLease Requested lease.
* @return list<DrydockResource> Resources which may be able to allocate		* @return list<DrydockResource> Resources which may be able to allocate
* the lease.		* the lease.
* @task allocator		* @task allocator
*/		*/
private function loadResourcesForAllocatingLease(		private function loadAcquirableResourcesForLease(
array $blueprints,		array $blueprints,
DrydockLease $lease) {		DrydockLease $lease) {
assert_instances_of($blueprints, 'DrydockBlueprint');		assert_instances_of($blueprints, 'DrydockBlueprint');
$viewer = $this->getViewer();		$viewer = $this->getViewer();

$resources = id(new DrydockResourceQuery())		$resources = id(new DrydockResourceQuery())
->setViewer($viewer)		->setViewer($viewer)
->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))		->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
->withTypes(array($lease->getResourceType()))		->withTypes(array($lease->getResourceType()))
->withStatuses(		->withStatuses(
array(		array(
DrydockResourceStatus::STATUS_PENDING,
DrydockResourceStatus::STATUS_ACTIVE,		DrydockResourceStatus::STATUS_ACTIVE,
))		))
->execute();		->execute();

return $this->removeUnacquirableResources($resources, $lease);		return $this->removeUnacquirableResources($resources, $lease);
}		}


▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines	private function allocateResource(
DrydockBlueprint $blueprint,		DrydockBlueprint $blueprint,
DrydockLease $lease) {		DrydockLease $lease) {
$resource = $blueprint->allocateResource($lease);		$resource = $blueprint->allocateResource($lease);
$this->validateAllocatedResource($blueprint, $resource, $lease);		$this->validateAllocatedResource($blueprint, $resource, $lease);

// If this resource was allocated as a pending resource, queue a task to		// If this resource was allocated as a pending resource, queue a task to
// activate it.		// activate it.
if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {		if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {

		$lease->addAllocatedResourcePHIDs(
		array(
		$resource->getPHID(),
		));
		$lease->save();

PhabricatorWorker::scheduleTask(		PhabricatorWorker::scheduleTask(
'DrydockResourceUpdateWorker',		'DrydockResourceUpdateWorker',
array(		array(
'resourcePHID' => $resource->getPHID(),		'resourcePHID' => $resource->getPHID(),

// This task will generally yield while the resource activates, so		// This task will generally yield while the resource activates, so
// wake it back up once the resource comes online. Most of the time,		// wake it back up once the resource comes online. Most of the time,
// we'll be able to lease the newly activated resource.		// we'll be able to lease the newly activated resource.
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines	private function validateAllocatedResource(
}		}
}		}

private function reclaimResources(		private function reclaimResources(
DrydockBlueprint $blueprint,		DrydockBlueprint $blueprint,
DrydockLease $lease) {		DrydockLease $lease) {
$viewer = $this->getViewer();		$viewer = $this->getViewer();

// If this lease is marked as already in the process of reclaiming a
// resource, don't let it reclaim another one until the first reclaim
// completes. This stops one lease from reclaiming a large number of
// resources if the reclaims take a while to complete.
$reclaiming_phid = $lease->getAttribute('drydock.reclaimingPHID');
if ($reclaiming_phid) {
$reclaiming_resource = id(new DrydockResourceQuery())
->setViewer($viewer)
->withPHIDs(array($reclaiming_phid))
->withStatuses(
array(
DrydockResourceStatus::STATUS_ACTIVE,
DrydockResourceStatus::STATUS_RELEASED,
))
->executeOne();
if ($reclaiming_resource) {
return null;
}
}

$resources = id(new DrydockResourceQuery())		$resources = id(new DrydockResourceQuery())
->setViewer($viewer)		->setViewer($viewer)
->withBlueprintPHIDs(array($blueprint->getPHID()))		->withBlueprintPHIDs(array($blueprint->getPHID()))
->withStatuses(		->withStatuses(
array(		array(
DrydockResourceStatus::STATUS_ACTIVE,		DrydockResourceStatus::STATUS_ACTIVE,
))		))
->execute();		->execute();
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines	if ($lease_phid !== $resource_phid) {
'Blueprint "%s" (of type "%s") is not properly implemented: it '.		'Blueprint "%s" (of type "%s") is not properly implemented: it '.
'returned from "%s" with a lease acquired on the wrong resource.',		'returned from "%s" with a lease acquired on the wrong resource.',
$blueprint->getBlueprintName(),		$blueprint->getBlueprintName(),
$blueprint->getClassName(),		$blueprint->getClassName(),
'acquireLease()'));		'acquireLease()'));
}		}
}		}

private function newResourceForAcquisition(
DrydockResource $resource,
DrydockLease $lease) {

// If the resource has no leases against it, never build a new one. This is
// likely already a new resource that just activated.
$viewer = $this->getViewer();

$statuses = array(
DrydockLeaseStatus::STATUS_PENDING,
DrydockLeaseStatus::STATUS_ACQUIRED,
DrydockLeaseStatus::STATUS_ACTIVE,
);

$leases = id(new DrydockLeaseQuery())
->setViewer($viewer)
->withResourcePHIDs(array($resource->getPHID()))
->withStatuses($statuses)
->setLimit(1)
->execute();
if (!$leases) {
return $resource;
}

// If we're about to get a lease on a resource, check if the blueprint
// wants to allocate a supplemental resource. If it does, try to perform a
// new allocation instead.
$blueprint = $resource->getBlueprint();
if (!$blueprint->shouldAllocateSupplementalResource($resource, $lease)) {
return $resource;
}

// If the blueprint is already overallocated, we can't allocate a new
// resource. Just return the existing resource.
$remaining = $this->removeOverallocatedBlueprints(
array($blueprint),
$lease);
if (!$remaining) {
return $resource;
}

// Try to build a new resource.
try {
$new_resource = $this->allocateResource($blueprint, $lease);
} catch (Exception $ex) {
$blueprint->logEvent(
DrydockResourceAllocationFailureLogType::LOGCONST,
array(
'class' => get_class($ex),
'message' => $ex->getMessage(),
));

return $resource;
}

// If we can't actually acquire the new resource yet, just yield.
// (We could try to move forward with the original resource instead.)
$acquirable = $this->removeUnacquirableResources(
array($new_resource),
$lease);
if (!$acquirable) {
throw new PhabricatorWorkerYieldException(15);
}

return $new_resource;
}


/* -( Activating Leases )-------------------------------------------------- */		/* -( Activating Leases )-------------------------------------------------- */


/**		/**
* @task activate		* @task activate
*/		*/
private function activateLease(DrydockLease $lease) {		private function activateLease(DrydockLease $lease) {
▲ Show 20 Lines • Show All 153 Lines • Show Last 20 Lines