diff --git a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php index 4724a2c807..12ccf8a343 100644 --- a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php @@ -1,317 +1,327 @@ loadServices($blueprint); $bindings = $this->loadAllBindings($services); if (!$bindings) { // If there are no devices bound to the services for this blueprint, // we can not allocate resources. return false; } return true; } + public function shouldAllocateSupplementalResource( + DrydockBlueprint $blueprint, + DrydockResource $resource, + DrydockLease $lease) { + // We want to use every host in an Almanac service, since the amount of + // hardware is fixed and there's normally no value in packing leases onto a + // subset of it. Always build a new supplemental resource if we can. + return true; + } + public function canAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease) { // We will only allocate one resource per unique device bound to the // services for this blueprint. Make sure we have a free device somewhere. $free_bindings = $this->loadFreeBindings($blueprint); if (!$free_bindings) { return false; } return true; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $free_bindings = $this->loadFreeBindings($blueprint); shuffle($free_bindings); $exceptions = array(); foreach ($free_bindings as $binding) { $device = $binding->getDevice(); $device_name = $device->getName(); $binding_phid = $binding->getPHID(); $resource = $this->newResourceTemplate($blueprint) ->setActivateWhenAllocated(true) ->setAttribute('almanacDeviceName', $device_name) ->setAttribute('almanacServicePHID', $binding->getServicePHID()) ->setAttribute('almanacBindingPHID', $binding_phid) ->needSlotLock("almanac.host.binding({$binding_phid})"); try { return $resource->allocateResource(); } catch (Exception $ex) { $exceptions[] = $ex; } } throw new PhutilAggregateException( pht('Unable to allocate any binding as a resource.'), $exceptions); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { // We don't create anything when allocating hosts, so we don't need to do // any cleanup here. return; } public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource) { $device_name = $resource->getAttribute( 'almanacDeviceName', pht('')); return pht('Host (%s)', $device_name); } public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // Require the binding to a given host be active before we'll hand out more // leases on the corresponding resource. $binding = $this->loadBindingForResource($resource); if ($binding->getIsDisabled()) { return false; } return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->setActivateWhenAcquired(true) ->acquireOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // Almanac hosts stick around indefinitely so we don't need to recycle them // if they don't have any leases. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We don't create anything when activating a lease, so we don't need to // throw anything away. return; } public function getType() { return 'host'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $credential_phid = $blueprint->getFieldValue('credentialPHID'); $binding = $this->loadBindingForResource($resource); $interface = $binding->getInterface(); return id(new DrydockSSHCommandInterface()) ->setConfig('credentialPHID', $credential_phid) ->setConfig('host', $interface->getAddress()) ->setConfig('port', $interface->getPort()); } } protected function getCustomFieldSpecifications() { return array( 'almanacServicePHIDs' => array( 'name' => pht('Almanac Services'), 'type' => 'datasource', 'datasource.class' => 'AlmanacServiceDatasource', 'datasource.parameters' => array( 'serviceTypes' => $this->getAlmanacServiceTypes(), ), 'required' => true, ), 'credentialPHID' => array( 'name' => pht('Credentials'), 'type' => 'credential', 'credential.provides' => PassphraseSSHPrivateKeyCredentialType::PROVIDES_TYPE, 'credential.type' => PassphraseSSHPrivateKeyTextCredentialType::CREDENTIAL_TYPE, ), ); } private function loadServices(DrydockBlueprint $blueprint) { if (!$this->services) { $service_phids = $blueprint->getFieldValue('almanacServicePHIDs'); if (!$service_phids) { throw new Exception( pht( 'This blueprint ("%s") does not define any Almanac Service PHIDs.', $blueprint->getBlueprintName())); } $viewer = $this->getViewer(); $services = id(new AlmanacServiceQuery()) ->setViewer($viewer) ->withPHIDs($service_phids) ->withServiceTypes($this->getAlmanacServiceTypes()) ->needBindings(true) ->execute(); $services = mpull($services, null, 'getPHID'); if (count($services) != count($service_phids)) { $missing_phids = array_diff($service_phids, array_keys($services)); throw new Exception( pht( 'Some of the Almanac Services defined by this blueprint '. 'could not be loaded. They may be invalid, no longer exist, '. 'or be of the wrong type: %s.', implode(', ', $missing_phids))); } $this->services = $services; } return $this->services; } private function loadAllBindings(array $services) { assert_instances_of($services, 'AlmanacService'); $bindings = array_mergev(mpull($services, 'getBindings')); return mpull($bindings, null, 'getPHID'); } private function loadFreeBindings(DrydockBlueprint $blueprint) { if ($this->freeBindings === null) { $viewer = $this->getViewer(); $pool = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(array($blueprint->getPHID())) ->withStatuses( array( DrydockResourceStatus::STATUS_PENDING, DrydockResourceStatus::STATUS_ACTIVE, DrydockResourceStatus::STATUS_BROKEN, DrydockResourceStatus::STATUS_RELEASED, )) ->execute(); $allocated_phids = array(); foreach ($pool as $resource) { $allocated_phids[] = $resource->getAttribute('almanacBindingPHID'); } $allocated_phids = array_fuse($allocated_phids); $services = $this->loadServices($blueprint); $bindings = $this->loadAllBindings($services); $free = array(); foreach ($bindings as $binding) { // Don't consider disabled bindings to be available. if ($binding->getIsDisabled()) { continue; } if (empty($allocated_phids[$binding->getPHID()])) { $free[] = $binding; } } $this->freeBindings = $free; } return $this->freeBindings; } private function getAlmanacServiceTypes() { return array( AlmanacDrydockPoolServiceType::SERVICETYPE, ); } private function loadBindingForResource(DrydockResource $resource) { $binding_phid = $resource->getAttribute('almanacBindingPHID'); if (!$binding_phid) { throw new Exception( pht( 'Drydock resource ("%s") has no Almanac binding PHID, so its '. 'binding can not be loaded.', $resource->getPHID())); } $viewer = $this->getViewer(); $binding = id(new AlmanacBindingQuery()) ->setViewer($viewer) ->withPHIDs(array($binding_phid)) ->executeOne(); if (!$binding) { throw new Exception( pht( 'Unable to load Almanac binding ("%s") for resource ("%s").', $binding_phid, $resource->getPHID())); } return $binding; } } diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php index 5acf63bf6b..88bc4d935a 100644 --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -1,494 +1,526 @@ getCustomFieldSpecifications(); if ($this->shouldUseConcurrentResourceLimit()) { $fields += array( 'allocator.limit' => array( 'name' => pht('Limit'), 'caption' => pht( 'Maximum number of resources this blueprint can have active '. 'concurrently.'), 'type' => 'int', ), ); } return $fields; } protected function getCustomFieldSpecifications() { return array(); } public function getViewer() { return PhabricatorUser::getOmnipotentUser(); } /* -( Lease Acquisition )-------------------------------------------------- */ /** * Enforce basic checks on lease/resource compatibility. Allows resources to * reject leases if they are incompatible, even if the resource types match. * * For example, if a resource represents a 32-bit host, this method might * reject leases that need a 64-bit host. The blueprint might also reject * a resource if the lease needs 8GB of RAM and the resource only has 6GB * free. * * This method should not acquire locks or expect anything to be locked. This * is a coarse compatibility check between a lease and a resource. * * @param DrydockBlueprint Concrete blueprint to allocate for. * @param DrydockResource Candidate resource to allocate the lease on. * @param DrydockLease Pending lease that wants to allocate here. * @return bool True if the resource and lease are compatible. * @task lease */ abstract public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * Acquire a lease. Allows resources to perform setup as leases are brought * online. * * If acquisition fails, throw an exception. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Requested lease. * @return void * @task lease */ abstract public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * @return void * @task lease */ public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { throw new PhutilMethodNotImplementedException(); } /** * React to a lease being released. * * This callback is primarily useful for automatically releasing resources * once all leases are released. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource a lease was released on. * @param DrydockLease Recently released lease. * @return void * @task lease */ abstract public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * Destroy any temporary data associated with a lease. * * If a lease creates temporary state while held, destroy it here. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource the lease is acquired on. * @param DrydockLease The lease being destroyed. * @return void * @task lease */ abstract public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); + /** + * Return true to try to allocate a new resource and expand the resource + * pool instead of permitting an otherwise valid acquisition on an existing + * resource. + * + * This allows the blueprint to provide a soft hint about when the resource + * pool should grow. + * + * Returning "true" in all cases generally makes sense when a blueprint + * controls a fixed pool of resources, like a particular number of physical + * hosts: you want to put all the hosts in service, so whenever it is + * possible to allocate a new host you want to do this. + * + * Returning "false" in all cases generally make sense when a blueprint + * has a flexible pool of expensive resources and you want to pack leases + * onto them as tightly as possible. + * + * @param DrydockBlueprint The blueprint for an existing resource being + * acquired. + * @param DrydockResource The resource being acquired, which we may want to + * build a supplemental resource for. + * @param DrydockLease The current lease performing acquisition. + * @return bool True to prefer allocating a supplemental resource. + * + * @task lease + */ + public function shouldAllocateSupplementalResource( + DrydockBlueprint $blueprint, + DrydockResource $resource, + DrydockLease $lease) { + return false; + } /* -( Resource Allocation )------------------------------------------------ */ /** * Enforce fundamental implementation/lease checks. Allows implementations to * reject a lease which no concrete blueprint can ever satisfy. * * For example, if a lease only builds ARM hosts and the lease needs a * PowerPC host, it may be rejected here. * * This is the earliest rejection phase, and followed by * @{method:canEverAllocateResourceForLease}. * * This method should not actually check if a resource can be allocated * right now, or even if a blueprint which can allocate a suitable resource * really exists, only if some blueprint may conceivably exist which could * plausibly be able to build a suitable resource. * * @param DrydockLease Requested lease. * @return bool True if some concrete blueprint of this implementation's * type might ever be able to build a resource for the lease. * @task resource */ abstract public function canAnyBlueprintEverAllocateResourceForLease( DrydockLease $lease); /** * Enforce basic blueprint/lease checks. Allows blueprints to reject a lease * which they can not build a resource for. * * This is the second rejection phase. It follows * @{method:canAnyBlueprintEverAllocateResourceForLease} and is followed by * @{method:canAllocateResourceForLease}. * * This method should not check if a resource can be built right now, only * if the blueprint as configured may, at some time, be able to build a * suitable resource. * * @param DrydockBlueprint Blueprint which may be asked to allocate a * resource. * @param DrydockLease Requested lease. * @return bool True if this blueprint can eventually build a suitable * resource for the lease, as currently configured. * @task resource */ abstract public function canEverAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease); /** * Enforce basic availability limits. Allows blueprints to reject resource * allocation if they are currently overallocated. * * This method should perform basic capacity/limit checks. For example, if * it has a limit of 6 resources and currently has 6 resources allocated, * it might reject new leases. * * This method should not acquire locks or expect locks to be acquired. This * is a coarse check to determine if the operation is likely to succeed * right now without needing to acquire locks. * * It is expected that this method will sometimes return `true` (indicating * that a resource can be allocated) but find that another allocator has * eaten up free capacity by the time it actually tries to build a resource. * This is normal and the allocator will recover from it. * * @param DrydockBlueprint The blueprint which may be asked to allocate a * resource. * @param DrydockLease Requested lease. * @return bool True if this blueprint appears likely to be able to allocate * a suitable resource. * @task resource */ abstract public function canAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease); /** * Allocate a suitable resource for a lease. * * This method MUST acquire, hold, and manage locks to prevent multiple * allocations from racing. World state is not locked before this method is * called. Blueprints are entirely responsible for any lock handling they * need to perform. * * @param DrydockBlueprint The blueprint which should allocate a resource. * @param DrydockLease Requested lease. * @return DrydockResource Allocated resource. * @task resource */ abstract public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease); /** * @task resource */ public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { throw new PhutilMethodNotImplementedException(); } /** * Destroy any temporary data associated with a resource. * * If a resource creates temporary state when allocated, destroy that state * here. For example, you might shut down a virtual host or destroy a working * copy on disk. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource being destroyed. * @return void * @task resource */ abstract public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource); /** * Get a human readable name for a resource. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource to get the name of. * @return string Human-readable resource name. * @task resource */ abstract public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource); /* -( Resource Interfaces )------------------------------------------------ */ abstract public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type); /* -( Logging )------------------------------------------------------------ */ public static function getAllBlueprintImplementations() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) ->execute(); } public static function getNamedImplementation($class) { return idx(self::getAllBlueprintImplementations(), $class); } protected function newResourceTemplate(DrydockBlueprint $blueprint) { $resource = id(new DrydockResource()) ->setBlueprintPHID($blueprint->getPHID()) ->attachBlueprint($blueprint) ->setType($this->getType()) ->setStatus(DrydockResourceStatus::STATUS_PENDING); // Pre-allocate the resource PHID. $resource->setPHID($resource->generatePHID()); return $resource; } protected function newLease(DrydockBlueprint $blueprint) { return DrydockLease::initializeNewLease() ->setAuthorizingPHID($blueprint->getPHID()); } protected function requireActiveLease(DrydockLease $lease) { $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: throw new PhabricatorWorkerYieldException(15); case DrydockLeaseStatus::STATUS_ACTIVE: return; default: throw new Exception( pht( 'Lease ("%s") is in bad state ("%s"), expected "%s".', $lease->getPHID(), $lease_status, DrydockLeaseStatus::STATUS_ACTIVE)); } } /** * Does this implementation use concurrent resource limits? * * Implementations can override this method to opt into standard limit * behavior, which provides a simple concurrent resource limit. * * @return bool True to use limits. */ protected function shouldUseConcurrentResourceLimit() { return false; } /** * Get the effective concurrent resource limit for this blueprint. * * @param DrydockBlueprint Blueprint to get the limit for. * @return int|null Limit, or `null` for no limit. */ protected function getConcurrentResourceLimit(DrydockBlueprint $blueprint) { if ($this->shouldUseConcurrentResourceLimit()) { $limit = $blueprint->getFieldValue('allocator.limit'); $limit = (int)$limit; if ($limit > 0) { return $limit; } else { return null; } } return null; } protected function getConcurrentResourceLimitSlotLock( DrydockBlueprint $blueprint) { $limit = $this->getConcurrentResourceLimit($blueprint); if ($limit === null) { return; } $blueprint_phid = $blueprint->getPHID(); // TODO: This logic shouldn't do anything awful, but is a little silly. It // would be nice to unify the "huge limit" and "small limit" cases // eventually but it's a little tricky. // If the limit is huge, just pick a random slot. This is just stopping // us from exploding if someone types a billion zillion into the box. if ($limit > 1024) { $slot = mt_rand(0, $limit - 1); return "allocator({$blueprint_phid}).limit({$slot})"; } // For reasonable limits, actually check for an available slot. $slots = range(0, $limit - 1); shuffle($slots); $lock_names = array(); foreach ($slots as $slot) { $lock_names[] = "allocator({$blueprint_phid}).limit({$slot})"; } $locks = DrydockSlotLock::loadHeldLocks($lock_names); $locks = mpull($locks, null, 'getLockKey'); foreach ($lock_names as $lock_name) { if (empty($locks[$lock_name])) { return $lock_name; } } // If we found no free slot, just return whatever we checked last (which // is just a random slot). There's a small chance we'll get lucky and the // lock will be free by the time we try to take it, but usually we'll just // fail to grab the lock, throw an appropriate lock exception, and get back // on the right path to retry later. return $lock_name; } /** * Apply standard limits on resource allocation rate. * * @param DrydockBlueprint The blueprint requesting an allocation. * @return bool True if further allocations should be limited. */ protected function shouldLimitAllocatingPoolSize( DrydockBlueprint $blueprint) { // TODO: If this mechanism sticks around, these values should be // configurable by the blueprint implementation. // Limit on total number of active resources. $total_limit = $this->getConcurrentResourceLimit($blueprint); // Always allow at least this many allocations to be in flight at once. $min_allowed = 1; // Allow this fraction of allocating resources as a fraction of active // resources. $growth_factor = 0.25; $resource = new DrydockResource(); $conn_r = $resource->establishConnection('r'); $counts = queryfx_all( $conn_r, 'SELECT status, COUNT(*) N FROM %T WHERE blueprintPHID = %s AND status != %s GROUP BY status', $resource->getTableName(), $blueprint->getPHID(), DrydockResourceStatus::STATUS_DESTROYED); $counts = ipull($counts, 'N', 'status'); $n_alloc = idx($counts, DrydockResourceStatus::STATUS_PENDING, 0); $n_active = idx($counts, DrydockResourceStatus::STATUS_ACTIVE, 0); $n_broken = idx($counts, DrydockResourceStatus::STATUS_BROKEN, 0); $n_released = idx($counts, DrydockResourceStatus::STATUS_RELEASED, 0); // If we're at the limit on total active resources, limit additional // allocations. if ($total_limit !== null) { $n_total = ($n_alloc + $n_active + $n_broken + $n_released); if ($n_total >= $total_limit) { return true; } } // If the number of in-flight allocations is fewer than the minimum number // of allowed allocations, don't impose a limit. if ($n_alloc < $min_allowed) { return false; } $allowed_alloc = (int)ceil($n_active * $growth_factor); // If the number of in-flight allocation is fewer than the number of // allowed allocations according to the pool growth factor, don't impose // a limit. if ($n_alloc < $allowed_alloc) { return false; } return true; } } diff --git a/src/applications/drydock/storage/DrydockBlueprint.php b/src/applications/drydock/storage/DrydockBlueprint.php index f7654a8b31..61e2dbfcfa 100644 --- a/src/applications/drydock/storage/DrydockBlueprint.php +++ b/src/applications/drydock/storage/DrydockBlueprint.php @@ -1,389 +1,398 @@ setViewer($actor) ->withClasses(array('PhabricatorDrydockApplication')) ->executeOne(); $view_policy = $app->getPolicy( DrydockDefaultViewCapability::CAPABILITY); $edit_policy = $app->getPolicy( DrydockDefaultEditCapability::CAPABILITY); return id(new DrydockBlueprint()) ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setBlueprintName('') ->setIsDisabled(0); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'details' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'className' => 'text255', 'blueprintName' => 'sort255', 'isDisabled' => 'bool', ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( DrydockBlueprintPHIDType::TYPECONST); } public function getImplementation() { return $this->assertAttached($this->implementation); } public function attachImplementation(DrydockBlueprintImplementation $impl) { $this->implementation = $impl; return $this; } public function hasImplementation() { return ($this->implementation !== self::ATTACHABLE); } public function getDetail($key, $default = null) { return idx($this->details, $key, $default); } public function setDetail($key, $value) { $this->details[$key] = $value; return $this; } public function getFieldValue($key) { $key = "std:drydock:core:{$key}"; $fields = $this->loadCustomFields(); $field = idx($fields, $key); if (!$field) { throw new Exception( pht( 'Unknown blueprint field "%s"!', $key)); } return $field->getBlueprintFieldValue(); } private function loadCustomFields() { if ($this->fields === null) { $field_list = PhabricatorCustomField::getObjectFields( $this, PhabricatorCustomField::ROLE_VIEW); $field_list->readFieldsFromStorage($this); $this->fields = $field_list->getFields(); } return $this->fields; } public function logEvent($type, array $data = array()) { $log = id(new DrydockLog()) ->setEpoch(PhabricatorTime::getNow()) ->setType($type) ->setData($data); $log->setBlueprintPHID($this->getPHID()); return $log->save(); } public function getURI() { $id = $this->getID(); return "/drydock/blueprint/{$id}/"; } /* -( Allocating Resources )----------------------------------------------- */ /** * @task resource */ public function canEverAllocateResourceForLease(DrydockLease $lease) { return $this->getImplementation()->canEverAllocateResourceForLease( $this, $lease); } /** * @task resource */ public function canAllocateResourceForLease(DrydockLease $lease) { return $this->getImplementation()->canAllocateResourceForLease( $this, $lease); } /** * @task resource */ public function allocateResource(DrydockLease $lease) { return $this->getImplementation()->allocateResource( $this, $lease); } /** * @task resource */ public function activateResource(DrydockResource $resource) { return $this->getImplementation()->activateResource( $this, $resource); } /** * @task resource */ public function destroyResource(DrydockResource $resource) { $this->getImplementation()->destroyResource( $this, $resource); return $this; } /** * @task resource */ public function getResourceName(DrydockResource $resource) { return $this->getImplementation()->getResourceName( $this, $resource); } /* -( Acquiring Leases )--------------------------------------------------- */ /** * @task lease */ public function canAcquireLeaseOnResource( DrydockResource $resource, DrydockLease $lease) { return $this->getImplementation()->canAcquireLeaseOnResource( $this, $resource, $lease); } /** * @task lease */ public function acquireLease( DrydockResource $resource, DrydockLease $lease) { return $this->getImplementation()->acquireLease( $this, $resource, $lease); } /** * @task lease */ public function activateLease( DrydockResource $resource, DrydockLease $lease) { return $this->getImplementation()->activateLease( $this, $resource, $lease); } /** * @task lease */ public function didReleaseLease( DrydockResource $resource, DrydockLease $lease) { $this->getImplementation()->didReleaseLease( $this, $resource, $lease); return $this; } /** * @task lease */ public function destroyLease( DrydockResource $resource, DrydockLease $lease) { $this->getImplementation()->destroyLease( $this, $resource, $lease); return $this; } public function getInterface( DrydockResource $resource, DrydockLease $lease, $type) { $interface = $this->getImplementation() ->getInterface($this, $resource, $lease, $type); if (!$interface) { throw new Exception( pht( 'Unable to build resource interface of type "%s".', $type)); } return $interface; } + public function shouldAllocateSupplementalResource( + DrydockResource $resource, + DrydockLease $lease) { + return $this->getImplementation()->shouldAllocateSupplementalResource( + $this, + $resource, + $lease); + } + /* -( PhabricatorApplicationTransactionInterface )------------------------- */ public function getApplicationTransactionEditor() { return new DrydockBlueprintEditor(); } public function getApplicationTransactionObject() { return $this; } public function getApplicationTransactionTemplate() { return new DrydockBlueprintTransaction(); } public function willRenderTimeline( PhabricatorApplicationTransactionView $timeline, AphrontRequest $request) { return $timeline; } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: return $this->getViewPolicy(); case PhabricatorPolicyCapability::CAN_EDIT: return $this->getEditPolicy(); } } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return false; } /* -( PhabricatorCustomFieldInterface )------------------------------------ */ public function getCustomFieldSpecificationForRole($role) { return array(); } public function getCustomFieldBaseClass() { return 'DrydockBlueprintCustomField'; } public function getCustomFields() { return $this->assertAttached($this->customFields); } public function attachCustomFields(PhabricatorCustomFieldAttachment $fields) { $this->customFields = $fields; return $this; } /* -( PhabricatorNgramsInterface )----------------------------------------- */ public function newNgrams() { return array( id(new DrydockBlueprintNameNgrams()) ->setValue($this->getBlueprintName()), ); } /* -( PhabricatorConduitResultInterface )---------------------------------- */ public function getFieldSpecificationsForConduit() { return array( id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('name') ->setType('string') ->setDescription(pht('The name of this blueprint.')), id(new PhabricatorConduitSearchFieldSpecification()) ->setKey('type') ->setType('string') ->setDescription(pht('The type of resource this blueprint provides.')), ); } public function getFieldValuesForConduit() { return array( 'name' => $this->getBlueprintName(), 'type' => $this->getImplementation()->getType(), ); } public function getConduitSearchAttachments() { return array( ); } } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 3ff9900239..b83022f720 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,954 +1,1026 @@ getTaskDataValue('leasePHID'); $hash = PhabricatorHash::digestForIndex($lease_phid); $lock_key = 'drydock.lease:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $lease = $this->loadLease($lease_phid); $this->handleUpdate($lease); } catch (Exception $ex) { $lock->unlock(); $this->flushDrydockTaskQueue(); throw $ex; } $lock->unlock(); } /* -( Updating Leases )---------------------------------------------------- */ /** * @task update */ private function handleUpdate(DrydockLease $lease) { try { $this->updateLease($lease); } catch (DrydockAcquiredBrokenResourceException $ex) { // If this lease acquired a resource but failed to activate, we don't // need to break the lease. We can throw it back in the pool and let // it take another shot at acquiring a new resource. // Before we throw it back, release any locks the lease is holding. DrydockSlotLock::releaseLocks($lease->getPHID()); $lease ->setStatus(DrydockLeaseStatus::STATUS_PENDING) ->setResourcePHID(null) ->save(); $lease->logEvent( DrydockLeaseReacquireLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); $this->yieldLease($lease, $ex); } catch (Exception $ex) { if ($this->isTemporaryException($ex)) { $this->yieldLease($lease, $ex); } else { $this->breakLease($lease, $ex); } } } /** * @task update */ private function updateLease(DrydockLease $lease) { $this->processLeaseCommands($lease); $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: $this->executeAllocator($lease); break; case DrydockLeaseStatus::STATUS_ACQUIRED: $this->activateLease($lease); break; case DrydockLeaseStatus::STATUS_ACTIVE: // Nothing to do. break; case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_BROKEN: $this->destroyLease($lease); break; case DrydockLeaseStatus::STATUS_DESTROYED: break; } $this->yieldIfExpiringLease($lease); } /** * @task update */ private function yieldLease(DrydockLease $lease, Exception $ex) { $duration = $this->getYieldDurationFromException($ex); $lease->logEvent( DrydockLeaseActivationYieldLogType::LOGCONST, array( 'duration' => $duration, )); throw new PhabricatorWorkerYieldException($duration); } /* -( Processing Commands )------------------------------------------------ */ /** * @task command */ private function processLeaseCommands(DrydockLease $lease) { if (!$lease->canReceiveCommands()) { return; } $this->checkLeaseExpiration($lease); $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { if (!$lease->canReceiveCommands()) { break; } $this->processLeaseCommand($lease, $command); $command ->setIsConsumed(true) ->save(); } } /** * @task command */ private function processLeaseCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseLease($lease); break; } } /* -( Drydock Allocator )-------------------------------------------------- */ /** * Find or build a resource which can satisfy a given lease request, then * acquire the lease. * * @param DrydockLease Requested lease. * @return void * @task allocator */ private function executeAllocator(DrydockLease $lease) { $blueprints = $this->loadBlueprintsForAllocatingLease($lease); // If we get nothing back, that means no blueprint is defined which can // ever build the requested resource. This is a permanent failure, since // we don't expect to succeed no matter how many times we try. if (!$blueprints) { throw new PhabricatorWorkerPermanentFailureException( pht( 'No active Drydock blueprint exists which can ever allocate a '. 'resource for lease "%s".', $lease->getPHID())); } // First, try to find a suitable open resource which we can acquire a new // lease on. $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); // If no resources exist yet, see if we can build one. if (!$resources) { $usable_blueprints = $this->removeOverallocatedBlueprints( $blueprints, $lease); // If we get nothing back here, some blueprint claims it can eventually // satisfy the lease, just not right now. This is a temporary failure, // and we expect allocation to succeed eventually. if (!$usable_blueprints) { $blueprints = $this->rankBlueprints($blueprints, $lease); // Try to actively reclaim unused resources. If we succeed, jump back // into the queue in an effort to claim it. foreach ($blueprints as $blueprint) { $reclaimed = $this->reclaimResources($blueprint, $lease); if ($reclaimed) { $lease->logEvent( DrydockLeaseReclaimLogType::LOGCONST, array( 'resourcePHIDs' => array($reclaimed->getPHID()), )); throw new PhabricatorWorkerYieldException(15); } } $lease->logEvent( DrydockLeaseWaitingForResourcesLogType::LOGCONST, array( 'blueprintPHIDs' => mpull($blueprints, 'getPHID'), )); throw new PhabricatorWorkerYieldException(15); } $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease); $exceptions = array(); foreach ($usable_blueprints as $blueprint) { try { $resources[] = $this->allocateResource($blueprint, $lease); // Bail after allocating one resource, we don't need any more than // this. break; } catch (Exception $ex) { // This failure is not normally expected, so log it. It can be // caused by something mundane and recoverable, however (see below // for discussion). // We log to the blueprint separately from the log to the lease: // the lease is not attached to a blueprint yet so the lease log // will not show up on the blueprint; more than one blueprint may // fail; and the lease is not really impacted (and won't log) if at // least one blueprint actually works. $blueprint->logEvent( DrydockResourceAllocationFailureLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); $exceptions[] = $ex; } } if (!$resources) { // If one or more blueprints claimed that they would be able to // allocate resources but none are actually able to allocate resources, // log the failure and yield so we try again soon. // This can happen if some unexpected issue occurs during allocation // (for example, a call to build a VM fails for some reason) or if we // raced another allocator and the blueprint is now full. $ex = new PhutilAggregateException( pht( 'All blueprints failed to allocate a suitable new resource when '. 'trying to allocate lease ("%s").', $lease->getPHID()), $exceptions); $lease->logEvent( DrydockLeaseAllocationFailureLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); throw new PhabricatorWorkerYieldException(15); } $resources = $this->removeUnacquirableResources($resources, $lease); if (!$resources) { // If we make it here, we just built a resource but aren't allowed // to acquire it. We expect this during routine operation if the // resource prevents acquisition until it activates. Yield and wait // for activation. throw new PhabricatorWorkerYieldException(15); } // NOTE: We have not acquired the lease yet, so it is possible that the // resource we just built will be snatched up by some other lease before // we can acquire it. This is not problematic: we'll retry a little later // and should succeed eventually. } $resources = $this->rankResources($resources, $lease); $exceptions = array(); $yields = array(); $allocated = false; foreach ($resources as $resource) { try { + $resource = $this->newResourceForAcquisition($resource, $lease); $this->acquireLease($resource, $lease); $allocated = true; break; } catch (DrydockResourceLockException $ex) { // We need to lock the resource to actually acquire it. If we aren't // able to acquire the lock quickly enough, we can yield and try again // later. $yields[] = $ex; } catch (DrydockAcquiredBrokenResourceException $ex) { // If a resource was reclaimed or destroyed by the time we actually // got around to acquiring it, we just got unlucky. We can yield and // try again later. $yields[] = $ex; + } catch (PhabricatorWorkerYieldException $ex) { + // We can be told to yield, particularly by the supplemental allocator + // trying to give us a supplemental resource. + $yields[] = $ex; } catch (Exception $ex) { $exceptions[] = $ex; } } if (!$allocated) { if ($yields) { throw new PhabricatorWorkerYieldException(15); } else { throw new PhutilAggregateException( pht( 'Unable to acquire lease "%s" on any resource.', $lease->getPHID()), $exceptions); } } } /** * Get all the @{class:DrydockBlueprintImplementation}s which can possibly * build a resource to satisfy a lease. * * This method returns blueprints which might, at some time, be able to * build a resource which can satisfy the lease. They may not be able to * build that resource right now. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprint * implementations. * @task allocator */ private function loadBlueprintImplementationsForAllocatingLease( DrydockLease $lease) { $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); $keep = array(); foreach ($impls as $key => $impl) { // Don't use disabled blueprint types. if (!$impl->isEnabled()) { continue; } // Don't use blueprint types which can't allocate the correct kind of // resource. if ($impl->getType() != $lease->getResourceType()) { continue; } if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $impl; } return $keep; } /** * Get all the concrete @{class:DrydockBlueprint}s which can possibly * build a resource to satisfy a lease. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprints. * @task allocator */ private function loadBlueprintsForAllocatingLease( DrydockLease $lease) { $viewer = $this->getViewer(); $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); if (!$impls) { return array(); } $blueprint_phids = $lease->getAllowedBlueprintPHIDs(); if (!$blueprint_phids) { $lease->logEvent(DrydockLeaseNoBlueprintsLogType::LOGCONST); return array(); } $query = id(new DrydockBlueprintQuery()) ->setViewer($viewer) ->withPHIDs($blueprint_phids) ->withBlueprintClasses(array_keys($impls)) ->withDisabled(false); // The Drydock application itself is allowed to authorize anything. This // is primarily used for leases generated by CLI administrative tools. $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $authorizing_phid = $lease->getAuthorizingPHID(); if ($authorizing_phid != $drydock_phid) { $blueprints = id(clone $query) ->withAuthorizedPHIDs(array($authorizing_phid)) ->execute(); if (!$blueprints) { // If we didn't hit any blueprints, check if this is an authorization // problem: re-execute the query without the authorization constraint. // If the second query hits blueprints, the overall configuration is // fine but this is an authorization problem. If the second query also // comes up blank, this is some other kind of configuration issue so // we fall through to the default pathway. $all_blueprints = $query->execute(); if ($all_blueprints) { $lease->logEvent( DrydockLeaseNoAuthorizationsLogType::LOGCONST, array( 'authorizingPHID' => $authorizing_phid, )); return array(); } } } else { $blueprints = $query->execute(); } $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Load a list of all resources which a given lease can possibly be * allocated against. * * @param list Blueprints which may produce suitable * resources. * @param DrydockLease Requested lease. * @return list Resources which may be able to allocate * the lease. * @task allocator */ private function loadResourcesForAllocatingLease( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $viewer = $this->getViewer(); $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) ->withTypes(array($lease->getResourceType())) ->withStatuses( array( DrydockResourceStatus::STATUS_PENDING, DrydockResourceStatus::STATUS_ACTIVE, )) ->execute(); return $this->removeUnacquirableResources($resources, $lease); } /** * Remove resources which can not be acquired by a given lease from a list. * * @param list Candidate resources. * @param DrydockLease Acquiring lease. * @return list Resources which the lease may be able to * acquire. * @task allocator */ private function removeUnacquirableResources( array $resources, DrydockLease $lease) { $keep = array(); foreach ($resources as $key => $resource) { $blueprint = $resource->getBlueprint(); if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { continue; } $keep[$key] = $resource; } return $keep; } /** * Remove blueprints which are too heavily allocated to build a resource for * a lease from a list of blueprints. * * @param list List of blueprints. * @return list List with blueprints that can not allocate * a resource for the lease right now removed. * @task allocator */ private function removeOverallocatedBlueprints( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Rank blueprints by suitability for building a new resource for a * particular lease. * * @param list List of blueprints. * @param DrydockLease Requested lease. * @return list Ranked list of blueprints. * @task allocator */ private function rankBlueprints(array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($blueprints); return $blueprints; } /** * Rank resources by suitability for allocating a particular lease. * * @param list List of resources. * @param DrydockLease Requested lease. * @return list Ranked list of resources. * @task allocator */ private function rankResources(array $resources, DrydockLease $lease) { assert_instances_of($resources, 'DrydockResource'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($resources); return $resources; } /** * Perform an actual resource allocation with a particular blueprint. * * @param DrydockBlueprint The blueprint to allocate a resource from. * @param DrydockLease Requested lease. * @return DrydockResource Allocated resource. * @task allocator */ private function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $blueprint->allocateResource($lease); $this->validateAllocatedResource($blueprint, $resource, $lease); // If this resource was allocated as a pending resource, queue a task to // activate it. if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { PhabricatorWorker::scheduleTask( 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $resource->getPHID(), // This task will generally yield while the resource activates, so // wake it back up once the resource comes online. Most of the time, // we'll be able to lease the newly activated resource. 'awakenOnActivation' => array( $this->getCurrentWorkerTaskID(), ), ), array( 'objectPHID' => $resource->getPHID(), )); } return $resource; } /** * Check that the resource a blueprint allocated is roughly the sort of * object we expect. * * @param DrydockBlueprint Blueprint which built the resource. * @param wild Thing which the blueprint claims is a valid resource. * @param DrydockLease Lease the resource was allocated for. * @return void * @task allocator */ private function validateAllocatedResource( DrydockBlueprint $blueprint, $resource, DrydockLease $lease) { if (!($resource instanceof DrydockResource)) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. 'return an object of type %s or throw, but returned something else.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()', 'DrydockResource')); } if (!$resource->isAllocatedResource()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 'must actually allocate the resource it returns.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()')); } $resource_type = $resource->getType(); $lease_type = $lease->getResourceType(); if ($resource_type !== $lease_type) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'built a resource of type "%s" to satisfy a lease requesting a '. 'resource of type "%s".', $blueprint->getBlueprintName(), $blueprint->getClassName(), $resource_type, $lease_type)); } } private function reclaimResources( DrydockBlueprint $blueprint, DrydockLease $lease) { $viewer = $this->getViewer(); // If this lease is marked as already in the process of reclaiming a // resource, don't let it reclaim another one until the first reclaim // completes. This stops one lease from reclaiming a large number of // resources if the reclaims take a while to complete. $reclaiming_phid = $lease->getAttribute('drydock.reclaimingPHID'); if ($reclaiming_phid) { $reclaiming_resource = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withPHIDs(array($reclaiming_phid)) ->withStatuses( array( DrydockResourceStatus::STATUS_ACTIVE, DrydockResourceStatus::STATUS_RELEASED, )) ->executeOne(); if ($reclaiming_resource) { return null; } } $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(array($blueprint->getPHID())) ->withStatuses( array( DrydockResourceStatus::STATUS_ACTIVE, )) ->execute(); // TODO: We could be much smarter about this and try to release long-unused // resources, resources with many similar copies, old resources, resources // that are cheap to rebuild, etc. shuffle($resources); foreach ($resources as $resource) { if ($this->canReclaimResource($resource)) { $this->reclaimResource($resource, $lease); return $resource; } } return null; } /* -( Acquiring Leases )--------------------------------------------------- */ /** * Perform an actual lease acquisition on a particular resource. * * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Lease to acquire. * @return void * @task acquire */ private function acquireLease( DrydockResource $resource, DrydockLease $lease) { $blueprint = $resource->getBlueprint(); $blueprint->acquireLease($resource, $lease); $this->validateAcquiredLease($blueprint, $resource, $lease); // If this lease has been acquired but not activated, queue a task to // activate it. if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { $this->queueTask( __CLASS__, array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); } } /** * Make sure that a lease was really acquired properly. * * @param DrydockBlueprint Blueprint which created the resource. * @param DrydockResource Resource which was acquired. * @param DrydockLease The lease which was supposedly acquired. * @return void * @task acquire */ private function validateAcquiredLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isAcquiredLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without acquiring a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } $lease_phid = $lease->getResourcePHID(); $resource_phid = $resource->getPHID(); if ($lease_phid !== $resource_phid) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" with a lease acquired on the wrong resource.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } + private function newResourceForAcquisition( + DrydockResource $resource, + DrydockLease $lease) { + + // If the resource has no leases against it, never build a new one. This is + // likely already a new resource that just activated. + $viewer = $this->getViewer(); + + $statuses = array( + DrydockLeaseStatus::STATUS_PENDING, + DrydockLeaseStatus::STATUS_ACQUIRED, + DrydockLeaseStatus::STATUS_ACTIVE, + ); + + $leases = id(new DrydockLeaseQuery()) + ->setViewer($viewer) + ->withResourcePHIDs(array($resource->getPHID())) + ->withStatuses($statuses) + ->setLimit(1) + ->execute(); + if (!$leases) { + return $resource; + } + + // If we're about to get a lease on a resource, check if the blueprint + // wants to allocate a supplemental resource. If it does, try to perform a + // new allocation instead. + $blueprint = $resource->getBlueprint(); + if (!$blueprint->shouldAllocateSupplementalResource($resource, $lease)) { + return $resource; + } + + // If the blueprint is already overallocated, we can't allocate a new + // resource. Just return the existing resource. + $remaining = $this->removeOverallocatedBlueprints( + array($blueprint), + $lease); + if (!$remaining) { + return $resource; + } + + // Try to build a new resource. + try { + $new_resource = $this->allocateResource($blueprint, $lease); + } catch (Exception $ex) { + $blueprint->logEvent( + DrydockResourceAllocationFailureLogType::LOGCONST, + array( + 'class' => get_class($ex), + 'message' => $ex->getMessage(), + )); + + return $resource; + } + + // If we can't actually acquire the new resource yet, just yield. + // (We could try to move forward with the original resource instead.) + $acquirable = $this->removeUnacquirableResources( + array($new_resource), + $lease); + if (!$acquirable) { + throw new PhabricatorWorkerYieldException(15); + } + + return $new_resource; + } + /* -( Activating Leases )-------------------------------------------------- */ /** * @task activate */ private function activateLease(DrydockLease $lease) { $resource = $lease->getResource(); if (!$resource) { throw new Exception( pht('Trying to activate lease with no resource.')); } $resource_status = $resource->getStatus(); if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { throw new PhabricatorWorkerYieldException(15); } if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { throw new DrydockAcquiredBrokenResourceException( pht( 'Trying to activate lease ("%s") on a resource ("%s") in '. 'the wrong status ("%s").', $lease->getPHID(), $resource->getPHID(), $resource_status)); } // NOTE: We can race resource destruction here. Between the time we // performed the read above and now, the resource might have closed, so // we may activate leases on dead resources. At least for now, this seems // fine: a resource dying right before we activate a lease on it should not // be distinguishable from a resource dying right after we activate a lease // on it. We end up with an active lease on a dead resource either way, and // can not prevent resources dying from lightning strikes. $blueprint = $resource->getBlueprint(); $blueprint->activateLease($resource, $lease); $this->validateActivatedLease($blueprint, $resource, $lease); } /** * @task activate */ private function validateActivatedLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isActivatedLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without activating a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } /* -( Releasing Leases )--------------------------------------------------- */ /** * @task release */ private function releaseLease(DrydockLease $lease) { $lease ->setStatus(DrydockLeaseStatus::STATUS_RELEASED) ->save(); $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST); $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); } $this->destroyLease($lease); } /* -( Breaking Leases )---------------------------------------------------- */ /** * @task break */ protected function breakLease(DrydockLease $lease, Exception $ex) { switch ($lease->getStatus()) { case DrydockLeaseStatus::STATUS_BROKEN: case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: throw new PhutilProxyException( pht( 'Unexpected failure while destroying lease ("%s").', $lease->getPHID()), $ex); } $lease ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) ->save(); $lease->logEvent( DrydockLeaseActivationFailureLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); $lease->awakenTasks(); $this->queueTask( __CLASS__, array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); throw new PhabricatorWorkerPermanentFailureException( pht( 'Permanent failure while activating lease ("%s"): %s', $lease->getPHID(), $ex->getMessage())); } /* -( Destroying Leases )-------------------------------------------------- */ /** * @task destroy */ private function destroyLease(DrydockLease $lease) { $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->destroyLease($resource, $lease); } DrydockSlotLock::releaseLocks($lease->getPHID()); $lease ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) ->save(); $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); $lease->awakenTasks(); } }