diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -570,6 +570,7 @@ 'DoorkeeperRemarkupRuleJIRA' => 'applications/doorkeeper/remarkup/DoorkeeperRemarkupRuleJIRA.php', 'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php', 'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php', + 'DrydockAllocationContext' => 'applications/drydock/util/DrydockAllocationContext.php', 'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php', 'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php', 'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php', @@ -620,6 +621,7 @@ 'DrydockManagementLeaseWorkflow' => 'applications/drydock/management/DrydockManagementLeaseWorkflow.php', 'DrydockManagementReleaseWorkflow' => 'applications/drydock/management/DrydockManagementReleaseWorkflow.php', 'DrydockManagementWorkflow' => 'applications/drydock/management/DrydockManagementWorkflow.php', + 'DrydockMinMaxTestBlueprintImplementation' => 'applications/drydock/blueprint/DrydockMinMaxTestBlueprintImplementation.php', 'DrydockPreallocatedHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php', 'DrydockQuery' => 'applications/drydock/query/DrydockQuery.php', 'DrydockResource' => 'applications/drydock/storage/DrydockResource.php', @@ -3327,6 +3329,7 @@ 'DoorkeeperRemarkupRuleJIRA' => 'DoorkeeperRemarkupRule', 'DoorkeeperTagView' => 'AphrontView', 'DoorkeeperTagsController' => 'PhabricatorController', + 'DrydockAllocationContext' => 'Phobject', 'DrydockAllocatorWorker' => 'PhabricatorWorker', 'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface', 'DrydockBlueprint' => array( @@ -3386,6 +3389,7 @@ 'DrydockManagementLeaseWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementReleaseWorkflow' => 'DrydockManagementWorkflow', 'DrydockManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'DrydockMinMaxTestBlueprintImplementation' => 'DrydockBlueprintImplementation', 'DrydockPreallocatedHostBlueprintImplementation' => 'DrydockBlueprintImplementation', 'DrydockQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'DrydockResource' => array( diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -19,6 +19,10 @@ abstract public function isEnabled(); + public function isTest() { + return false; + } + abstract public function getBlueprintName(); abstract public function getDescription(); @@ -42,7 +46,7 @@ return $lease; } - protected function getInstance() { + public function getInstance() { if (!$this->instance) { throw new Exception( 'Attach the blueprint instance to the implementation.'); @@ -127,6 +131,11 @@ $allocated = false; $allocation_exception = null; + $context = new DrydockAllocationContext( + $this->getInstance(), + $resource, + $lease); + $resource->openTransaction(); $resource->beginReadLocking(); $resource->reload(); @@ -142,9 +151,9 @@ try { $allocated = $this->shouldAllocateLease( + $context, $resource, - $ephemeral_lease, - $other_leases); + $ephemeral_lease); } catch (Exception $ex) { $allocation_exception = $ex; } @@ -197,19 +206,15 @@ * better implemented in @{method:canAllocateLease}, which serves as a * cheap filter before lock acquisition. * + * @param DrydockAllocationContext Relevant contextual information. * @param DrydockResource Candidate resource to allocate the lease on. * @param DrydockLease Pending lease that wants to allocate here. - * @param list Other allocated and acquired leases on the - * resource. The implementation can inspect them - * to verify it can safely add the new lease. - * @return bool True to allocate the lease on the resource; - * false to reject it. * @task lease */ abstract protected function shouldAllocateLease( + DrydockAllocationContext $context, DrydockResource $resource, - DrydockLease $lease, - array $other_leases); + DrydockLease $lease); /** @@ -265,10 +270,25 @@ DrydockLease $lease); + /** + * Release an allocated lease, performing any desired cleanup. + * + * After this method executes, the lease status is moved to `RELEASED`. + * + * If release fails, throw an exception. + * + * @param DrydockResource Resource to release the lease from. + * @param DrydockLease Lease to release. + * @return void + */ + abstract protected function executeReleaseLease( + DrydockResource $resource, + DrydockLease $lease); final public function releaseLease( DrydockResource $resource, - DrydockLease $lease) { + DrydockLease $lease, + $caused_by_closing_resource = false) { $scope = $this->pushActiveScope(null, $lease); $released = false; @@ -286,10 +306,36 @@ $lease->endReadLocking(); $lease->saveTransaction(); + // Execute clean up outside of the lock and don't perform clean up if the + // resource is closing anyway, because in that scenario, the closing + // resource will clean up all the leases anyway (e.g. an EC2 host being + // terminated that contains leases on it's instance storage). + if ($released && !$caused_by_closing_resource) { + $this->executeReleaseLease($resource, $lease); + } + if (!$released) { throw new Exception('Unable to release lease: lease not active!'); } + if (!$caused_by_closing_resource) { + // Check to see if the resource has no more leases, and if so, ask the + // blueprint as to whether this resource should be closed. + $context = new DrydockAllocationContext( + $this->getInstance(), + $resource, + $lease); + + if ($context->getCurrentResourceLeaseCount() === 0) { + if ($this->shouldCloseUnleasedResource($context, $resource)) { + DrydockBlueprintImplementation::writeLog( + $resource, + null, + pht('Closing resource because it has no more active leases')); + $this->closeResource($resource); + } + } + } } @@ -301,10 +347,79 @@ return true; } - abstract protected function executeAllocateResource(DrydockLease $lease); + abstract protected function executeAllocateResource( + DrydockResource $resource, + DrydockLease $lease); + + /** + * Closes a previously allocated resource, performing any desired + * cleanup. + * + * After this method executes, the release status is moved to `CLOSED`. + * + * If release fails, throw an exception. + * + * @param DrydockResource Resource to close. + * @return void + */ + abstract protected function executeCloseResource( + DrydockResource $resource); + + /** + * Return whether or not a resource that now has no leases on it + * should be automatically closed. + * + * @param DrydockAllocationContext Relevant contextual information. + * @param DrydockResource The resource that has no more leases on it. + * @return bool + */ + abstract protected function shouldCloseUnleasedResource( + DrydockAllocationContext $context, + DrydockResource $resource); + + final public function closeResource(DrydockResource $resource) { + $resource->openTransaction(); + $resource->setStatus(DrydockResourceStatus::STATUS_CLOSING); + $resource->save(); + + $statuses = array( + DrydockLeaseStatus::STATUS_PENDING, + DrydockLeaseStatus::STATUS_ACTIVE, + ); + + $leases = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withResourceIDs(array($resource->getID())) + ->withStatuses($statuses) + ->execute(); + + foreach ($leases as $lease) { + switch ($lease->getStatus()) { + case DrydockLeaseStatus::STATUS_PENDING: + $message = pht('Breaking pending lease (resource closing).');; + $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); + break; + case DrydockLeaseStatus::STATUS_ACTIVE: + $message = pht('Releasing active lease (resource closing).'); + $this->releaseLease($resource, $lease, true); + $lease->setStatus(DrydockLeaseStatus::STATUS_RELEASED); + break; + } + DrydockBlueprintImplementation::writeLog($resource, $lease, $message); + $lease->save(); + } + + $this->executeCloseResource($resource); + $resource->setStatus(DrydockResourceStatus::STATUS_CLOSED); + $resource->save(); + $resource->saveTransaction(); + } + + final public function allocateResource( + DrydockResource $resource, + DrydockLease $lease) { - final public function allocateResource(DrydockLease $lease) { $scope = $this->pushActiveScope(null, $lease); $this->log( @@ -314,7 +429,7 @@ $lease->getLeaseName())); try { - $resource = $this->executeAllocateResource($lease); + $this->executeAllocateResource($resource, $lease); $this->validateAllocatedResource($resource); } catch (Exception $ex) { $this->logException($ex); @@ -401,25 +516,6 @@ return idx(self::getAllBlueprintImplementations(), $class); } - protected function newResourceTemplate($name) { - $resource = id(new DrydockResource()) - ->setBlueprintPHID($this->getInstance()->getPHID()) - ->setBlueprintClass($this->getBlueprintClass()) - ->setType($this->getType()) - ->setStatus(DrydockResourceStatus::STATUS_PENDING) - ->setName($name) - ->save(); - - $this->activeResource = $resource; - - $this->log( - pht( - "Blueprint '%s': Created New Template", - $this->getBlueprintClass())); - - return $resource; - } - /** * Sanity checks that the blueprint is implemented properly. */ diff --git a/src/applications/drydock/blueprint/DrydockMinMaxTestBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockMinMaxTestBlueprintImplementation.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/blueprint/DrydockMinMaxTestBlueprintImplementation.php @@ -0,0 +1,181 @@ +getDetail('max-count'); + return count($pool) < $max_count; + } + + protected function executeAllocateResource( + DrydockResource $resource, + DrydockLease $lease) { + + $path = '/srv/alloctest/'.$resource->getID(); + + $resource + ->setName($path) + ->setStatus(DrydockResourceStatus::STATUS_PENDING) + ->setAttributes(array( + 'path' => $path)) + ->save(); + + mkdir($path); + + sleep(10); + + $resource->setStatus(DrydockResourceStatus::STATUS_OPEN); + $resource->save(); + } + + protected function canAllocateLease( + DrydockResource $resource, + DrydockLease $lease) { + + return true; + } + + protected function shouldAllocateLease( + DrydockAllocationContext $context, + DrydockResource $resource, + DrydockLease $lease) { + + // If the current resource can allocate a lease, allow it. + if ($context->getCurrentResourceLeaseCount() < + $this->getDetail('leases-per-resource')) { + return true; + } + + // We don't have enough room under the `leases-per-instance` limit, but + // this limit can be bypassed if we've allocated all of the resources + // we allow. + $open_count = $context->getBlueprintOpenResourceCount(); + if ($open_count < $this->getDetail('max-count')) { + return false; + } + + // Find the resource that has the least leases. + $all_lease_counts_grouped = $context->getResourceLeaseCounts(); + $minimum_lease_count = $all_lease_counts_grouped[$resource->getID()]; + $minimum_lease_resource_id = $resource->getID(); + foreach ($all_lease_counts_grouped as $resource_id => $lease_count) { + if ($minimum_lease_count > $lease_count) { + $minimum_lease_count = $lease_count; + $minimum_lease_resource_id = $resource_id; + } + } + + // If we are that resource, then allow it, otherwise let the other + // less-leased resource run through this logic and allocate the lease. + return $minimum_lease_resource_id === $resource->getID(); + } + + protected function executeAcquireLease( + DrydockResource $resource, + DrydockLease $lease) { + + while ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { + // This resource is still being set up by another allocator, wait until + // it is set to open. + sleep(5); + $resource->reload(); + } + + $path = $resource->getAttribute('path').'/'.$lease->getID(); + + mkdir($path); + + sleep(10); + + $lease->setAttribute('path', $path); + } + + public function getType() { + return 'storage'; + } + + public function getInterface( + DrydockResource $resource, + DrydockLease $lease, + $type) { + + throw new Exception("No interface of type '{$type}'."); + } + + protected function executeReleaseLease( + DrydockResource $resource, + DrydockLease $lease) { + + execx('rm -R %s', $lease->getAttribute('path')); + } + + protected function shouldCloseUnleasedResource( + DrydockAllocationContext $context, + DrydockResource $resource) { + + return $context->getBlueprintOpenResourceCount() > + $this->getDetail('min-count'); + } + + protected function executeCloseResource(DrydockResource $resource) { + execx('rm -R %s', $resource->getAttribute('path')); + } + + + public function getFieldSpecifications() { + return array( + 'min-count' => array( + 'name' => pht('Minimum Resources'), + 'type' => 'int', + 'required' => true, + 'caption' => pht( + 'The minimum number of resources to keep open in '. + 'this pool at all times.') + ), + 'max-count' => array( + 'name' => pht('Maximum Resources'), + 'type' => 'int', + 'caption' => pht( + 'The maximum number of resources to allow open at any time. '. + 'If the number of resources currently open are equal to '. + '`max-count` and another lease is requested, Drydock will place '. + 'leases on existing resources and thus exceeding '. + '`leases-per-resource`. If this parameter is left blank, then '. + 'this blueprint has no limit on the number of resources it '. + 'can allocate.') + ), + 'leases-per-resource' => array( + 'name' => pht('Maximum Leases Per Resource'), + 'type' => 'int', + 'required' => true, + 'caption' => pht( + 'The soft limit on the number of leases to allocate to an '. + 'individual resource in the pool. Drydock will choose the '. + 'resource with the lowest number of leases when selecting a '. + 'resource to lease on. If all current resources have '. + '`leases-per-resource` leases on them, then Drydock will allocate '. + 'another resource providing `max-count` would not be exceeded.'. + ' If `max-count` would be exceeded, Drydock will instead '. + 'overallocate the lease to an existing resource and '. + 'exceed the limit specified here.') + ), + ); + } +} diff --git a/src/applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockPreallocatedHostBlueprintImplementation.php @@ -19,7 +19,10 @@ return false; } - protected function executeAllocateResource(DrydockLease $lease) { + protected function executeAllocateResource( + DrydockResource $resource, + DrydockLease $lease) { + throw new Exception("Preallocated hosts can't be dynamically allocated."); } @@ -31,9 +34,9 @@ } protected function shouldAllocateLease( + DrydockAllocationContext $context, DrydockResource $resource, - DrydockLease $lease, - array $other_leases) { + DrydockLease $lease) { return true; } @@ -121,4 +124,22 @@ throw new Exception("No interface of type '{$type}'."); } + protected function executeReleaseLease( + DrydockResource $resource, + DrydockLease $lease) { + + // TODO: Remove leased directory + } + + protected function shouldCloseUnleasedResource( + DrydockAllocationContext $context, + DrydockResource $resource) { + + return false; + } + + protected function executeCloseResource(DrydockResource $resource) { + + } + } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -26,14 +26,17 @@ } protected function shouldAllocateLease( + DrydockAllocationContext $context, DrydockResource $resource, - DrydockLease $lease, - array $other_leases) { + DrydockLease $lease) { - return !$other_leases; + return $context->getCurrentResourceLeaseCount() === 0; } - protected function executeAllocateResource(DrydockLease $lease) { + protected function executeAllocateResource( + DrydockResource $resource, + DrydockLease $lease) { + $repository_id = $lease->getAttribute('repositoryID'); if (!$repository_id) { throw new Exception( @@ -74,13 +77,13 @@ $this->log(pht('Complete.')); - $resource = $this->newResourceTemplate( - 'Working Copy ('.$repository->getCallsign().')'); - $resource->setStatus(DrydockResourceStatus::STATUS_OPEN); - $resource->setAttribute('lease.host', $host_lease->getID()); - $resource->setAttribute('path', $path); - $resource->setAttribute('repositoryID', $repository->getID()); - $resource->save(); + $resource + ->setName('Working Copy ('.$repository->getCallsign().')') + ->setStatus(DrydockResourceStatus::STATUS_OPEN) + ->setAttribute('lease.host', $host_lease->getID()) + ->setAttribute('path', $path) + ->setAttribute('repositoryID', $repository->getID()) + ->save(); return $resource; } @@ -110,4 +113,20 @@ throw new Exception("No interface of type '{$type}'."); } + protected function executeReleaseLease( + DrydockResource $resource, + DrydockLease $lease) { + } + + protected function shouldCloseUnleasedResource( + DrydockAllocationContext $context, + DrydockResource $resource) { + + return false; + } + + protected function executeCloseResource(DrydockResource $resource) { + // TODO: Remove leased directory + } + } diff --git a/src/applications/drydock/constants/DrydockResourceStatus.php b/src/applications/drydock/constants/DrydockResourceStatus.php --- a/src/applications/drydock/constants/DrydockResourceStatus.php +++ b/src/applications/drydock/constants/DrydockResourceStatus.php @@ -7,12 +7,16 @@ const STATUS_CLOSED = 2; const STATUS_BROKEN = 3; const STATUS_DESTROYED = 4; + const STATUS_CLOSING = 5; + const STATUS_ALLOCATING = 6; public static function getNameForStatus($status) { $map = array( self::STATUS_PENDING => pht('Pending'), + self::STATUS_ALLOCATING => pht('Allocating'), self::STATUS_OPEN => pht('Open'), self::STATUS_CLOSED => pht('Closed'), + self::STATUS_CLOSING => pht('Closing'), self::STATUS_BROKEN => pht('Broken'), self::STATUS_DESTROYED => pht('Destroyed'), ); @@ -23,8 +27,10 @@ public static function getAllStatuses() { return array( self::STATUS_PENDING, + self::STATUS_ALLOCATING, self::STATUS_OPEN, self::STATUS_CLOSED, + self::STATUS_CLOSING, self::STATUS_BROKEN, self::STATUS_DESTROYED, ); diff --git a/src/applications/drydock/controller/DrydockBlueprintCreateController.php b/src/applications/drydock/controller/DrydockBlueprintCreateController.php --- a/src/applications/drydock/controller/DrydockBlueprintCreateController.php +++ b/src/applications/drydock/controller/DrydockBlueprintCreateController.php @@ -35,6 +35,11 @@ ->setError($e_blueprint); foreach ($implementations as $implementation_name => $implementation) { + if ($implementation->isTest()) { + // Never show testing blueprints in the interface. + continue; + } + $disabled = !$implementation->isEnabled(); $control->addButton( diff --git a/src/applications/drydock/storage/DrydockResource.php b/src/applications/drydock/storage/DrydockResource.php --- a/src/applications/drydock/storage/DrydockResource.php +++ b/src/applications/drydock/storage/DrydockResource.php @@ -62,36 +62,8 @@ } public function closeResource() { - $this->openTransaction(); - $statuses = array( - DrydockLeaseStatus::STATUS_PENDING, - DrydockLeaseStatus::STATUS_ACTIVE, - ); - - $leases = id(new DrydockLeaseQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withResourceIDs(array($this->getID())) - ->withStatuses($statuses) - ->execute(); - - foreach ($leases as $lease) { - switch ($lease->getStatus()) { - case DrydockLeaseStatus::STATUS_PENDING: - $message = pht('Breaking pending lease (resource closing).'); - $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); - break; - case DrydockLeaseStatus::STATUS_ACTIVE: - $message = pht('Releasing active lease (resource closing).'); - $lease->setStatus(DrydockLeaseStatus::STATUS_RELEASED); - break; - } - DrydockBlueprintImplementation::writeLog($this, $lease, $message); - $lease->save(); - } - - $this->setStatus(DrydockResourceStatus::STATUS_CLOSED); - $this->save(); - $this->saveTransaction(); + $blueprint = $this->getBlueprint(); + $blueprint->closeResource($this); } diff --git a/src/applications/drydock/util/DrydockAllocationContext.php b/src/applications/drydock/util/DrydockAllocationContext.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/util/DrydockAllocationContext.php @@ -0,0 +1,94 @@ +getTableName(); + $table_resource = id(new DrydockResource())->getTableName(); + $table_lease = id(new DrydockLease())->getTableName(); + + $conn = $blueprint->establishConnection('r'); + + $result = queryfx_one( + $conn, + 'SELECT COUNT(id) AS count '. + 'FROM %T '. + 'WHERE blueprintPHID = %s '. + 'AND status IN (%Ld)', + $table_resource, + $blueprint->getPHID(), + array( + DrydockResourceStatus::STATUS_PENDING, + DrydockResourceStatus::STATUS_OPEN, + DrydockResourceStatus::STATUS_ALLOCATING)); + $this->setBlueprintOpenResourceCount($result['count']); + + $results = queryfx_all( + $conn, + 'SELECT '. + ' resource.id AS resourceID, '. + ' COUNT(lease.id) AS leaseCount '. + 'FROM %T AS resource '. + 'LEFT JOIN %T AS lease '. + ' ON lease.resourceID = resource.id '. + 'WHERE resource.blueprintPHID = %s '. + 'AND resource.status IN (%Ld) '. + 'AND lease.status IN (%Ld) ', + $table_resource, + $table_lease, + $blueprint->getPHID(), + array( + DrydockResourceStatus::STATUS_PENDING, + DrydockResourceStatus::STATUS_OPEN, + DrydockResourceStatus::STATUS_ALLOCATING), + array( + DrydockLeaseStatus::STATUS_PENDING, + DrydockLeaseStatus::STATUS_ACQUIRING, + DrydockLeaseStatus::STATUS_ACTIVE)); + $results = ipull($results, 'leaseCount', 'resourceID'); + $this->setResourceLeaseCounts($results); + + if ($resource !== null) { + $this->setCurrentResourceLeaseCount(idx($results, $resource->getID(), 0)); + } + + // $lease is not yet used, but it's passed in so we can add additional + // contextual statistics later. + } + + public function setBlueprintOpenResourceCount($blueprint_resource_count) { + $this->blueprintOpenResourceCount = $blueprint_resource_count; + return $this; + } + + public function getBlueprintOpenResourceCount() { + return $this->blueprintOpenResourceCount; + } + + public function setResourceLeaseCounts($resource_lease_counts) { + $this->resourceLeaseCounts = $resource_lease_counts; + return $this; + } + + public function getResourceLeaseCounts() { + return $this->resourceLeaseCounts; + } + + public function setCurrentResourceLeaseCount($resource_lease_counts) { + $this->currentResourceLeaseCount = $resource_lease_counts; + return $this; + } + + public function getCurrentResourceLeaseCount() { + return $this->currentResourceLeaseCount; + } + +} diff --git a/src/applications/drydock/view/DrydockResourceListView.php b/src/applications/drydock/view/DrydockResourceListView.php --- a/src/applications/drydock/view/DrydockResourceListView.php +++ b/src/applications/drydock/view/DrydockResourceListView.php @@ -26,6 +26,7 @@ $item->addAttribute($status); switch ($resource->getStatus()) { + case DrydockResourceStatus::STATUS_ALLOCATING: case DrydockResourceStatus::STATUS_PENDING: $item->setBarColor('yellow'); break; diff --git a/src/applications/drydock/worker/DrydockAllocatorWorker.php b/src/applications/drydock/worker/DrydockAllocatorWorker.php --- a/src/applications/drydock/worker/DrydockAllocatorWorker.php +++ b/src/applications/drydock/worker/DrydockAllocatorWorker.php @@ -72,6 +72,9 @@ $blueprints = $this->loadAllBlueprints(); + $lock = PhabricatorGlobalLock::newLock('drydockallocation'); + $lock->lock(10000); + // TODO: Policy stuff. $pool = id(new DrydockResource())->loadAllWhere( 'type = %s AND status = %s', @@ -112,48 +115,132 @@ } if (!$resource) { - $blueprints = DrydockBlueprintImplementation - ::getAllBlueprintImplementationsForResource($type); + // Attempt to use pending resources if we can. + $pool = id(new DrydockResource())->loadAllWhere( + 'type = %s AND status = %s', + $lease->getResourceType(), + DrydockResourceStatus::STATUS_PENDING); $this->logToDrydock( - pht('Found %d Blueprints', count($blueprints))); + pht('Found %d Pending Resource(s)', count($pool))); - foreach ($blueprints as $key => $candidate_blueprint) { - if (!$candidate_blueprint->isEnabled()) { - unset($blueprints[$key]); + $candidates = array(); + foreach ($pool as $key => $candidate) { + if (!isset($blueprints[$candidate->getBlueprintPHID()])) { + unset($pool[$key]); continue; } + + $blueprint = $blueprints[$candidate->getBlueprintPHID()]; + $implementation = $blueprint->getImplementation(); + + if ($implementation->filterResource($candidate, $lease)) { + $candidates[] = $candidate; + } } $this->logToDrydock( - pht('%d Blueprints Enabled', count($blueprints))); + pht('%d Pending Resource(s) Remain', + count($candidates))); + + $resource = null; + if ($candidates) { + shuffle($candidates); + foreach ($candidates as $candidate_resource) { + $blueprint = $blueprints[$candidate_resource->getBlueprintPHID()] + ->getImplementation(); + if ($blueprint->allocateLease($candidate_resource, $lease)) { + $resource = $candidate_resource; + break; + } + } + } + } + + if ($resource) { + $lock->unlock(); + } else { + $blueprints = id(new DrydockBlueprintQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->execute(); + $blueprints = mpull($blueprints, 'getImplementation', 'getPHID'); + + $this->logToDrydock( + pht('Found %d Blueprints', count($blueprints))); foreach ($blueprints as $key => $candidate_blueprint) { - if (!$candidate_blueprint->canAllocateMoreResources($pool)) { + if (!$candidate_blueprint->isEnabled()) { unset($blueprints[$key]); continue; } } $this->logToDrydock( - pht('%d Blueprints Can Allocate', count($blueprints))); + pht('%d Blueprints Enabled', count($blueprints))); - if (!$blueprints) { - $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); - $lease->save(); + $resources_per_blueprint = id(new DrydockResourceQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withStatuses(array( + DrydockResourceStatus::STATUS_PENDING, + DrydockResourceStatus::STATUS_OPEN, + DrydockResourceStatus::STATUS_ALLOCATING)) + ->execute(); + $resources_per_blueprint = mgroup( + $resources_per_blueprint, + 'getBlueprintPHID'); + + try { + foreach ($blueprints as $key => $candidate_blueprint) { + $rpool = idx($resources_per_blueprint, $key, array()); + if (!$candidate_blueprint->canAllocateMoreResources($rpool)) { + unset($blueprints[$key]); + continue; + } + } $this->logToDrydock( - "There are no resources of type '{$type}' available, and no ". - "blueprints which can allocate new ones."); + pht('%d Blueprints Can Allocate', count($blueprints))); - return; - } + if (!$blueprints) { + $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); + $lease->save(); - // TODO: Rank intelligently. - shuffle($blueprints); + $this->logToDrydock( + "There are no resources of type '{$type}' available, and no ". + "blueprints which can allocate new ones."); + + $lock->unlock(); + return; + } - $blueprint = head($blueprints); - $resource = $blueprint->allocateResource($lease); + // TODO: Rank intelligently. + shuffle($blueprints); + + $blueprint = head($blueprints); + + // Create and save the resource preemptively with STATUS_ALLOCATING + // before we unlock, so that other workers will correctly count the + // new resource "to be allocated" when determining if they can allocate + // more resources to a blueprint. + $resource = id(new DrydockResource()) + ->setBlueprintPHID($blueprint->getInstance()->getPHID()) + ->setType($blueprint->getType()) + ->setName(pht('Pending Allocation')) + ->setStatus(DrydockResourceStatus::STATUS_ALLOCATING) + ->save(); + + $lock->unlock(); + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } + + try { + $blueprint->allocateResource($resource, $lease); + } catch (Exception $ex) { + $resource->delete(); + throw $ex; + } if (!$blueprint->allocateLease($resource, $lease)) { // TODO: This "should" happen only if we lost a race with another lease,