diff --git a/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php b/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php index e740073dd9..9a509a5c04 100644 --- a/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php +++ b/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php @@ -1,89 +1,98 @@ getViewer(); $id = $request->getURIData('id'); $operation = id(new DrydockRepositoryOperationQuery()) ->setViewer($viewer) ->withIDs(array($id)) ->executeOne(); if (!$operation) { return new Aphront404Response(); } $id = $operation->getID(); $title = pht('Repository Operation %d', $id); $header = id(new PHUIHeaderView()) ->setHeader($title) ->setUser($viewer) ->setPolicyObject($operation); $state = $operation->getOperationState(); $icon = DrydockRepositoryOperation::getOperationStateIcon($state); $name = DrydockRepositoryOperation::getOperationStateName($state); $header->setStatus($icon, null, $name); $actions = $this->buildActionListView($operation); $properties = $this->buildPropertyListView($operation); $properties->setActionList($actions); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb( pht('Operations'), $this->getApplicationURI('operation/')); $crumbs->addTextCrumb($title); $object_box = id(new PHUIObjectBoxView()) ->setHeader($header) ->addPropertyList($properties); return $this->buildApplicationPage( array( $crumbs, $object_box, ), array( 'title' => $title, )); } private function buildActionListView(DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); $id = $operation->getID(); $view = id(new PhabricatorActionListView()) ->setUser($viewer) ->setObjectURI($this->getRequest()->getRequestURI()) ->setObject($operation); return $view; } private function buildPropertyListView( DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); $view = new PHUIPropertyListView(); $view->addProperty( pht('Repository'), $viewer->renderHandle($operation->getRepositoryPHID())); $view->addProperty( pht('Object'), $viewer->renderHandle($operation->getObjectPHID())); + $lease_phid = $operation->getWorkingCopyLeasePHID(); + if ($lease_phid) { + $lease_display = $viewer->renderHandle($lease_phid); + } else { + $lease_display = phutil_tag('em', array(), pht('None')); + } + + $view->addProperty(pht('Working Copy'), $lease_display); + return $view; } } diff --git a/src/applications/drydock/storage/DrydockLease.php b/src/applications/drydock/storage/DrydockLease.php index af475c5d61..0dd4e36b48 100644 --- a/src/applications/drydock/storage/DrydockLease.php +++ b/src/applications/drydock/storage/DrydockLease.php @@ -1,490 +1,500 @@ setPHID($lease->generatePHID()); return $lease; } /** * Flag this lease to be released when its destructor is called. This is * mostly useful if you have a script which acquires, uses, and then releases * a lease, as you don't need to explicitly handle exceptions to properly * release the lease. */ public function releaseOnDestruction() { $this->releaseOnDestruction = true; return $this; } public function __destruct() { if (!$this->releaseOnDestruction) { return; } if (!$this->canRelease()) { return; } $actor = PhabricatorUser::getOmnipotentUser(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $command = DrydockCommand::initializeNewCommand($actor) ->setTargetPHID($this->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); $this->scheduleUpdate(); } public function getLeaseName() { return pht('Lease %d', $this->getID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'attributes' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'status' => 'text32', 'until' => 'epoch?', 'resourceType' => 'text128', 'ownerPHID' => 'phid?', 'resourcePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_resource' => array( 'columns' => array('resourcePHID', 'status'), ), ), ) + parent::getConfiguration(); } public function setAttribute($key, $value) { $this->attributes[$key] = $value; return $this; } public function getAttribute($key, $default = null) { return idx($this->attributes, $key, $default); } public function generatePHID() { return PhabricatorPHID::generateNewPHID(DrydockLeasePHIDType::TYPECONST); } public function getInterface($type) { return $this->getResource()->getInterface($this, $type); } public function getResource() { return $this->assertAttached($this->resource); } public function attachResource(DrydockResource $resource = null) { $this->resource = $resource; return $this; } public function hasAttachedResource() { return ($this->resource !== null); } public function getUnconsumedCommands() { return $this->assertAttached($this->unconsumedCommands); } public function attachUnconsumedCommands(array $commands) { $this->unconsumedCommands = $commands; return $this; } public function isReleasing() { foreach ($this->getUnconsumedCommands() as $command) { if ($command->getCommand() == DrydockCommand::COMMAND_RELEASE) { return true; } } return false; } public function queueForActivation() { if ($this->getID()) { throw new Exception( pht('Only new leases may be queued for activation!')); } if (!$this->getAuthorizingPHID()) { throw new Exception( pht( 'Trying to queue a lease for activation without an authorizing '. 'object. Use "%s" to specify the PHID of the authorizing object. '. 'The authorizing object must be approved to use the allowed '. 'blueprints.', 'setAuthorizingPHID()')); } if (!$this->getAllowedBlueprintPHIDs()) { throw new Exception( pht( 'Trying to queue a lease for activation without any allowed '. 'Blueprints. Use "%s" to specify allowed blueprints. The '. 'authorizing object must be approved to use the allowed blueprints.', 'setAllowedBlueprintPHIDs()')); } $this ->setStatus(DrydockLeaseStatus::STATUS_PENDING) ->save(); $this->scheduleUpdate(); $this->logEvent(DrydockLeaseQueuedLogType::LOGCONST); return $this; } public function isActivating() { switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: return true; } return false; } public function isActive() { switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_ACTIVE: return true; } return false; } public function waitUntilActive() { while (true) { $lease = $this->reload(); if (!$lease) { throw new Exception(pht('Failed to reload lease.')); } $status = $lease->getStatus(); switch ($status) { case DrydockLeaseStatus::STATUS_ACTIVE: return; case DrydockLeaseStatus::STATUS_RELEASED: throw new Exception(pht('Lease has already been released!')); case DrydockLeaseStatus::STATUS_DESTROYED: throw new Exception(pht('Lease has already been destroyed!')); case DrydockLeaseStatus::STATUS_BROKEN: throw new Exception(pht('Lease has been broken!')); case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: break; default: throw new Exception( pht( 'Lease has unknown status "%s".', $status)); } sleep(1); } } public function setActivateWhenAcquired($activate) { $this->activateWhenAcquired = true; return $this; } public function needSlotLock($key) { $this->slotLocks[] = $key; return $this; } public function acquireOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to acquire a lease on a resource which is in the wrong '. 'state: status must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($this->activateWhenAcquired) { $new_status = DrydockLeaseStatus::STATUS_ACTIVE; } else { $new_status = DrydockLeaseStatus::STATUS_ACQUIRED; } if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { throw new Exception( pht( 'Trying to acquire an active lease on a pending resource. '. 'You can not immediately activate leases on resources which '. 'need time to start up.')); } } $this->openTransaction(); try { DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); } catch (DrydockSlotLockException $ex) { $this->killTransaction(); $this->logEvent( DrydockSlotLockFailureLogType::LOGCONST, array( 'locks' => $ex->getLockMap(), )); throw $ex; } $this ->setResourcePHID($resource->getPHID()) ->attachResource($resource) ->setStatus($new_status) ->save(); $this->saveTransaction(); $this->isAcquired = true; $this->logEvent(DrydockLeaseAcquiredLogType::LOGCONST); if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { $this->didActivate(); } return $this; } public function isAcquiredLease() { return $this->isAcquired; } public function activateOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_ACQUIRED; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to activate a lease which has the wrong status: status '. 'must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { // TODO: Be stricter about this? throw new Exception( pht( 'Trying to activate a lease on a pending resource.')); } $this->openTransaction(); try { DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); } catch (DrydockSlotLockException $ex) { $this->killTransaction(); $this->logEvent( DrydockSlotLockFailureLogType::LOGCONST, array( 'locks' => $ex->getLockMap(), )); throw $ex; } $this ->setStatus(DrydockLeaseStatus::STATUS_ACTIVE) ->save(); $this->saveTransaction(); $this->isActivated = true; $this->didActivate(); return $this; } public function isActivatedLease() { return $this->isActivated; } public function canRelease() { if (!$this->getID()) { return false; } switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: return false; default: return true; } } public function canReceiveCommands() { switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: return false; default: return true; } } public function scheduleUpdate($epoch = null) { PhabricatorWorker::scheduleTask( 'DrydockLeaseUpdateWorker', array( 'leasePHID' => $this->getPHID(), 'isExpireTask' => ($epoch !== null), ), array( 'objectPHID' => $this->getPHID(), 'delayUntil' => ($epoch ? (int)$epoch : null), )); } public function setAwakenTaskIDs(array $ids) { $this->setAttribute('internal.awakenTaskIDs', $ids); return $this; } public function setAllowedBlueprintPHIDs(array $phids) { $this->setAttribute('internal.blueprintPHIDs', $phids); return $this; } public function getAllowedBlueprintPHIDs() { return $this->getAttribute('internal.blueprintPHIDs', array()); } private function didActivate() { $viewer = PhabricatorUser::getOmnipotentUser(); $need_update = false; $this->logEvent(DrydockLeaseActivatedLogType::LOGCONST); $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($this->getPHID())) ->withConsumed(false) ->execute(); if ($commands) { $need_update = true; } if ($need_update) { $this->scheduleUpdate(); } $expires = $this->getUntil(); if ($expires) { $this->scheduleUpdate($expires); } - $awaken_ids = $this->getAttribute('internal.awakenTaskIDs'); - if (is_array($awaken_ids) && $awaken_ids) { - PhabricatorWorker::awakenTaskIDs($awaken_ids); - } + $this->awakenTasks(); } public function logEvent($type, array $data = array()) { $log = id(new DrydockLog()) ->setEpoch(PhabricatorTime::getNow()) ->setType($type) ->setData($data); $log->setLeasePHID($this->getPHID()); $resource_phid = $this->getResourcePHID(); if ($resource_phid) { $resource = $this->getResource(); $log->setResourcePHID($resource->getPHID()); $log->setBlueprintPHID($resource->getBlueprintPHID()); } return $log->save(); } + /** + * Awaken yielded tasks after a state change. + * + * @return this + */ + public function awakenTasks() { + $awaken_ids = $this->getAttribute('internal.awakenTaskIDs'); + if (is_array($awaken_ids) && $awaken_ids) { + PhabricatorWorker::awakenTaskIDs($awaken_ids); + } + + return $this; + } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { if ($this->getResource()) { return $this->getResource()->getPolicy($capability); } // TODO: Implement reasonable policies. return PhabricatorPolicies::getMostOpenPolicy(); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { if ($this->getResource()) { return $this->getResource()->hasAutomaticCapability($capability, $viewer); } return false; } public function describeAutomaticCapability($capability) { return pht('Leases inherit policies from the resources they lease.'); } } diff --git a/src/applications/drydock/storage/DrydockRepositoryOperation.php b/src/applications/drydock/storage/DrydockRepositoryOperation.php index 32e6ded9f3..6d7151b25d 100644 --- a/src/applications/drydock/storage/DrydockRepositoryOperation.php +++ b/src/applications/drydock/storage/DrydockRepositoryOperation.php @@ -1,195 +1,204 @@ setOperationState(self::STATE_WAIT) ->setOperationType($op->getOperationConstant()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'properties' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'repositoryTarget' => 'bytes', 'operationType' => 'text32', 'operationState' => 'text32', ), self::CONFIG_KEY_SCHEMA => array( 'key_object' => array( 'columns' => array('objectPHID'), ), 'key_repository' => array( 'columns' => array('repositoryPHID', 'operationState'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( DrydockRepositoryOperationPHIDType::TYPECONST); } public function attachRepository(PhabricatorRepository $repository) { $this->repository = $repository; return $this; } public function getRepository() { return $this->assertAttached($this->repository); } public function attachObject($object) { $this->object = $object; return $this; } public function getObject() { return $this->assertAttached($this->object); } public function attachImplementation(DrydockRepositoryOperationType $impl) { $this->implementation = $impl; return $this; } public function getImplementation() { return $this->implementation; } public function getProperty($key, $default = null) { return idx($this->properties, $key, $default); } public function setProperty($key, $value) { $this->properties[$key] = $value; return $this; } public static function getOperationStateIcon($state) { $map = array( self::STATE_WAIT => 'fa-clock-o', self::STATE_WORK => 'fa-refresh blue', self::STATE_DONE => 'fa-check green', self::STATE_FAIL => 'fa-times red', ); return idx($map, $state, null); } public static function getOperationStateName($state) { $map = array( self::STATE_WAIT => pht('Waiting'), self::STATE_WORK => pht('Working'), self::STATE_DONE => pht('Done'), self::STATE_FAIL => pht('Failed'), ); return idx($map, $state, pht('', $state)); } public function scheduleUpdate() { PhabricatorWorker::scheduleTask( 'DrydockRepositoryOperationUpdateWorker', array( 'operationPHID' => $this->getPHID(), ), array( 'objectPHID' => $this->getPHID(), 'priority' => PhabricatorWorker::PRIORITY_ALERTS, )); } public function applyOperation(DrydockInterface $interface) { return $this->getImplementation()->applyOperation( $this, $interface); } public function getOperationDescription(PhabricatorUser $viewer) { return $this->getImplementation()->getOperationDescription( $this, $viewer); } public function getOperationCurrentStatus(PhabricatorUser $viewer) { return $this->getImplementation()->getOperationCurrentStatus( $this, $viewer); } public function isUnderway() { switch ($this->getOperationState()) { case self::STATE_WAIT: case self::STATE_WORK: return true; } return false; } public function isDone() { return ($this->getOperationState() === self::STATE_DONE); } public function getWorkingCopyMerges() { return $this->getImplementation()->getWorkingCopyMerges( $this); } + public function setWorkingCopyLeasePHID($lease_phid) { + return $this->setProperty('exec.leasePHID', $lease_phid); + } + + public function getWorkingCopyLeasePHID() { + return $this->getProperty('exec.leasePHID'); + } + + /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { return $this->getRepository()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getRepository()->hasAutomaticCapability($capability, $viewer); } public function describeAutomaticCapability($capability) { return pht( 'A repository operation inherits the policies of the repository it '. 'affects.'); } } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 7d8cc98219..0c93647b89 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,801 +1,805 @@ getTaskDataValue('leasePHID'); $hash = PhabricatorHash::digestForIndex($lease_phid); $lock_key = 'drydock.lease:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $lease = $this->loadLease($lease_phid); $this->handleUpdate($lease); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); } /* -( Updating Leases )---------------------------------------------------- */ /** * @task update */ private function handleUpdate(DrydockLease $lease) { try { $this->updateLease($lease); } catch (Exception $ex) { if ($this->isTemporaryException($ex)) { $this->yieldLease($lease, $ex); } else { $this->breakLease($lease, $ex); } } } /** * @task update */ private function updateLease(DrydockLease $lease) { $this->processLeaseCommands($lease); $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: $this->executeAllocator($lease); break; case DrydockLeaseStatus::STATUS_ACQUIRED: $this->activateLease($lease); break; case DrydockLeaseStatus::STATUS_ACTIVE: // Nothing to do. break; case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_BROKEN: $this->destroyLease($lease); break; case DrydockLeaseStatus::STATUS_DESTROYED: break; } $this->yieldIfExpiringLease($lease); } /** * @task update */ private function yieldLease(DrydockLease $lease, Exception $ex) { $duration = $this->getYieldDurationFromException($ex); $lease->logEvent( DrydockLeaseActivationYieldLogType::LOGCONST, array( 'duration' => $duration, )); throw new PhabricatorWorkerYieldException($duration); } /* -( Processing Commands )------------------------------------------------ */ /** * @task command */ private function processLeaseCommands(DrydockLease $lease) { if (!$lease->canReceiveCommands()) { return; } $this->checkLeaseExpiration($lease); $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { if (!$lease->canReceiveCommands()) { break; } $this->processLeaseCommand($lease, $command); $command ->setIsConsumed(true) ->save(); } } /** * @task command */ private function processLeaseCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseLease($lease); break; } } /* -( Drydock Allocator )-------------------------------------------------- */ /** * Find or build a resource which can satisfy a given lease request, then * acquire the lease. * * @param DrydockLease Requested lease. * @return void * @task allocator */ private function executeAllocator(DrydockLease $lease) { $blueprints = $this->loadBlueprintsForAllocatingLease($lease); // If we get nothing back, that means no blueprint is defined which can // ever build the requested resource. This is a permanent failure, since // we don't expect to succeed no matter how many times we try. if (!$blueprints) { throw new PhabricatorWorkerPermanentFailureException( pht( 'No active Drydock blueprint exists which can ever allocate a '. 'resource for lease "%s".', $lease->getPHID())); } // First, try to find a suitable open resource which we can acquire a new // lease on. $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); // If no resources exist yet, see if we can build one. if (!$resources) { $usable_blueprints = $this->removeOverallocatedBlueprints( $blueprints, $lease); // If we get nothing back here, some blueprint claims it can eventually // satisfy the lease, just not right now. This is a temporary failure, // and we expect allocation to succeed eventually. if (!$usable_blueprints) { $lease->logEvent( DrydockLeaseWaitingForResourcesLogType::LOGCONST, array( 'blueprintPHIDs' => mpull($blueprints, 'getPHID'), )); throw new PhabricatorWorkerYieldException(15); } $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease); $exceptions = array(); foreach ($usable_blueprints as $blueprint) { try { $resources[] = $this->allocateResource($blueprint, $lease); // Bail after allocating one resource, we don't need any more than // this. break; } catch (Exception $ex) { $exceptions[] = $ex; } } if (!$resources) { throw new PhutilAggregateException( pht( 'All blueprints failed to allocate a suitable new resource when '. 'trying to allocate lease "%s".', $lease->getPHID()), $exceptions); } $resources = $this->removeUnacquirableResources($resources, $lease); if (!$resources) { // If we make it here, we just built a resource but aren't allowed // to acquire it. We expect this during routine operation if the // resource prevents acquisition until it activates. Yield and wait // for activation. throw new PhabricatorWorkerYieldException(15); } // NOTE: We have not acquired the lease yet, so it is possible that the // resource we just built will be snatched up by some other lease before // we can acquire it. This is not problematic: we'll retry a little later // and should suceed eventually. } $resources = $this->rankResources($resources, $lease); $exceptions = array(); $allocated = false; foreach ($resources as $resource) { try { $this->acquireLease($resource, $lease); $allocated = true; break; } catch (Exception $ex) { $exceptions[] = $ex; } } if (!$allocated) { throw new PhutilAggregateException( pht( 'Unable to acquire lease "%s" on any resouce.', $lease->getPHID()), $exceptions); } } /** * Get all the @{class:DrydockBlueprintImplementation}s which can possibly * build a resource to satisfy a lease. * * This method returns blueprints which might, at some time, be able to * build a resource which can satisfy the lease. They may not be able to * build that resource right now. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprint * implementations. * @task allocator */ private function loadBlueprintImplementationsForAllocatingLease( DrydockLease $lease) { $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); $keep = array(); foreach ($impls as $key => $impl) { // Don't use disabled blueprint types. if (!$impl->isEnabled()) { continue; } // Don't use blueprint types which can't allocate the correct kind of // resource. if ($impl->getType() != $lease->getResourceType()) { continue; } if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $impl; } return $keep; } /** * Get all the concrete @{class:DrydockBlueprint}s which can possibly * build a resource to satisfy a lease. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprints. * @task allocator */ private function loadBlueprintsForAllocatingLease( DrydockLease $lease) { $viewer = $this->getViewer(); $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); if (!$impls) { return array(); } $query = id(new DrydockBlueprintQuery()) ->setViewer($viewer) ->withBlueprintClasses(array_keys($impls)) ->withDisabled(false); $blueprint_phids = $lease->getAllowedBlueprintPHIDs(); if (!$blueprint_phids) { $lease->logEvent(DrydockLeaseNoBlueprintsLogType::LOGCONST); return array(); } // The Drydock application itself is allowed to authorize anything. This // is primarily used for leases generated by CLI administrative tools. $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $authorizing_phid = $lease->getAuthorizingPHID(); if ($authorizing_phid != $drydock_phid) { $blueprints = id(clone $query) ->withAuthorizedPHIDs(array($authorizing_phid)) ->execute(); if (!$blueprints) { // If we didn't hit any blueprints, check if this is an authorization // problem: re-execute the query without the authorization constraint. // If the second query hits blueprints, the overall configuration is // fine but this is an authorization problem. If the second query also // comes up blank, this is some other kind of configuration issue so // we fall through to the default pathway. $all_blueprints = $query->execute(); if ($all_blueprints) { $lease->logEvent( DrydockLeaseNoAuthorizationsLogType::LOGCONST, array( 'authorizingPHID' => $authorizing_phid, )); return array(); } } } else { $blueprints = $query->execute(); } $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Load a list of all resources which a given lease can possibly be * allocated against. * * @param list Blueprints which may produce suitable * resources. * @param DrydockLease Requested lease. * @return list Resources which may be able to allocate * the lease. * @task allocator */ private function loadResourcesForAllocatingLease( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $viewer = $this->getViewer(); $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) ->withTypes(array($lease->getResourceType())) ->withStatuses( array( DrydockResourceStatus::STATUS_PENDING, DrydockResourceStatus::STATUS_ACTIVE, )) ->execute(); return $this->removeUnacquirableResources($resources, $lease); } /** * Remove resources which can not be acquired by a given lease from a list. * * @param list Candidate resources. * @param DrydockLease Acquiring lease. * @return list Resources which the lease may be able to * acquire. * @task allocator */ private function removeUnacquirableResources( array $resources, DrydockLease $lease) { $keep = array(); foreach ($resources as $key => $resource) { $blueprint = $resource->getBlueprint(); if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { continue; } $keep[$key] = $resource; } return $keep; } /** * Remove blueprints which are too heavily allocated to build a resource for * a lease from a list of blueprints. * * @param list List of blueprints. * @return list List with blueprints that can not allocate * a resource for the lease right now removed. * @task allocator */ private function removeOverallocatedBlueprints( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Rank blueprints by suitability for building a new resource for a * particular lease. * * @param list List of blueprints. * @param DrydockLease Requested lease. * @return list Ranked list of blueprints. * @task allocator */ private function rankBlueprints(array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($blueprints); return $blueprints; } /** * Rank resources by suitability for allocating a particular lease. * * @param list List of resources. * @param DrydockLease Requested lease. * @return list Ranked list of resources. * @task allocator */ private function rankResources(array $resources, DrydockLease $lease) { assert_instances_of($resources, 'DrydockResource'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($resources); return $resources; } /** * Perform an actual resource allocation with a particular blueprint. * * @param DrydockBlueprint The blueprint to allocate a resource from. * @param DrydockLease Requested lease. * @return DrydockResource Allocated resource. * @task allocator */ private function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $blueprint->allocateResource($lease); $this->validateAllocatedResource($blueprint, $resource, $lease); // If this resource was allocated as a pending resource, queue a task to // activate it. if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { PhabricatorWorker::scheduleTask( 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $resource->getPHID(), ), array( 'objectPHID' => $resource->getPHID(), )); } return $resource; } /** * Check that the resource a blueprint allocated is roughly the sort of * object we expect. * * @param DrydockBlueprint Blueprint which built the resource. * @param wild Thing which the blueprint claims is a valid resource. * @param DrydockLease Lease the resource was allocated for. * @return void * @task allocator */ private function validateAllocatedResource( DrydockBlueprint $blueprint, $resource, DrydockLease $lease) { if (!($resource instanceof DrydockResource)) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. 'return an object of type %s or throw, but returned something else.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()', 'DrydockResource')); } if (!$resource->isAllocatedResource()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 'must actually allocate the resource it returns.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()')); } $resource_type = $resource->getType(); $lease_type = $lease->getResourceType(); if ($resource_type !== $lease_type) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'built a resource of type "%s" to satisfy a lease requesting a '. 'resource of type "%s".', $blueprint->getBlueprintName(), $blueprint->getClassName(), $resource_type, $lease_type)); } } /* -( Acquiring Leases )--------------------------------------------------- */ /** * Perform an actual lease acquisition on a particular resource. * * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Lease to acquire. * @return void * @task acquire */ private function acquireLease( DrydockResource $resource, DrydockLease $lease) { $blueprint = $resource->getBlueprint(); $blueprint->acquireLease($resource, $lease); $this->validateAcquiredLease($blueprint, $resource, $lease); // If this lease has been acquired but not activated, queue a task to // activate it. if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { $this->queueTask( __CLASS__, array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); } } /** * Make sure that a lease was really acquired properly. * * @param DrydockBlueprint Blueprint which created the resource. * @param DrydockResource Resource which was acquired. * @param DrydockLease The lease which was supposedly acquired. * @return void * @task acquire */ private function validateAcquiredLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isAcquiredLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without acquiring a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } $lease_phid = $lease->getResourcePHID(); $resource_phid = $resource->getPHID(); if ($lease_phid !== $resource_phid) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" with a lease acquired on the wrong resource.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } /* -( Activating Leases )-------------------------------------------------- */ /** * @task activate */ private function activateLease(DrydockLease $lease) { $resource = $lease->getResource(); if (!$resource) { throw new Exception( pht('Trying to activate lease with no resource.')); } $resource_status = $resource->getStatus(); if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { throw new PhabricatorWorkerYieldException(15); } if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { throw new Exception( pht( 'Trying to activate lease on a dead resource (in status "%s").', $resource_status)); } // NOTE: We can race resource destruction here. Between the time we // performed the read above and now, the resource might have closed, so // we may activate leases on dead resources. At least for now, this seems // fine: a resource dying right before we activate a lease on it should not // be distinguisahble from a resource dying right after we activate a lease // on it. We end up with an active lease on a dead resource either way, and // can not prevent resources dying from lightning strikes. $blueprint = $resource->getBlueprint(); $blueprint->activateLease($resource, $lease); $this->validateActivatedLease($blueprint, $resource, $lease); } /** * @task activate */ private function validateActivatedLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isActivatedLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without activating a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } /* -( Releasing Leases )--------------------------------------------------- */ /** * @task release */ private function releaseLease(DrydockLease $lease) { $lease ->setStatus(DrydockLeaseStatus::STATUS_RELEASED) ->save(); $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST); $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); } $this->destroyLease($lease); } /* -( Breaking Leases )---------------------------------------------------- */ /** * @task break */ protected function breakLease(DrydockLease $lease, Exception $ex) { switch ($lease->getStatus()) { case DrydockLeaseStatus::STATUS_BROKEN: case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: throw new PhutilProxyException( pht( 'Unexpected failure while destroying lease ("%s").', $lease->getPHID()), $ex); } $lease ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) ->save(); + $lease->logEvent( + DrydockLeaseActivationFailureLogType::LOGCONST, + array( + 'class' => get_class($ex), + 'message' => $ex->getMessage(), + )); + + $lease->awakenTasks(); + $this->queueTask( __CLASS__, array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); - $lease->logEvent( - DrydockLeaseActivationFailureLogType::LOGCONST, - array( - 'class' => get_class($ex), - 'message' => $ex->getMessage(), - )); - throw new PhabricatorWorkerPermanentFailureException( pht( 'Permanent failure while activating lease ("%s"): %s', $lease->getPHID(), $ex->getMessage())); } /* -( Destroying Leases )-------------------------------------------------- */ /** * @task destroy */ private function destroyLease(DrydockLease $lease) { $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->destroyLease($resource, $lease); } DrydockSlotLock::releaseLocks($lease->getPHID()); $lease ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) ->save(); $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); + + $lease->awakenTasks(); } } diff --git a/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php b/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php index e0361d3eed..a0dc1e5ebd 100644 --- a/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php @@ -1,179 +1,179 @@ getTaskDataValue('operationPHID'); $hash = PhabricatorHash::digestForIndex($operation_phid); $lock_key = 'drydock.operation:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $operation = $this->loadOperation($operation_phid); $this->handleUpdate($operation); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); } private function handleUpdate(DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); $operation_state = $operation->getOperationState(); switch ($operation_state) { case DrydockRepositoryOperation::STATE_WAIT: $operation ->setOperationState(DrydockRepositoryOperation::STATE_WORK) ->save(); break; case DrydockRepositoryOperation::STATE_WORK: break; case DrydockRepositoryOperation::STATE_DONE: case DrydockRepositoryOperation::STATE_FAIL: // No more processing for these requests. return; } // TODO: We should probably check for other running operations with lower // IDs and the same repository target and yield to them here? That is, // enforce sequential evaluation of operations against the same target so // that if you land "A" and then land "B", we always finish "A" first. // For now, just let stuff happen in any order. We can't lease until // we know we're good to move forward because we might deadlock if we do: // we're waiting for another operation to complete, and that operation is // waiting for a lease we're holding. try { $operation->getImplementation() ->setViewer($viewer); $lease = $this->loadWorkingCopyLease($operation); $interface = $lease->getInterface( DrydockCommandInterface::INTERFACE_TYPE); // No matter what happens here, destroy the lease away once we're done. $lease->releaseOnDestruction(true); $operation->applyOperation($interface); } catch (PhabricatorWorkerYieldException $ex) { throw $ex; } catch (Exception $ex) { $operation ->setOperationState(DrydockRepositoryOperation::STATE_FAIL) ->save(); throw $ex; } $operation ->setOperationState(DrydockRepositoryOperation::STATE_DONE) ->save(); // TODO: Once we have sequencing, we could awaken the next operation // against this target after finishing or failing. } private function loadWorkingCopyLease( DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); // TODO: This is very similar to leasing in Harbormaster, maybe we can // share some of the logic? $lease_phid = $operation->getProperty('exec.leasePHID'); if ($lease_phid) { $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new PhabricatorWorkerPermanentFailureException( pht( 'Lease "%s" could not be loaded.', $lease_phid)); } } else { $working_copy_type = id(new DrydockWorkingCopyBlueprintImplementation()) ->getType(); $repository = $operation->getRepository(); $allowed_phids = $repository->getAutomationBlueprintPHIDs(); $authorizing_phid = $repository->getPHID(); $lease = DrydockLease::initializeNewLease() ->setResourceType($working_copy_type) ->setOwnerPHID($operation->getPHID()) ->setAuthorizingPHID($authorizing_phid) ->setAllowedBlueprintPHIDs($allowed_phids); $map = $this->buildRepositoryMap($operation); $lease->setAttribute('repositories.map', $map); $task_id = $this->getCurrentWorkerTaskID(); if ($task_id) { $lease->setAwakenTaskIDs(array($task_id)); } $operation - ->setProperty('exec.leasePHID', $lease->getPHID()) + ->setWorkingCopyLeasePHID($lease->getPHID()) ->save(); $lease->queueForActivation(); } if ($lease->isActivating()) { throw new PhabricatorWorkerYieldException(15); } if (!$lease->isActive()) { throw new PhabricatorWorkerPermanentFailureException( pht( 'Lease "%s" never activated.', $lease->getPHID())); } return $lease; } private function buildRepositoryMap(DrydockRepositoryOperation $operation) { $repository = $operation->getRepository(); $target = $operation->getRepositoryTarget(); list($type, $name) = explode(':', $target, 2); switch ($type) { case 'branch': $spec = array( 'branch' => $name, ); break; default: throw new Exception( pht( 'Unknown repository operation target type "%s" (in target "%s").', $type, $target)); } $spec['merges'] = $operation->getWorkingCopyMerges(); $map = array(); $map[$repository->getCloneName()] = array( 'phid' => $repository->getPHID(), 'default' => true, ) + $spec; return $map; } }