diff --git a/src/applications/drydock/controller/DrydockLeaseViewController.php b/src/applications/drydock/controller/DrydockLeaseViewController.php index 1d305652fd..42f675a8c0 100644 --- a/src/applications/drydock/controller/DrydockLeaseViewController.php +++ b/src/applications/drydock/controller/DrydockLeaseViewController.php @@ -1,160 +1,169 @@ getViewer(); $id = $request->getURIData('id'); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withIDs(array($id)) ->executeOne(); if (!$lease) { return new Aphront404Response(); } $lease_uri = $this->getApplicationURI('lease/'.$lease->getID().'/'); $title = pht('Lease %d', $lease->getID()); $header = id(new PHUIHeaderView()) ->setHeader($title); $actions = $this->buildActionListView($lease); $properties = $this->buildPropertyListView($lease, $actions); $pager = new PHUIPagerView(); $pager->setURI(new PhutilURI($lease_uri), 'offset'); $pager->setOffset($request->getInt('offset')); $logs = id(new DrydockLogQuery()) ->setViewer($viewer) ->withLeaseIDs(array($lease->getID())) ->executeWithOffsetPager($pager); $log_table = id(new DrydockLogListView()) ->setUser($viewer) ->setLogs($logs) ->render(); $log_table->appendChild($pager); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb($title, $lease_uri); $locks = $this->buildLocksTab($lease->getPHID()); $commands = $this->buildCommandsTab($lease->getPHID()); $object_box = id(new PHUIObjectBoxView()) ->setHeader($header) ->addPropertyList($properties, pht('Properties')) ->addPropertyList($locks, pht('Slot Locks')) ->addPropertyList($commands, pht('Commands')); $log_box = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Lease Logs')) ->setTable($log_table); return $this->buildApplicationPage( array( $crumbs, $object_box, $log_box, ), array( 'title' => $title, )); } private function buildActionListView(DrydockLease $lease) { $viewer = $this->getViewer(); $view = id(new PhabricatorActionListView()) ->setUser($viewer) ->setObjectURI($this->getRequest()->getRequestURI()) ->setObject($lease); $id = $lease->getID(); $can_release = $lease->canRelease(); $can_edit = PhabricatorPolicyFilter::hasCapability( $viewer, $lease, PhabricatorPolicyCapability::CAN_EDIT); $view->addAction( id(new PhabricatorActionView()) ->setName(pht('Release Lease')) ->setIcon('fa-times') ->setHref($this->getApplicationURI("/lease/{$id}/release/")) ->setWorkflow(true) ->setDisabled(!$can_release || !$can_edit)); return $view; } private function buildPropertyListView( DrydockLease $lease, PhabricatorActionListView $actions) { + $viewer = $this->getViewer(); $view = new PHUIPropertyListView(); $view->setActionList($actions); switch ($lease->getStatus()) { case DrydockLeaseStatus::STATUS_ACTIVE: $status = pht('Active'); break; case DrydockLeaseStatus::STATUS_RELEASED: $status = pht('Released'); break; case DrydockLeaseStatus::STATUS_EXPIRED: $status = pht('Expired'); break; case DrydockLeaseStatus::STATUS_PENDING: $status = pht('Pending'); break; case DrydockLeaseStatus::STATUS_BROKEN: $status = pht('Broken'); break; default: $status = pht('Unknown'); break; } $view->addProperty( pht('Status'), $status); $view->addProperty( pht('Resource Type'), $lease->getResourceType()); $resource = id(new DrydockResourceQuery()) ->setViewer($this->getViewer()) ->withIDs(array($lease->getResourceID())) ->executeOne(); if ($resource !== null) { $view->addProperty( pht('Resource'), $this->getViewer()->renderHandle($resource->getPHID())); } else { $view->addProperty( pht('Resource'), pht('No Resource')); } + $until = $lease->getUntil(); + if ($until) { + $until_display = phabricator_datetime($until, $viewer); + } else { + $until_display = phutil_tag('em', array(), pht('Never')); + } + $view->addProperty(pht('Expires'), $until_display); + $attributes = $lease->getAttributes(); if ($attributes) { $view->addSectionHeader( pht('Attributes'), 'fa-list-ul'); foreach ($attributes as $key => $value) { $view->addProperty($key, $value); } } return $view; } } diff --git a/src/applications/drydock/management/DrydockManagementLeaseWorkflow.php b/src/applications/drydock/management/DrydockManagementLeaseWorkflow.php index 117ab2b6a2..4c1e0d875c 100644 --- a/src/applications/drydock/management/DrydockManagementLeaseWorkflow.php +++ b/src/applications/drydock/management/DrydockManagementLeaseWorkflow.php @@ -1,63 +1,85 @@ setName('lease') ->setSynopsis(pht('Lease a resource.')) ->setArguments( array( array( 'name' => 'type', 'param' => 'resource_type', 'help' => pht('Resource type.'), ), + array( + 'name' => 'until', + 'param' => 'time', + 'help' => pht('Set lease expiration time.'), + ), array( 'name' => 'attributes', 'param' => 'name=value,...', 'help' => pht('Resource specficiation.'), ), )); } public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $resource_type = $args->getArg('type'); if (!$resource_type) { throw new PhutilArgumentUsageException( pht( 'Specify a resource type with `%s`.', '--type')); } + $until = $args->getArg('until'); + if (strlen($until)) { + $until = strtotime($until); + if ($until <= 0) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to parse argument to "%s".', + '--until')); + } + } + $attributes = $args->getArg('attributes'); if ($attributes) { $options = new PhutilSimpleOptions(); $options->setCaseSensitive(true); $attributes = $options->parse($attributes); } $lease = id(new DrydockLease()) ->setResourceType($resource_type); + if ($attributes) { $lease->setAttributes($attributes); } + + if ($until) { + $lease->setUntil($until); + } + $lease->queueForActivation(); echo tsprintf( "%s\n", pht('Waiting for daemons to activate lease...')); $lease->waitUntilActive(); echo tsprintf( "%s\n", pht('Activated lease "%s".', $lease->getID())); return 0; } } diff --git a/src/applications/drydock/storage/DrydockLease.php b/src/applications/drydock/storage/DrydockLease.php index 871b5a8e93..a2a6abdfa2 100644 --- a/src/applications/drydock/storage/DrydockLease.php +++ b/src/applications/drydock/storage/DrydockLease.php @@ -1,359 +1,366 @@ releaseOnDestruction = true; return $this; } public function __destruct() { if (!$this->releaseOnDestruction) { return; } if (!$this->canRelease()) { return; } $actor = PhabricatorUser::getOmnipotentUser(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $command = DrydockCommand::initializeNewCommand($actor) ->setTargetPHID($this->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); $this->scheduleUpdate(); } public function getLeaseName() { return pht('Lease %d', $this->getID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'attributes' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'status' => 'uint32', 'until' => 'epoch?', 'resourceType' => 'text128', 'ownerPHID' => 'phid?', 'resourceID' => 'id?', ), self::CONFIG_KEY_SCHEMA => array( 'key_phid' => null, 'phid' => array( 'columns' => array('phid'), 'unique' => true, ), ), ) + parent::getConfiguration(); } public function setAttribute($key, $value) { $this->attributes[$key] = $value; return $this; } public function getAttribute($key, $default = null) { return idx($this->attributes, $key, $default); } public function generatePHID() { return PhabricatorPHID::generateNewPHID(DrydockLeasePHIDType::TYPECONST); } public function getInterface($type) { return $this->getResource()->getInterface($this, $type); } public function getResource() { return $this->assertAttached($this->resource); } public function attachResource(DrydockResource $resource = null) { $this->resource = $resource; return $this; } public function hasAttachedResource() { return ($this->resource !== null); } public function queueForActivation() { if ($this->getID()) { throw new Exception( pht('Only new leases may be queued for activation!')); } $this ->setStatus(DrydockLeaseStatus::STATUS_PENDING) ->save(); $task = PhabricatorWorker::scheduleTask( 'DrydockAllocatorWorker', array( 'leasePHID' => $this->getPHID(), ), array( 'objectPHID' => $this->getPHID(), )); return $this; } public function isActive() { switch ($this->status) { case DrydockLeaseStatus::STATUS_ACQUIRED: case DrydockLeaseStatus::STATUS_ACTIVE: return true; } return false; } private function assertActive() { if (!$this->isActive()) { throw new Exception( pht( 'Lease is not active! You can not interact with resources through '. 'an inactive lease.')); } } public function waitUntilActive() { while (true) { $lease = $this->reload(); if (!$lease) { throw new Exception(pht('Failed to reload lease.')); } $status = $lease->getStatus(); switch ($status) { case DrydockLeaseStatus::STATUS_ACTIVE: return; case DrydockLeaseStatus::STATUS_RELEASED: throw new Exception(pht('Lease has already been released!')); case DrydockLeaseStatus::STATUS_EXPIRED: throw new Exception(pht('Lease has already expired!')); case DrydockLeaseStatus::STATUS_BROKEN: throw new Exception(pht('Lease has been broken!')); case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: break; default: throw new Exception( pht( 'Lease has unknown status "%s".', $status)); } sleep(1); } } public function setActivateWhenAcquired($activate) { $this->activateWhenAcquired = true; return $this; } public function needSlotLock($key) { $this->slotLocks[] = $key; return $this; } public function acquireOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to acquire a lease on a resource which is in the wrong '. 'state: status must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($this->activateWhenAcquired) { $new_status = DrydockLeaseStatus::STATUS_ACTIVE; } else { $new_status = DrydockLeaseStatus::STATUS_ACQUIRED; } if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { throw new Exception( pht( 'Trying to acquire an active lease on a pending resource. '. 'You can not immediately activate leases on resources which '. 'need time to start up.')); } } $this->openTransaction(); $this ->setResourceID($resource->getID()) ->setStatus($new_status) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isAcquired = true; if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { $this->didActivate(); } return $this; } public function isAcquiredLease() { return $this->isAcquired; } public function activateOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_ACQUIRED; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to activate a lease which has the wrong status: status '. 'must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { // TODO: Be stricter about this? throw new Exception( pht( 'Trying to activate a lease on a pending resource.')); } $this->openTransaction(); $this ->setStatus(DrydockLeaseStatus::STATUS_ACTIVE) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isActivated = true; $this->didActivate(); return $this; } public function isActivatedLease() { return $this->isActivated; } public function canRelease() { if (!$this->getID()) { return false; } switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_RELEASED: return false; default: return true; } } - public function scheduleUpdate() { + public function scheduleUpdate($epoch = null) { PhabricatorWorker::scheduleTask( 'DrydockLeaseUpdateWorker', array( 'leasePHID' => $this->getPHID(), + 'isExpireTask' => ($epoch !== null), ), array( 'objectPHID' => $this->getPHID(), + 'delayUntil' => $epoch, )); } private function didActivate() { $viewer = PhabricatorUser::getOmnipotentUser(); $need_update = false; $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($this->getPHID())) ->withConsumed(false) ->execute(); if ($commands) { $need_update = true; } if ($need_update) { $this->scheduleUpdate(); } + + $expires = $this->getUntil(); + if ($expires) { + $this->scheduleUpdate($expires); + } } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { if ($this->getResource()) { return $this->getResource()->getPolicy($capability); } // TODO: Implement reasonable policies. return PhabricatorPolicies::getMostOpenPolicy(); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { if ($this->getResource()) { return $this->getResource()->hasAutomaticCapability($capability, $viewer); } return false; } public function describeAutomaticCapability($capability) { return pht('Leases inherit policies from the resources they lease.'); } } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 3206d6ea13..98ecf2b498 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,71 +1,107 @@ getTaskDataValue('leasePHID'); $hash = PhabricatorHash::digestForIndex($lease_phid); $lock_key = 'drydock.lease:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); - $lease = $this->loadLease($lease_phid); - $this->updateLease($lease); + try { + $lease = $this->loadLease($lease_phid); + $this->updateLease($lease); + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } $lock->unlock(); } private function updateLease(DrydockLease $lease) { + if ($lease->getStatus() != DrydockLeaseStatus::STATUS_ACTIVE) { + return; + } + + $viewer = $this->getViewer(); + $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); + + // Check if the lease has expired. If it is, we're going to send it a + // release command. This command will be handled immediately below, it + // just generates a command log and improves consistency. + $now = PhabricatorTime::getNow(); + $expires = $lease->getUntil(); + if ($expires && ($expires <= $now)) { + $command = DrydockCommand::initializeNewCommand($viewer) + ->setTargetPHID($lease->getPHID()) + ->setAuthorPHID($drydock_phid) + ->setCommand(DrydockCommand::COMMAND_RELEASE) + ->save(); + } + $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { if ($lease->getStatus() != DrydockLeaseStatus::STATUS_ACTIVE) { // Leases can't receive commands before they activate or after they // release. break; } $this->processCommand($lease, $command); + $command ->setIsConsumed(true) ->save(); } + + // If this is the task which will eventually release the lease after it + // expires but it is still active, reschedule the task to run after the + // lease expires. This can happen if the lease's expiration was pushed + // forward. + if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACTIVE) { + if ($this->getTaskDataValue('isExpireTask') && $expires) { + throw new PhabricatorWorkerYieldException($expires - $now); + } + } } private function processCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseLease($lease); break; } } private function releaseLease(DrydockLease $lease) { $lease->openTransaction(); $lease ->setStatus(DrydockLeaseStatus::STATUS_RELEASED) ->save(); // TODO: Hold slot locks until destruction? DrydockSlotLock::releaseLocks($lease->getPHID()); $lease->saveTransaction(); PhabricatorWorker::scheduleTask( 'DrydockLeaseDestroyWorker', array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); $resource = $lease->getResource(); $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); } } diff --git a/src/infrastructure/daemon/workers/PhabricatorWorker.php b/src/infrastructure/daemon/workers/PhabricatorWorker.php index 83dc40b681..4e71440604 100644 --- a/src/infrastructure/daemon/workers/PhabricatorWorker.php +++ b/src/infrastructure/daemon/workers/PhabricatorWorker.php @@ -1,206 +1,211 @@ data = $data; } final protected function getTaskData() { return $this->data; } final protected function getTaskDataValue($key, $default = null) { $data = $this->getTaskData(); if (!is_array($data)) { throw new PhabricatorWorkerPermanentFailureException( pht('Expected task data to be a dictionary.')); } return idx($data, $key, $default); } final public function executeTask() { $this->doWork(); } final public static function scheduleTask( $task_class, $data, $options = array()) { $priority = idx($options, 'priority'); if ($priority === null) { $priority = self::PRIORITY_DEFAULT; } $object_phid = idx($options, 'objectPHID'); $task = id(new PhabricatorWorkerActiveTask()) ->setTaskClass($task_class) ->setData($data) ->setPriority($priority) ->setObjectPHID($object_phid); + $delay = idx($options, 'delayUntil'); + if ($delay) { + $task->setLeaseExpires($delay); + } + if (self::$runAllTasksInProcess) { // Do the work in-process. $worker = newv($task_class, array($data)); while (true) { try { $worker->doWork(); foreach ($worker->getQueuedTasks() as $queued_task) { list($queued_class, $queued_data, $queued_priority) = $queued_task; $queued_options = array('priority' => $queued_priority); self::scheduleTask($queued_class, $queued_data, $queued_options); } break; } catch (PhabricatorWorkerYieldException $ex) { phlog( pht( 'In-process task "%s" yielded for %s seconds, sleeping...', $task_class, $ex->getDuration())); sleep($ex->getDuration()); } } // Now, save a task row and immediately archive it so we can return an // object with a valid ID. $task->openTransaction(); $task->save(); $archived = $task->archiveTask( PhabricatorWorkerArchiveTask::RESULT_SUCCESS, 0); $task->saveTransaction(); return $archived; } else { $task->save(); return $task; } } public function renderForDisplay(PhabricatorUser $viewer) { return null; } /** * Set this flag to execute scheduled tasks synchronously, in the same * process. This is useful for debugging, and otherwise dramatically worse * in every way imaginable. */ public static function setRunAllTasksInProcess($all) { self::$runAllTasksInProcess = $all; } final protected function log($pattern /* , ... */) { $console = PhutilConsole::getConsole(); $argv = func_get_args(); call_user_func_array(array($console, 'writeLog'), $argv); return $this; } /** * Queue a task to be executed after this one succeeds. * * The followup task will be queued only if this task completes cleanly. * * @param string Task class to queue. * @param array Data for the followup task. * @param int|null Priority for the followup task. * @return this */ final protected function queueTask($class, array $data, $priority = null) { $this->queuedTasks[] = array($class, $data, $priority); return $this; } /** * Get tasks queued as followups by @{method:queueTask}. * * @return list> Queued task specifications. */ final public function getQueuedTasks() { return $this->queuedTasks; } }