diff --git a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php index e62c7b7558..19e476c4ba 100644 --- a/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php @@ -1,299 +1,288 @@ loadServices($blueprint); $bindings = $this->loadAllBindings($services); if (!$bindings) { // If there are no devices bound to the services for this blueprint, // we can not allocate resources. return false; } return true; } public function canAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease) { // We will only allocate one resource per unique device bound to the // services for this blueprint. Make sure we have a free device somewhere. $free_bindings = $this->loadFreeBindings($blueprint); if (!$free_bindings) { return false; } return true; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $free_bindings = $this->loadFreeBindings($blueprint); shuffle($free_bindings); $exceptions = array(); foreach ($free_bindings as $binding) { $device = $binding->getDevice(); $device_name = $device->getName(); $binding_phid = $binding->getPHID(); $resource = $this->newResourceTemplate($blueprint) ->setActivateWhenAllocated(true) ->setAttribute('almanacDeviceName', $device_name) ->setAttribute('almanacServicePHID', $binding->getServicePHID()) ->setAttribute('almanacBindingPHID', $binding_phid) ->needSlotLock("almanac.host.binding({$binding_phid})"); try { return $resource->allocateResource(); } catch (Exception $ex) { $exceptions[] = $ex; } } throw new PhutilAggregateException( pht('Unable to allocate any binding as a resource.'), $exceptions); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { // We don't create anything when allocating hosts, so we don't need to do // any cleanup here. return; } public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource) { $device_name = $resource->getAttribute( 'almanacDeviceName', pht('')); return pht('Host (%s)', $device_name); } public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { - - if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { - return false; - } - return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->setActivateWhenAcquired(true) - ->needSlotLock($this->getLeaseSlotLock($resource)) ->acquireOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // Almanac hosts stick around indefinitely so we don't need to recycle them // if they don't have any leases. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We don't create anything when activating a lease, so we don't need to // throw anything away. return; } - private function getLeaseSlotLock(DrydockResource $resource) { - $resource_phid = $resource->getPHID(); - return "almanac.host.lease({$resource_phid})"; - } - public function getType() { return 'host'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { $viewer = PhabricatorUser::getOmnipotentUser(); switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $credential_phid = $blueprint->getFieldValue('credentialPHID'); $binding_phid = $resource->getAttribute('almanacBindingPHID'); $binding = id(new AlmanacBindingQuery()) ->setViewer($viewer) ->withPHIDs(array($binding_phid)) ->executeOne(); if (!$binding) { throw new Exception( pht( 'Unable to load binding "%s" to create command interface.', $binding_phid)); } $interface = $binding->getInterface(); return id(new DrydockSSHCommandInterface()) ->setConfig('credentialPHID', $credential_phid) ->setConfig('host', $interface->getAddress()) ->setConfig('port', $interface->getPort()); } } - public function getFieldSpecifications() { + protected function getCustomFieldSpecifications() { return array( 'almanacServicePHIDs' => array( 'name' => pht('Almanac Services'), 'type' => 'datasource', 'datasource.class' => 'AlmanacServiceDatasource', 'datasource.parameters' => array( 'serviceClasses' => $this->getAlmanacServiceClasses(), ), 'required' => true, ), 'credentialPHID' => array( 'name' => pht('Credentials'), 'type' => 'credential', 'credential.provides' => PassphraseSSHPrivateKeyCredentialType::PROVIDES_TYPE, 'credential.type' => PassphraseSSHPrivateKeyTextCredentialType::CREDENTIAL_TYPE, ), - ) + parent::getFieldSpecifications(); + ); } private function loadServices(DrydockBlueprint $blueprint) { if (!$this->services) { $service_phids = $blueprint->getFieldValue('almanacServicePHIDs'); if (!$service_phids) { throw new Exception( pht( 'This blueprint ("%s") does not define any Almanac Service PHIDs.', $blueprint->getBlueprintName())); } $viewer = PhabricatorUser::getOmnipotentUser(); $services = id(new AlmanacServiceQuery()) ->setViewer($viewer) ->withPHIDs($service_phids) ->withServiceClasses($this->getAlmanacServiceClasses()) ->needBindings(true) ->execute(); $services = mpull($services, null, 'getPHID'); if (count($services) != count($service_phids)) { $missing_phids = array_diff($service_phids, array_keys($services)); throw new Exception( pht( 'Some of the Almanac Services defined by this blueprint '. 'could not be loaded. They may be invalid, no longer exist, '. 'or be of the wrong type: %s.', implode(', ', $missing_phids))); } $this->services = $services; } return $this->services; } private function loadAllBindings(array $services) { assert_instances_of($services, 'AlmanacService'); $bindings = array_mergev(mpull($services, 'getBindings')); return mpull($bindings, null, 'getPHID'); } private function loadFreeBindings(DrydockBlueprint $blueprint) { if ($this->freeBindings === null) { $viewer = PhabricatorUser::getOmnipotentUser(); $pool = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(array($blueprint->getPHID())) ->withStatuses( array( DrydockResourceStatus::STATUS_PENDING, DrydockResourceStatus::STATUS_ACTIVE, DrydockResourceStatus::STATUS_BROKEN, DrydockResourceStatus::STATUS_RELEASED, )) ->execute(); $allocated_phids = array(); foreach ($pool as $resource) { $allocated_phids[] = $resource->getAttribute('almanacBindingPHID'); } $allocated_phids = array_fuse($allocated_phids); $services = $this->loadServices($blueprint); $bindings = $this->loadAllBindings($services); $free = array(); foreach ($bindings as $binding) { if (empty($allocated_phids[$binding->getPHID()])) { $free[] = $binding; } } $this->freeBindings = $free; } return $this->freeBindings; } private function getAlmanacServiceClasses() { return array( 'AlmanacDrydockPoolServiceType', ); } } diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php index 0c53b19fcf..01c2067280 100644 --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -1,384 +1,485 @@ getCustomFieldSpecifications(); + + if ($this->shouldUseConcurrentResourceLimit()) { + $fields += array( + 'allocator.limit' => array( + 'name' => pht('Limit'), + 'caption' => pht( + 'Maximum number of resources this blueprint can have active '. + 'concurrently.'), + 'type' => 'int', + ), + ); + } + + return $fields; + } + + protected function getCustomFieldSpecifications() { return array(); } public function getViewer() { return PhabricatorUser::getOmnipotentUser(); } /* -( Lease Acquisition )-------------------------------------------------- */ /** * Enforce basic checks on lease/resource compatibility. Allows resources to * reject leases if they are incompatible, even if the resource types match. * * For example, if a resource represents a 32-bit host, this method might * reject leases that need a 64-bit host. The blueprint might also reject * a resource if the lease needs 8GB of RAM and the resource only has 6GB * free. * * This method should not acquire locks or expect anything to be locked. This * is a coarse compatibility check between a lease and a resource. * * @param DrydockBlueprint Concrete blueprint to allocate for. * @param DrydockResource Candidiate resource to allocate the lease on. * @param DrydockLease Pending lease that wants to allocate here. * @return bool True if the resource and lease are compatible. * @task lease */ abstract public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * Acquire a lease. Allows resources to peform setup as leases are brought * online. * * If acquisition fails, throw an exception. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Requested lease. * @return void * @task lease */ abstract public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * @return void * @task lease */ public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { throw new PhutilMethodNotImplementedException(); } /** * React to a lease being released. * * This callback is primarily useful for automatically releasing resources * once all leases are released. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource a lease was released on. * @param DrydockLease Recently released lease. * @return void * @task lease */ abstract public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * Destroy any temporary data associated with a lease. * * If a lease creates temporary state while held, destroy it here. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource the lease is acquired on. * @param DrydockLease The lease being destroyed. * @return void * @task lease */ abstract public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /* -( Resource Allocation )------------------------------------------------ */ /** * Enforce fundamental implementation/lease checks. Allows implementations to * reject a lease which no concrete blueprint can ever satisfy. * * For example, if a lease only builds ARM hosts and the lease needs a * PowerPC host, it may be rejected here. * * This is the earliest rejection phase, and followed by * @{method:canEverAllocateResourceForLease}. * * This method should not actually check if a resource can be allocated * right now, or even if a blueprint which can allocate a suitable resource * really exists, only if some blueprint may conceivably exist which could * plausibly be able to build a suitable resource. * * @param DrydockLease Requested lease. * @return bool True if some concrete blueprint of this implementation's * type might ever be able to build a resource for the lease. * @task resource */ abstract public function canAnyBlueprintEverAllocateResourceForLease( DrydockLease $lease); /** * Enforce basic blueprint/lease checks. Allows blueprints to reject a lease * which they can not build a resource for. * * This is the second rejection phase. It follows * @{method:canAnyBlueprintEverAllocateResourceForLease} and is followed by * @{method:canAllocateResourceForLease}. * * This method should not check if a resource can be built right now, only * if the blueprint as configured may, at some time, be able to build a * suitable resource. * * @param DrydockBlueprint Blueprint which may be asked to allocate a * resource. * @param DrydockLease Requested lease. * @return bool True if this blueprint can eventually build a suitable * resource for the lease, as currently configured. * @task resource */ abstract public function canEverAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease); /** * Enforce basic availability limits. Allows blueprints to reject resource * allocation if they are currently overallocated. * * This method should perform basic capacity/limit checks. For example, if * it has a limit of 6 resources and currently has 6 resources allocated, * it might reject new leases. * * This method should not acquire locks or expect locks to be acquired. This * is a coarse check to determine if the operation is likely to succeed * right now without needing to acquire locks. * * It is expected that this method will sometimes return `true` (indicating * that a resource can be allocated) but find that another allocator has * eaten up free capacity by the time it actually tries to build a resource. * This is normal and the allocator will recover from it. * * @param DrydockBlueprint The blueprint which may be asked to allocate a * resource. * @param DrydockLease Requested lease. * @return bool True if this blueprint appears likely to be able to allocate * a suitable resource. * @task resource */ abstract public function canAllocateResourceForLease( DrydockBlueprint $blueprint, DrydockLease $lease); /** * Allocate a suitable resource for a lease. * * This method MUST acquire, hold, and manage locks to prevent multiple * allocations from racing. World state is not locked before this method is * called. Blueprints are entirely responsible for any lock handling they * need to perform. * * @param DrydockBlueprint The blueprint which should allocate a resource. * @param DrydockLease Requested lease. * @return DrydockResource Allocated resource. * @task resource */ abstract public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease); /** * @task resource */ public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { throw new PhutilMethodNotImplementedException(); } /** * Destroy any temporary data associated with a resource. * * If a resource creates temporary state when allocated, destroy that state * here. For example, you might shut down a virtual host or destroy a working * copy on disk. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource being destroyed. * @return void * @task resource */ abstract public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource); /** * Get a human readable name for a resource. * * @param DrydockBlueprint Blueprint which built the resource. * @param DrydockResource Resource to get the name of. * @return string Human-readable resource name. * @task resource */ abstract public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource); /* -( Resource Interfaces )------------------------------------------------ */ abstract public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type); /* -( Logging )------------------------------------------------------------ */ public static function getAllBlueprintImplementations() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) ->execute(); } public static function getNamedImplementation($class) { return idx(self::getAllBlueprintImplementations(), $class); } protected function newResourceTemplate(DrydockBlueprint $blueprint) { $resource = id(new DrydockResource()) ->setBlueprintPHID($blueprint->getPHID()) ->attachBlueprint($blueprint) ->setType($this->getType()) ->setStatus(DrydockResourceStatus::STATUS_PENDING); // Pre-allocate the resource PHID. $resource->setPHID($resource->generatePHID()); return $resource; } protected function newLease(DrydockBlueprint $blueprint) { return DrydockLease::initializeNewLease() ->setAuthorizingPHID($blueprint->getPHID()); } protected function requireActiveLease(DrydockLease $lease) { $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: throw new PhabricatorWorkerYieldException(15); case DrydockLeaseStatus::STATUS_ACTIVE: return; default: throw new Exception( pht( 'Lease ("%s") is in bad state ("%s"), expected "%s".', $lease->getPHID(), $lease_status, DrydockLeaseStatus::STATUS_ACTIVE)); } } + /** + * Does this implementation use concurrent resource limits? + * + * Implementations can override this method to opt into standard limit + * behavior, which provides a simple concurrent resource limit. + * + * @return bool True to use limits. + */ + protected function shouldUseConcurrentResourceLimit() { + return false; + } + + + /** + * Get the effective concurrent resource limit for this blueprint. + * + * @param DrydockBlueprint Blueprint to get the limit for. + * @return int|null Limit, or `null` for no limit. + */ + protected function getConcurrentResourceLimit(DrydockBlueprint $blueprint) { + if ($this->shouldUseConcurrentResourceLimit()) { + $limit = $blueprint->getFieldValue('allocator.limit'); + $limit = (int)$limit; + if ($limit > 0) { + return $limit; + } else { + return null; + } + } + + return null; + } + + + protected function getConcurrentResourceLimitSlotLock( + DrydockBlueprint $blueprint) { + + $limit = $this->getConcurrentResourceLimit($blueprint); + if ($limit === null) { + return; + } + + $blueprint_phid = $blueprint->getPHID(); + + // TODO: This logic shouldn't do anything awful, but is a little silly. It + // would be nice to unify the "huge limit" and "small limit" cases + // eventually but it's a little tricky. + + // If the limit is huge, just pick a random slot. This is just stopping + // us from exploding if someone types a billion zillion into the box. + if ($limit > 1024) { + $slot = mt_rand(0, $limit - 1); + return "allocator({$blueprint_phid}).limit({$slot})"; + } + + // For reasonable limits, actually check for an available slot. + $locks = DrydockSlotLock::loadLocks($blueprint_phid); + $locks = mpull($locks, null, 'getLockKey'); + + $slots = range(0, $limit - 1); + shuffle($slots); + + foreach ($slots as $slot) { + $slot_lock = "allocator({$blueprint_phid}).limit({$slot})"; + if (empty($locks[$slot_lock])) { + return $slot_lock; + } + } + + // If we found no free slot, just return whatever we checked last (which + // is just a random slot). There's a small chance we'll get lucky and the + // lock will be free by the time we try to take it, but usually we'll just + // fail to grab the lock, throw an appropriate lock exception, and get back + // on the right path to retry later. + return $slot_lock; + } + + + /** * Apply standard limits on resource allocation rate. * * @param DrydockBlueprint The blueprint requesting an allocation. * @return bool True if further allocations should be limited. */ protected function shouldLimitAllocatingPoolSize( DrydockBlueprint $blueprint) { // TODO: If this mechanism sticks around, these values should be // configurable by the blueprint implementation. // Limit on total number of active resources. - $total_limit = 1; + $total_limit = $this->getConcurrentResourceLimit($blueprint); // Always allow at least this many allocations to be in flight at once. $min_allowed = 1; // Allow this fraction of allocating resources as a fraction of active // resources. $growth_factor = 0.25; $resource = new DrydockResource(); $conn_r = $resource->establishConnection('r'); $counts = queryfx_all( $conn_r, 'SELECT status, COUNT(*) N FROM %T WHERE blueprintPHID = %s AND status != %s GROUP BY status', $resource->getTableName(), $blueprint->getPHID(), DrydockResourceStatus::STATUS_DESTROYED); $counts = ipull($counts, 'N', 'status'); $n_alloc = idx($counts, DrydockResourceStatus::STATUS_PENDING, 0); $n_active = idx($counts, DrydockResourceStatus::STATUS_ACTIVE, 0); $n_broken = idx($counts, DrydockResourceStatus::STATUS_BROKEN, 0); $n_released = idx($counts, DrydockResourceStatus::STATUS_RELEASED, 0); // If we're at the limit on total active resources, limit additional // allocations. - $n_total = ($n_alloc + $n_active + $n_broken + $n_released); - if ($n_total >= $total_limit) { - return true; + if ($total_limit !== null) { + $n_total = ($n_alloc + $n_active + $n_broken + $n_released); + if ($n_total >= $total_limit) { + return true; + } } // If the number of in-flight allocations is fewer than the minimum number // of allowed allocations, don't impose a limit. if ($n_alloc < $min_allowed) { return false; } $allowed_alloc = (int)ceil($n_active * $growth_factor); // If the number of in-flight allocation is fewer than the number of // allowed allocations according to the pool growth factor, don't impose // a limit. if ($n_alloc < $allowed_alloc) { return false; } return true; } } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php index 1d7776af14..f4b33adb8b 100644 --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -1,407 +1,411 @@ getViewer(); if ($this->shouldLimitAllocatingPoolSize($blueprint)) { return false; } // TODO: If we have a pending resource which is compatible with the // configuration for this lease, prevent a new allocation? Otherwise the // queue can fill up with copies of requests from the same lease. But // maybe we can deal with this with "pre-leasing"? return true; } public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // Don't hand out leases on working copies which have not activated, since // it may take an arbitrarily long time for them to acquire a host. if (!$resource->isActive()) { return false; } $need_map = $lease->getAttribute('repositories.map'); if (!is_array($need_map)) { return false; } $have_map = $resource->getAttribute('repositories.map'); if (!is_array($have_map)) { return false; } $have_as = ipull($have_map, 'phid'); $need_as = ipull($need_map, 'phid'); foreach ($need_as as $need_directory => $need_phid) { if (empty($have_as[$need_directory])) { // This resource is missing a required working copy. return false; } if ($have_as[$need_directory] != $need_phid) { // This resource has a required working copy, but it contains // the wrong repository. return false; } unset($have_as[$need_directory]); } if ($have_as && $lease->getAttribute('repositories.strict')) { // This resource has extra repositories, but the lease is strict about // which repositories are allowed to exist. return false; } if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { return false; } return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->needSlotLock($this->getLeaseSlotLock($resource)) ->acquireOnResource($resource); } private function getLeaseSlotLock(DrydockResource $resource) { $resource_phid = $resource->getPHID(); return "workingcopy.lease({$resource_phid})"; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $this->newResourceTemplate($blueprint); $resource_phid = $resource->getPHID(); $blueprint_phids = $blueprint->getFieldValue('blueprintPHIDs'); $host_lease = $this->newLease($blueprint) ->setResourceType('host') ->setOwnerPHID($resource_phid) ->setAttribute('workingcopy.resourcePHID', $resource_phid) ->setAllowedBlueprintPHIDs($blueprint_phids); - - $resource - ->setAttribute('host.leasePHID', $host_lease->getPHID()) - ->save(); - - $host_lease->queueForActivation(); - - // TODO: Add some limits to the number of working copies we can have at - // once? + $resource->setAttribute('host.leasePHID', $host_lease->getPHID()); $map = $lease->getAttribute('repositories.map'); foreach ($map as $key => $value) { $map[$key] = array_select_keys( $value, array( 'phid', )); } + $resource->setAttribute('repositories.map', $map); + + $slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint); + if ($slot_lock !== null) { + $resource->needSlotLock($slot_lock); + } - return $resource - ->setAttribute('repositories.map', $map) - ->allocateResource(); + $resource->allocateResource(); + + $host_lease->queueForActivation(); + + return $resource; } public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { $lease = $this->loadHostLease($resource); $this->requireActiveLease($lease); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); // TODO: Make this configurable. $resource_id = $resource->getID(); $root = "/var/drydock/workingcopy-{$resource_id}"; $map = $resource->getAttribute('repositories.map'); $repositories = $this->loadRepositories(ipull($map, 'phid')); foreach ($map as $directory => $spec) { // TODO: Validate directory isn't goofy like "/etc" or "../../lol" // somewhere? $repository = $repositories[$spec['phid']]; $path = "{$root}/repo/{$directory}/"; // TODO: Run these in parallel? $interface->execx( 'git clone -- %s %s', (string)$repository->getCloneURIObject(), $path); } $resource ->setAttribute('workingcopy.root', $root) ->activateResource(); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { try { $lease = $this->loadHostLease($resource); } catch (Exception $ex) { // If we can't load the lease, assume we don't need to take any actions // to destroy it. return; } // Destroy the lease on the host. $lease->releaseOnDestruction(); if ($lease->isActive()) { // Destroy the working copy on disk. $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); $root_key = 'workingcopy.root'; $root = $resource->getAttribute($root_key); if (strlen($root)) { $interface->execx('rm -rf -- %s', $root); } } } public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource) { return pht('Working Copy'); } public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $host_lease = $this->loadHostLease($resource); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $host_lease->getInterface($command_type); $map = $lease->getAttribute('repositories.map'); $root = $resource->getAttribute('workingcopy.root'); $default = null; foreach ($map as $directory => $spec) { $cmd = array(); $arg = array(); $cmd[] = 'cd %s'; $arg[] = "{$root}/repo/{$directory}/"; $cmd[] = 'git clean -d --force'; $cmd[] = 'git fetch'; $commit = idx($spec, 'commit'); $branch = idx($spec, 'branch'); $ref = idx($spec, 'ref'); if ($commit !== null) { $cmd[] = 'git reset --hard %s'; $arg[] = $commit; } else if ($branch !== null) { $cmd[] = 'git checkout %s'; $arg[] = $branch; $cmd[] = 'git reset --hard origin/%s'; $arg[] = $branch; } else if ($ref) { $ref_uri = $ref['uri']; $ref_ref = $ref['ref']; $cmd[] = 'git fetch --no-tags -- %s +%s:%s'; $arg[] = $ref_uri; $arg[] = $ref_ref; $arg[] = $ref_ref; $cmd[] = 'git checkout %s'; $arg[] = $ref_ref; $cmd[] = 'git reset --hard %s'; $arg[] = $ref_ref; } else { $cmd[] = 'git reset --hard HEAD'; } $cmd = implode(' && ', $cmd); $argv = array_merge(array($cmd), $arg); $result = call_user_func_array( array($interface, 'execx'), $argv); if (idx($spec, 'default')) { $default = $directory; } } if ($default === null) { $default = head_key($map); } // TODO: Use working storage? $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/"); $lease->activateOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We leave working copies around even if there are no leases on them, // since the cost to maintain them is nearly zero but rebuilding them is // moderately expensive and it's likely that they'll be reused. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // When we activate a lease we just reset the working copy state and do // not create any new state, so we don't need to do anything special when // destroying a lease. return; } public function getType() { return 'working-copy'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $host_lease = $this->loadHostLease($resource); $command_interface = $host_lease->getInterface($type); $path = $lease->getAttribute('workingcopy.default'); $command_interface->setWorkingDirectory($path); return $command_interface; } } private function loadRepositories(array $phids) { $viewer = $this->getViewer(); $repositories = id(new PhabricatorRepositoryQuery()) ->setViewer($viewer) ->withPHIDs($phids) ->execute(); $repositories = mpull($repositories, null, 'getPHID'); foreach ($phids as $phid) { if (empty($repositories[$phid])) { throw new Exception( pht( 'Repository PHID "%s" does not exist.', $phid)); } } foreach ($repositories as $repository) { $repository_vcs = $repository->getVersionControlSystem(); switch ($repository_vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: break; default: throw new Exception( pht( 'Repository ("%s") has unsupported VCS ("%s").', $repository->getPHID(), $repository_vcs)); } } return $repositories; } private function loadHostLease(DrydockResource $resource) { $viewer = $this->getViewer(); $lease_phid = $resource->getAttribute('host.leasePHID'); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new Exception( pht( 'Unable to load lease ("%s").', $lease_phid)); } return $lease; } - public function getFieldSpecifications() { + protected function getCustomFieldSpecifications() { return array( 'blueprintPHIDs' => array( 'name' => pht('Use Blueprints'), 'type' => 'blueprints', 'required' => true, ), - ) + parent::getFieldSpecifications(); + ); + } + + protected function shouldUseConcurrentResourceLimit() { + return true; } } diff --git a/src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php index bc6a52018c..7110d98d6e 100644 --- a/src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php +++ b/src/applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php @@ -1,99 +1,108 @@ setName('build') ->setExamples('**build** [__options__] __buildable__ --plan __id__') ->setSynopsis(pht('Run plan __id__ on __buildable__.')) ->setArguments( array( array( 'name' => 'plan', 'param' => 'id', 'help' => pht('ID of build plan to run.'), ), + array( + 'name' => 'background', + 'help' => pht( + 'Submit builds into the build queue normally instead of '. + 'running them in the foreground.'), + ), array( 'name' => 'buildable', 'wildcard' => true, ), )); } public function execute(PhutilArgumentParser $args) { $viewer = $this->getViewer(); $names = $args->getArg('buildable'); if (count($names) != 1) { throw new PhutilArgumentUsageException( pht('Specify exactly one buildable object, by object name.')); } $name = head($names); $buildable = id(new PhabricatorObjectQuery()) ->setViewer($viewer) ->withNames($names) ->executeOne(); if (!$buildable) { throw new PhutilArgumentUsageException( pht('No such buildable "%s"!', $name)); } if (!($buildable instanceof HarbormasterBuildableInterface)) { throw new PhutilArgumentUsageException( pht('Object "%s" is not a buildable!', $name)); } $plan_id = $args->getArg('plan'); if (!$plan_id) { throw new PhutilArgumentUsageException( pht( 'Use %s to specify a build plan to run.', '--plan')); } $plan = id(new HarbormasterBuildPlanQuery()) ->setViewer($viewer) ->withIDs(array($plan_id)) ->executeOne(); if (!$plan) { throw new PhutilArgumentUsageException( pht('Build plan "%s" does not exist.', $plan_id)); } if (!$plan->canRunManually()) { throw new PhutilArgumentUsageException( pht('This build plan can not be run manually.')); } $console = PhutilConsole::getConsole(); $buildable = HarbormasterBuildable::initializeNewBuildable($viewer) ->setIsManualBuildable(true) ->setBuildablePHID($buildable->getHarbormasterBuildablePHID()) ->setContainerPHID($buildable->getHarbormasterContainerPHID()) ->save(); $console->writeOut( "%s\n", pht( 'Applying plan %s to new buildable %s...', $plan->getID(), 'B'.$buildable->getID())); $console->writeOut( "\n %s\n\n", PhabricatorEnv::getProductionURI('/B'.$buildable->getID())); - PhabricatorWorker::setRunAllTasksInProcess(true); + if (!$args->getArg('background')) { + PhabricatorWorker::setRunAllTasksInProcess(true); + } + $buildable->applyPlan($plan, array()); $console->writeOut("%s\n", pht('Done.')); return 0; } }