diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -701,6 +701,8 @@ 'DrydockBlueprintCoreCustomField' => 'applications/drydock/customfield/DrydockBlueprintCoreCustomField.php', 'DrydockBlueprintCreateController' => 'applications/drydock/controller/DrydockBlueprintCreateController.php', 'DrydockBlueprintCustomField' => 'applications/drydock/customfield/DrydockBlueprintCustomField.php', + 'DrydockBlueprintCustomFieldBlueprints' => 'applications/drydock/customfield/DrydockBlueprintCustomFieldBlueprints.php', + 'DrydockBlueprintDatasource' => 'applications/drydock/typeahead/DrydockBlueprintDatasource.php', 'DrydockBlueprintEditController' => 'applications/drydock/controller/DrydockBlueprintEditController.php', 'DrydockBlueprintEditor' => 'applications/drydock/editor/DrydockBlueprintEditor.php', 'DrydockBlueprintImplementation' => 'applications/drydock/blueprint/DrydockBlueprintImplementation.php', @@ -881,6 +883,7 @@ 'HarbormasterDAO' => 'applications/harbormaster/storage/HarbormasterDAO.php', 'HarbormasterHTTPRequestBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterHTTPRequestBuildStepImplementation.php', 'HarbormasterLeaseHostBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseHostBuildStepImplementation.php', + 'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php', 'HarbormasterManagePlansCapability' => 'applications/harbormaster/capability/HarbormasterManagePlansCapability.php', 'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php', 'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php', @@ -3982,6 +3985,8 @@ ), 'DrydockBlueprintCreateController' => 'DrydockBlueprintController', 'DrydockBlueprintCustomField' => 'PhabricatorCustomField', + 'DrydockBlueprintCustomFieldBlueprints' => 'PhabricatorStandardCustomFieldPHIDs', + 'DrydockBlueprintDatasource' => 'PhabricatorTypeaheadDatasource', 'DrydockBlueprintEditController' => 'DrydockBlueprintController', 'DrydockBlueprintEditor' => 'PhabricatorApplicationTransactionEditor', 'DrydockBlueprintListController' => 'DrydockBlueprintController', @@ -4213,6 +4218,7 @@ 'HarbormasterDAO' => 'PhabricatorLiskDAO', 'HarbormasterHTTPRequestBuildStepImplementation' => 'HarbormasterBuildStepImplementation', 'HarbormasterLeaseHostBuildStepImplementation' => 'HarbormasterBuildStepImplementation', + 'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation', 'HarbormasterManagePlansCapability' => 'PhabricatorPolicyCapability', 'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow', diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -10,6 +10,7 @@ private $activeResource; private $activeLease; private $instance; + private $scopes = array(); abstract public function getType(); abstract public function getInterface( @@ -606,10 +607,11 @@ DrydockResource $resource = null, DrydockLease $lease = null) { - if (($this->activeResource !== null) || - ($this->activeLease !== null)) { - throw new Exception(pht('There is already an active resource or lease!')); - } + $scope = array( + 'resource' => $resource, + 'lease' => $lease, + ); + array_push($this->scopes, $scope); $this->activeResource = $resource; $this->activeLease = $lease; @@ -618,8 +620,20 @@ } public function popActiveScope() { - $this->activeResource = null; - $this->activeLease = null; + if (count($this->scopes) === 0) { + throw new Exception('Unable to pop active scope; no scopes active'); + } + + array_pop($this->scopes); + + if (count($this->scopes) === 0) { + $this->activeResource = null; + $this->activeLease = null; + } else { + $current = last($this->scopes); + $this->activeResource = $current['resource']; + $this->activeLease = $current['lease']; + } } } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -3,6 +3,8 @@ final class DrydockWorkingCopyBlueprintImplementation extends DrydockBlueprintImplementation { + private $cachedHostBlueprint = null; + public function isEnabled() { return true; } @@ -15,14 +17,180 @@ return pht('Allows Drydock to check out working copies of repositories.'); } + private function resolveRelatedObjectsForLease(DrydockLease $lease) { + if ($lease->getAttribute('resolved.target') !== null) { + return; + } + + $url = $lease->getAttribute('url'); + $buildable_phid = $lease->getAttribute('buildablePHID'); + + if ($url) { + $lease->setAttribute('resolved.target', 'url'); + $lease->setAttribute('resolved.repositoryURL', $url); + $lease->save(); + + $this->log(pht( + 'Resolved working copy target as "url"')); + $this->log(pht( + 'Resolved working copy repository URL as "%s"', + $lease->getAttribute('resolved.repositoryURL'))); + } else if ($buildable_phid) { + $buildable = id(new HarbormasterBuildableQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($buildable_phid)) + ->needContainerObjects(true) + ->executeOne(); + if ($buildable === null) { + throw new Exception(pht( + 'No buildable found with PHID %s', + $buildable_phid)); + } + $buildable_object = $buildable->getBuildableObject(); + $container_object = $buildable->getContainerObject(); + + if ($buildable_object instanceof PhabricatorRepositoryCommit && + $container_object instanceof PhabricatorRepository) { + $lease->setAttribute('resolved.target', 'commit'); + $lease->setAttribute( + 'resolved.commitIdentifier', + $buildable_object->getCommitIdentifier()); + $lease->setAttribute( + 'resolved.repositoryPHID', + $container_object->getPHID()); + $lease->save(); + + $this->log(pht( + 'Resolved working copy target as "commit"')); + $this->log(pht( + 'Resolved working copy commit identifier as "%s"', + $lease->getAttribute('resolved.commitIdentifier'))); + $this->log(pht( + 'Resolved working copy repository PHID as "%s"', + $lease->getAttribute('resolved.repositoryPHID'))); + } else if ($buildable_object instanceof DifferentialDiff && + $container_object instanceof DifferentialRevision) { + $lease->setAttribute('resolved.target', 'diff'); + $lease->setAttribute( + 'resolved.diffID', + $buildable_object->getID()); + $lease->setAttribute( + 'resolved.revisionID', + $container_object->getID()); + $lease->setAttribute( + 'resolved.repositoryPHID', + $container_object->getRepository()->getPHID()); + $lease->setAttribute( + 'resolved.baseRevision', + $buildable_object->getSourceControlBaseRevision()); + $lease->save(); + + $this->log(pht( + 'Resolved working copy target as "diff"')); + $this->log(pht( + 'Resolved working copy diff ID as "%d"', + $lease->getAttribute('resolved.diffID'))); + $this->log(pht( + 'Resolved working copy revision ID as "%d"', + $lease->getAttribute('resolved.revisionID'))); + $this->log(pht( + 'Resolved working copy repository PHID as "%s"', + $lease->getAttribute('resolved.repositoryPHID'))); + $this->log(pht( + 'Resolved working copy base revision as "%s"', + $lease->getAttribute('resolved.baseRevision'))); + } + } + } + + private function getHostBlueprintPHID() { + return head(phutil_json_decode($this->getDetail('host-blueprint'))); + } + + private function getHostBlueprint() { + if ($this->cachedHostBlueprint !== null) { + return $this->cachedHostBlueprint; + } + + $host_blueprint = id(new DrydockBlueprintQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($this->getHostBlueprintPHID())) + ->executeOne(); + if (!$host_blueprint) { + throw new Exception(pht( + 'This blueprint does not have a valid host blueprint.')); + } + + $this->cachedHostBlueprint = $host_blueprint; + return $host_blueprint; + } + + public function canAllocateResourceForLease(DrydockLease $lease) { + $host_blueprint = $this->getHostBlueprint(); + + $host_platform = $host_blueprint->getDetail('platform'); + if ($host_platform === null) { + $host_platform = $this->getDetail('host-platform'); + } + + $lease_platform = $lease->getAttribute('platform'); + + return $host_platform === $lease_platform; + } + protected function canAllocateLease( DrydockResource $resource, DrydockLease $lease) { - $resource_repo = $resource->getAttribute('repositoryID'); - $lease_repo = $lease->getAttribute('repositoryID'); + $lease_platform = $lease->getAttribute('platform'); + $resource_platform = $resource->getAttribute('platform'); + + $platform_match = $lease_platform === $resource_platform; + + $custom_match = DrydockCustomAttributes::hasRequirements( + $lease->getAttributes(), + $this->getDetail('attributes')); + + $host_lease_id = $lease->getAttribute('hostLeaseID'); + if ($host_lease_id !== null) { + $host_lease = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array($host_lease_id)) + ->executeOne(); + if ($host_lease === null) { + throw new Exception(pht( + 'No lease found with ID %d', + $host_lease_id)); + } + if ($host_lease->getResource()->getBlueprint()->getPHID() + !== $this->getHostBlueprintPHID()) { + $this->log(pht( + 'This blueprint can not allocate a resource on the required host.')); + return false; + } + } + + $resource_repo = $resource->getAttribute('repositoryPHID'); + $resource_url = $resource->getAttribute('repositoryURL'); + + $this->resolveRelatedObjectsForLease($lease); - return ($resource_repo && $lease_repo && ($resource_repo == $lease_repo)); + $lease_repo = $lease->getAttribute('resolved.repositoryPHID'); + $lease_url = $lease->getAttribute('resolved.repositoryURL'); + + $can_allocate = $platform_match && $custom_match && + (($resource_repo && $lease_repo && ($resource_repo == $lease_repo)) || + ($resource_url && $lease_url && ($resource_url == $lease_url))); + + if ($can_allocate) { + $this->log(pht( + 'This blueprint can allocate a resource for the specified lease.')); + } else { + $this->log(pht( + 'This blueprint can not allocate a resource for the specified lease.')); + } + + return $can_allocate; } protected function shouldAllocateLease( @@ -30,77 +198,449 @@ DrydockResource $resource, DrydockLease $lease) { - return $context->getCurrentResourceLeaseCount() === 0; + return true; } protected function executeInitializePendingResource( DrydockResource $resource, - DrydockLease $lease) {} + DrydockLease $lease) { + + $this->resolveRelatedObjectsForLease($lease); + + $target = $lease->getAttribute('resolved.target'); + if (!$target) { + throw new Exception( + 'Unable to resolve working copy target for lease.'); + } + + $repository_phid = $lease->getAttribute('resolved.repositoryPHID'); + $repository_url = $lease->getAttribute('resolved.repositoryURL'); + + // We must set the platform so that other allocators will lease + // against it successfully. + $resource + ->setAttribute('platform', $lease->getAttribute('platform')) + ->setAttribute('repositoryPHID', $repository_phid) + ->setAttribute('repositoryURL', $repository_url) + ->save(); + } protected function executeAllocateResource( DrydockResource $resource, DrydockLease $lease) { - $repository_id = $lease->getAttribute('repositoryID'); - if (!$repository_id) { - throw new Exception( - pht( - "Lease is missing required '%s' attribute.", - 'repositoryID')); + $repository_phid = $lease->getAttribute('resolved.repositoryPHID'); + $repository_url = $lease->getAttribute('resolved.repositoryURL'); + + if ($repository_url) { + $resource + ->setName('Working Copy ('.$repository_url.')') + ->setStatus(DrydockResourceStatus::STATUS_PENDING) + ->setAttribute('repositoryURL', $repository_url) + ->setAttribute('platform', $lease->getAttribute('platform')) + ->save(); + } else if ($repository_phid) { + $repository = id(new PhabricatorRepositoryQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($repository_phid)) + ->executeOne(); + + if (!$repository) { + throw new Exception( + "Repository with PHID '{$repository_phid}' does not exist!"); + } + + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + break; + default: + throw new Exception(pht('Unsupported VCS!')); + } + + $resource + ->setName('Working Copy ('.$repository->getCallsign().')') + ->setStatus(DrydockResourceStatus::STATUS_PENDING) + ->setAttribute('repositoryPHID', $repository->getPHID()) + ->setAttribute('platform', $lease->getAttribute('platform')) + ->save(); } - $repository = id(new PhabricatorRepositoryQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withIDs(array($repository_id)) - ->executeOne(); - - if (!$repository) { - throw new Exception( - pht( - "Repository '%s' does not exist!", - $repository_id)); + // When allocating the resource, we always need to get a new lease + // that is owned by the resource. If the lease has specified an + // existing lease, we can a new lease against the same resource as + // that lease. + + $host_lease = null; + $host_lease_id = $lease->getAttribute('hostLeaseID'); + $resource_id = null; + if ($host_lease_id !== null) { + $host_lease = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array($host_lease_id)) + ->executeOne(); + if ($host_lease === null) { + throw new Exception(pht( + 'No lease found with ID %d', + $host_lease_id)); + } + $resource_id = $host_lease->getResource()->getID(); } - switch ($repository->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - break; - default: - throw new Exception(pht('Unsupported VCS!')); + if ($resource_id) { + $this->log(pht( + 'Limiting lease acquisition to resource ID %d.', + $resource_id)); } - // TODO: Policy stuff here too. - $host_lease = id(new DrydockLease()) + $this->log('Acquiring new host lease for working copy...'); + + $resource_lease = id(new DrydockLease()) ->setResourceType('host') + ->setAttributes( + array( + 'resourceID' => $resource_id, + 'platform' => $lease->getAttribute('platform'), + )) ->waitUntilActive(); - $path = $host_lease->getAttribute('path').$repository->getCallsign(); + $this->log(pht( + 'Lease %d acquired for working copy resource.', + $resource_lease->getID())); - $this->log( - pht('Cloning %s into %s....', $repository->getCallsign(), $path)); + if ($lease->getAttribute('platform') === 'windows') { + $cmd = $resource_lease->getInterface( + 'command-'.PhutilCommandString::MODE_WINDOWSCMD); + } else { + $cmd = $resource_lease->getInterface( + 'command-'.PhutilCommandString::MODE_BASH); + } - $cmd = $host_lease->getInterface('command'); - $cmd->execx( - 'git clone --origin origin %P %s', - $repository->getRemoteURIEnvelope(), - $path); + if ($repository_phid) { + $this->log(pht( + 'Cloning repository at "%s" to "%s"...', + $repository->getPublicCloneURI(), + $resource_lease->getAttribute('path'))); + + $cmd->execx( + 'git clone --bare %s .', + $repository->getPublicCloneURI()); + } else { + $this->log(pht( + 'Cloning repository at "%s" to "%s"...', + $resource->getAttribute('repositoryURL'), + $resource_lease->getAttribute('path'))); + + $cmd->execx( + 'git clone --bare %s .', + $resource->getAttribute('repositoryURL')); + } - $this->log(pht('Complete.')); + $this->log('Cloned repository cache.'); $resource - ->setName('Working Copy ('.$repository->getCallsign().')') ->setStatus(DrydockResourceStatus::STATUS_OPEN) - ->setAttribute('lease.host', $host_lease->getID()) - ->setAttribute('path', $path) - ->setAttribute('repositoryID', $repository->getID()) + ->setAttribute('host.lease', $resource_lease->getID()) + ->setAttribute('host.resource', $resource_lease->getResource()->getID()) + ->setAttribute('path', $resource_lease->getAttribute('path')) + ->setAttribute('platform', $resource_lease->getAttribute('platform')) ->save(); - return $resource; } protected function executeAcquireLease( DrydockResource $resource, DrydockLease $lease) { - return; + + $this->log(pht( + 'Starting acquisition of lease from resource %d', + $resource->getID())); + + while ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { + $this->log(pht( + 'Resource %d is still pending, waiting until it is in an open status', + $resource->getID())); + + // This resource is still being set up by another allocator, wait until + // it is set to open. + sleep(5); + $resource->reload(); + } + + $resource_lease = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array($resource->getAttribute('host.lease'))) + ->executeOne(); + if ($resource_lease === null) { + throw new Exception(pht( + 'No resource found with ID %d', + $resource->getAttribute('host.lease'))); + } + + // We must lock the resource while we perform the cache update, + // because otherwise we'll end up running multiple read-write + // VCS operations in the same directory at the same time. + $lock = PhabricatorGlobalLock::newLock( + 'drydock-working-copy-cache-update-'.$resource_lease->getID()); + $lock->lock(1000000); + try { + if ($lease->getAttribute('platform') === 'windows') { + $cmd = $resource_lease->getInterface( + 'command-'.PhutilCommandString::MODE_WINDOWSCMD); + } else { + $cmd = $resource_lease->getInterface( + 'command-'.PhutilCommandString::MODE_BASH); + } + + $this->log(pht( + 'Fetching latest commits for repository at "%s"', + $resource->getAttribute('path'))); + $cmd->exec('git fetch origin +refs/heads/*:refs/heads/*'); + $this->log(pht('Fetched latest commits.')); + + $lock->unlock(); + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } + + $host_lease = null; + if ($lease->getAttribute('cacheOnly')) { + // This is a cache-only lease; we just pass through the resource's + // attributes so we don't make an unnecessary copy. This means the + // acquirer of the lease intends to clone from the resource's cache + // directly. + $this->log(pht( + 'Cache-only lease requested, passing through resource lease directly')); + $lease + ->setAttribute('cacheOnly', true) + ->setAttribute('host.lease', $resource->getAttribute('host.lease')); + $host_lease = $resource_lease; + } else { + $host_lease_id = $lease->getAttribute('hostLeaseID'); + if ($host_lease_id !== null) { + $host_lease = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array($host_lease_id)) + ->executeOne(); + if ($host_lease === null) { + throw new Exception(pht( + 'No lease found with ID %d', + $host_lease_id)); + } + $this->log(pht( + 'Using existing host lease %d for working copy resource.', + $host_lease_id)); + } else { + $this->log('Acquiring new host lease for working copy...'); + $host_lease = id(new DrydockLease()) + ->setResourceType('host') + ->setAttributes( + array( + 'resourceID' => $resource->getAttribute('host.resource'), + 'platform' => $lease->getAttribute('platform'), + )) + ->waitUntilActive(); + $host_lease_id = $host_lease->getID(); + $this->log(pht( + 'Lease %d acquired for working copy resource.', + $host_lease_id)); + } + + $lease->setAttribute('host.lease', $host_lease_id); + + if ($lease->getAttribute('platform') === 'windows') { + $cmd = $lease->getInterface( + 'command-'.PhutilCommandString::MODE_WINDOWSCMD); + } else { + $cmd = $lease->getInterface( + 'command-'.PhutilCommandString::MODE_BASH); + } + + $this->log(pht( + 'Cloning from cache path "%s" to lease path "%s"', + $resource->getAttribute('path'), + $host_lease->getAttribute('path'))); + $cmd->execx( + 'git clone %s .', + $resource->getAttribute('path')); + $this->log(pht('Cloned from cache')); + + if ($lease->getAttribute('resolved.target') === 'commit') { + $this->log(pht( + 'Checking out target commit "%s"', + $lease->getAttribute('resolved.commitIdentifier'))); + $cmd->execx( + 'git checkout -f %s', + $lease->getAttribute('resolved.commitIdentifier')); + $this->log(pht('Checked out commit')); + } else if ($lease->getAttribute('resolved.target') === 'url') { + // Leave as default; we use the working copy for cloning only. + } else { + throw new Exception(pht( + 'Target type %s not yet supported.', + $lease->getAttribute('resolved.target'))); + } + + $this->initializeSubmodules( + $host_lease, + $host_lease->getAttribute('path')); + } + } + + private function initializeSubmodules( + DrydockLease $working_directory_lease, + $working_directory_path = null) { + + if ($working_directory_lease->getAttribute('platform') === 'windows') { + $working_directory_cmd = $working_directory_lease->getInterface( + 'command-'.PhutilCommandString::MODE_WINDOWSCMD); + } else { + $working_directory_cmd = $working_directory_lease->getInterface( + 'command-'.PhutilCommandString::MODE_BASH); + } + + $working_directory_cmd->setWorkingDirectory($working_directory_path); + + $this->log(pht( + 'Initializing submodules in %s', + $working_directory_path)); + $working_directory_cmd->execx('git submodule init'); + $this->log(pht( + 'Initialized submodules in %s', + $working_directory_path)); + + $this->log(pht( + 'Discovering initialized submodules in %s', + $working_directory_path)); + list($stdout, $stderr) = + $working_directory_cmd->execx('git config --local --list'); + $matches = null; + preg_match_all( + '/submodule\.(?.*)\.url=(?.*)/', + $stdout, + $matches); + $this->log(pht( + 'Standard output is %s', + $stdout)); + $this->log(pht( + 'Submodule array is %s', + print_r($matches, true))); + $submodules = array(); + for ($i = 0; $i < count($matches['name']); $i++) { + $name = $matches['name'][$i]; + $url = $matches['url'][$i]; + + $submodules[$name] = $url; + + $this->log(pht( + 'Discovered submodule %s registered with URL %s', + $name, + $url)); + } + + foreach ($submodules as $name => $url) { + $this->log(pht( + 'Caching submodule at URL %s by leasing working copy', + $url)); + + $submodule_lease = id(new DrydockLease()) + ->setResourceType('working-copy') + ->setAttributes( + array( + 'platform' => $working_directory_lease->getAttribute('platform'), + 'hostLeaseID' => + $working_directory_lease->getAttribute('hostLeaseID'), + 'url' => $url, + 'cacheOnly' => true, + )) + ->queueForActivation(); + + $this->log(pht( + 'Starting submodule lease %d', + $submodule_lease->getID())); + + $submodule_lease->waitUntilActive(); + + $this->log(pht( + 'Acquired submodule lease %d', + $submodule_lease->getID())); + + // Load the host leases for both the submodule working directory. + $submodule_working_directory_lease_id = + $submodule_lease->getAttribute('host.lease'); + + $this->log(pht( + 'Submodule working directory host lease ID is %d', + $submodule_working_directory_lease_id)); + + $submodule_leases = id(new DrydockLeaseQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array( + $submodule_working_directory_lease_id, + )) + ->execute(); + $submodule_leases = mpull($submodule_leases, null, 'getID'); + $submodule_working_directory_lease = + idx($submodule_leases, $submodule_working_directory_lease_id); + + $submodule_working_directory = + $submodule_working_directory_lease->getAttribute('path'); + + $this->log(pht( + 'Updating local submodule URL to point to %s', + $submodule_working_directory)); + + $working_directory_cmd->execx( + 'git config --local submodule.%s.url %s', + $name, + $submodule_working_directory); + + $this->log(pht( + 'Updating submodule %s', + $name)); + + $working_directory_cmd->execx('git submodule update %s', $name); + + $this->log(pht( + 'Loading leases for submodule %s', + $name)); + + $this->log(pht( + 'Recursively initializing submodules from cache for %s', + $name)); + + $this->initializeSubmodules( + $submodule_working_directory_lease, + $working_directory_path.'/'.$name); + + $this->log(pht( + 'Recursive submodule initialization complete for %s', + $name)); + + $submodule_lease_id = $submodule_lease->getID(); + $this->log(pht( + 'Releasing lease for %d', + $submodule_lease_id)); + + $submodule_lease->release(); + + $this->log(pht( + 'Released lease for %d', + $submodule_lease_id)); + + $this->log(pht( + 'Updating local submodule URL to point back to %s', + $url)); + + $working_directory_cmd->execx( + 'git config --local submodule.%s.url %s', + $name, + $url); + } + + $this->log(pht( + 'Submodules initialized for working directory at %s', + $working_directory_path)); } public function getType() { @@ -112,19 +652,40 @@ DrydockLease $lease, $type) { - switch ($type) { - case 'command': - return $this - ->loadLease($resource->getAttribute('lease.host')) - ->getInterface($type); - } - - throw new Exception(pht("No interface of type '%s'.", $type)); + return $this + ->loadLease($lease->getAttribute('host.lease')) + ->getInterface($type); } protected function executeReleaseLease( DrydockResource $resource, - DrydockLease $lease) {} + DrydockLease $lease) { + + if ($lease->getAttribute('cacheOnly')) { + $this->log(pht( + 'Cache-only lease; not releasing related lease')); + } else if (!$lease->getAttribute('hostLeaseID')) { + $this->log(pht( + 'Releasing host lease %d', + $lease->getAttribute('host.lease'))); + try { + $host_lease = $this->loadLease($lease->getAttribute('host.lease')); + + $host_resource = $host_lease->getResource(); + $host_blueprint = $host_resource->getBlueprint(); + $host_blueprint->releaseLease($host_resource, $host_lease); + + $this->log(pht( + 'Released host lease %d', + $lease->getAttribute('host.lease'))); + } catch (Exception $ex) { + $this->log(pht( + 'Unable to release host lease %d: "%s"', + $lease->getAttribute('host.lease'), + (string)$ex)); + } + } + } protected function shouldCloseUnleasedResource( DrydockAllocationContext $context, @@ -134,7 +695,67 @@ } protected function executeCloseResource(DrydockResource $resource) { - // TODO: Remove leased directory + $this->log(pht( + 'Releasing resource host lease %d', + $resource->getAttribute('host.lease'))); + try { + $host_lease = $this->loadLease($resource->getAttribute('host.lease')); + + $host_resource = $host_lease->getResource(); + $host_blueprint = $host_resource->getBlueprint(); + $host_blueprint->releaseLease($host_resource, $host_lease); + + $this->log(pht( + 'Released resource host lease %d', + $resource->getAttribute('host.lease'))); + } catch (Exception $ex) { + $this->log(pht( + 'Unable to release resource host lease %d: "%s"', + $resource->getAttribute('host.lease'), + (string)$ex)); + } + } + + public function getFieldSpecifications() { + return array( + 'host-config' => array( + 'name' => pht('Host Configuration'), + 'type' => 'header', + ), + 'host-blueprint' => array( + 'name' => pht('Host Blueprint'), + 'type' => 'blueprints', + 'required' => true, + 'limit' => 1, + 'blueprint-type' => 'host', + 'caption' => pht( + 'The blueprint which provides hosts that this '. + 'blueprint will operate on.'), + ), + 'host-platform' => array( + 'name' => pht('Host Platform'), + 'type' => 'text', + 'required' => true, + 'caption' => pht( + 'An optional host platform to specify; only used '. + 'if the host blueprint doesn\'t have a platform '. + 'specified already (such is the case for '. + 'preallocated hosts).'), + ), + 'attr-header' => array( + 'name' => pht('Working Copy Attributes'), + 'type' => 'header', + ), + 'attributes' => array( + 'name' => pht('Working Copy Attributes'), + 'type' => 'textarea', + 'caption' => pht( + 'A newline separated list of working copy attributes. '. + 'Each attribute should be specified in a key=value format.'), + 'monospace' => true, + ), + ) + parent::getFieldSpecifications(); } + } diff --git a/src/applications/drydock/customfield/DrydockBlueprintCustomFieldBlueprints.php b/src/applications/drydock/customfield/DrydockBlueprintCustomFieldBlueprints.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/customfield/DrydockBlueprintCustomFieldBlueprints.php @@ -0,0 +1,50 @@ +getFieldValue(); + + $control = id(new AphrontFormTokenizerControl()) + ->setUser($this->getViewer()) + ->setLabel($this->getFieldName()) + ->setName($this->getFieldKey()) + ->setDatasource(id(new DrydockBlueprintDatasource()) + ->setParameters(array( + 'type' => $this->getFieldConfigValue('blueprint-type'), + ))) + ->setCaption($this->getCaption()) + ->setValue(nonempty($value, array())); + + $limit = $this->getFieldConfigValue('limit'); + if ($limit) { + $control->setLimit($limit); + } + + return $control; + } + + public function appendToApplicationSearchForm( + PhabricatorApplicationSearchEngine $engine, + AphrontFormView $form, + $value, + array $handles) { + + $control = id(new AphrontFormTokenizerControl()) + ->setLabel($this->getFieldName()) + ->setName($this->getFieldKey()) + ->setDatasource(id(new DrydockBlueprintDatasource()) + ->setParameters(array( + 'type' => $this->getFieldConfigValue('blueprint-type'), + ))) + ->setValue(nonempty($value, array())); + + $form->appendControl($control); + } + +} diff --git a/src/applications/drydock/phid/DrydockBlueprintPHIDType.php b/src/applications/drydock/phid/DrydockBlueprintPHIDType.php --- a/src/applications/drydock/phid/DrydockBlueprintPHIDType.php +++ b/src/applications/drydock/phid/DrydockBlueprintPHIDType.php @@ -29,6 +29,7 @@ $blueprint = $objects[$phid]; $id = $blueprint->getID(); + $handle->setName($blueprint->getBlueprintName()); $handle->setURI("/drydock/blueprint/{$id}/"); } } diff --git a/src/applications/drydock/typeahead/DrydockBlueprintDatasource.php b/src/applications/drydock/typeahead/DrydockBlueprintDatasource.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/typeahead/DrydockBlueprintDatasource.php @@ -0,0 +1,52 @@ +getViewer(); + + $blueprint_type = $this->getParameter('type'); + + $blueprints = id(new DrydockBlueprintQuery()) + ->setViewer($viewer) + ->execute(); + $blueprints = mpull($blueprints, null, 'getPHID'); + + if (count($blueprints) === 0) { + return array(); + } + + $results = array(); + foreach ($blueprints as $phid => $blueprint) { + if ($blueprint_type !== null && + $blueprint->getImplementation()->getType() !== $blueprint_type) { + continue; + } + + $results[] = id(new PhabricatorTypeaheadResult()) + ->setName($blueprint->getBlueprintName()) + ->setURI('/') + ->setPHID($phid); + } + + return $results; + } + +} diff --git a/src/applications/drydock/worker/DrydockAllocatorWorker.php b/src/applications/drydock/worker/DrydockAllocatorWorker.php --- a/src/applications/drydock/worker/DrydockAllocatorWorker.php +++ b/src/applications/drydock/worker/DrydockAllocatorWorker.php @@ -104,6 +104,12 @@ continue; } + if ($lease->getAttribute('resourceID') !== null && + $candidate->getID() !== $lease->getAttribute('resourceID')) { + unset($pool[$key]); + continue; + } + $blueprint = $blueprints[$candidate->getBlueprintPHID()]; $implementation = $blueprint->getImplementation(); @@ -144,6 +150,12 @@ continue; } + if ($lease->getAttribute('resourceID') !== null && + $candidate->getID() !== $lease->getAttribute('resourceID')) { + unset($pool[$key]); + continue; + } + $blueprint = $blueprints[$candidate->getBlueprintPHID()]; $implementation = $blueprint->getImplementation(); diff --git a/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php b/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php new file mode 100644 --- /dev/null +++ b/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php @@ -0,0 +1,84 @@ +getSettings(); + + $custom_attributes = DrydockCustomAttributes::parse( + $settings['attributes']); + + // Create the lease. + $lease = id(new DrydockLease()) + ->setResourceType('working-copy') + ->setAttributes( + array( + 'platform' => $settings['platform'], + 'buildablePHID' => $build->getBuildablePHID(), + ) + $custom_attributes) + ->queueForActivation(); + + // Create the associated artifact. + $artifact = $build->createArtifact( + $build_target, + $settings['name'], + HarbormasterBuildArtifact::TYPE_HOST); + $artifact->setArtifactData(array( + 'drydock-lease' => $lease->getID(), + )); + $artifact->save(); + + // Wait until the lease is fulfilled. + // TODO: This will throw an exception if the lease can't be fulfilled; + // we should treat that as build failure not build error. + $lease->waitUntilActive(); + } + + public function getArtifactOutputs() { + return array( + array( + 'name' => pht('Leased Working Copy'), + 'key' => $this->getSetting('name'), + 'type' => HarbormasterBuildArtifact::TYPE_HOST, + ), + ); + } + + public function getFieldSpecifications() { + return array( + 'name' => array( + 'name' => pht('Artifact Name'), + 'type' => 'text', + 'required' => true, + ), + 'platform' => array( + 'name' => pht('Host Platform'), + 'type' => 'text', + 'required' => true, + ), + 'attributes' => array( + 'name' => pht('Required Attributes'), + 'type' => 'textarea', + 'caption' => pht( + 'A newline separated list of required working copy attributes. '. + 'Each attribute should be specified in a key=value format.'), + 'monospace' => true, + ), + ); + } + +}