diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -704,6 +704,7 @@ 'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php', 'DrydockAllocationContext' => 'applications/drydock/util/DrydockAllocationContext.php', 'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php', + 'DrydockAmazonEC2HostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAmazonEC2HostBlueprintImplementation.php', 'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php', 'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php', 'DrydockBlueprintController' => 'applications/drydock/controller/DrydockBlueprintController.php', @@ -4092,6 +4093,7 @@ 'DoorkeeperTagsController' => 'PhabricatorController', 'DrydockAllocationContext' => 'Phobject', 'DrydockAllocatorWorker' => 'PhabricatorWorker', + 'DrydockAmazonEC2HostBlueprintImplementation' => 'DrydockMinMaxExpiryBlueprintImplementation', 'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface', 'DrydockBlueprint' => array( 'DrydockDAO', diff --git a/src/applications/drydock/blueprint/DrydockAmazonEC2HostBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockAmazonEC2HostBlueprintImplementation.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/blueprint/DrydockAmazonEC2HostBlueprintImplementation.php @@ -0,0 +1,602 @@ +setAWSKeys( + PhabricatorEnv::getEnvConfig('amazon-ec2.access-key'), + PhabricatorEnv::getEnvConfig('amazon-ec2.secret-key')) + ->setAWSRegion($this->getDetail('region')); + } + + private function getAWSKeyPairName() { + return 'phabricator-'.$this->getDetail('keypair'); + } + + public function canAllocateResourceForLease(DrydockLease $lease) { + return + $lease->getAttribute('platform') === $this->getDetail('platform'); + } + + protected function canAllocateLease( + DrydockResource $resource, + DrydockLease $lease) { + return + $lease->getAttribute('platform') === $resource->getAttribute('platform'); + } + + protected function executeAllocateResource( + DrydockResource $resource, + DrydockLease $lease) { + + // We need to retrieve this as we need to use it for both importing the + // key and looking up the ID for the resource attributes. + $credential = id(new PassphraseCredentialQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($this->getDetail('keypair'))) + ->executeOne(); + + if ($credential === null) { + throw new Exception('Specified credential does not exist!'); + } + + // Determine the correct protocol for this platform type. + $protocol = 'ssh'; + if ($this->getDetail('platform') === 'windows') { + $protocol = 'winrm'; + } + + $winrm_auth_id = null; + if ($protocol === 'winrm') { + $winrm_auth = id(new PassphraseCredentialQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($this->getDetail('winrm-auth'))) + ->executeOne(); + + if ($winrm_auth === null) { + throw new Exception( + 'Specified credential for WinRM auth does not exist!'); + } + + $winrm_auth_id = $winrm_auth->getID(); + + $this->log(pht( + 'Using credential %d to authenticate over WinRM.', + $winrm_auth->getID())); + } + + try { + $existing_keys = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DescribeKeyPairs', + array( + 'KeyName.0' => $this->getAWSKeyPairName(), + )) + ->resolve(); + } catch (PhutilAWSException $ex) { + // The key pair does not exist, so we need to import it. + + $type = PassphraseCredentialType::getTypeByConstant( + $credential->getCredentialType()); + if (!$type) { + throw new Exception(pht('Credential has invalid type "%s"!', $type)); + } + + if (!$type->hasPublicKey()) { + throw new Exception(pht('Credential has no public key!')); + } + + $public_key = $type->getPublicKey( + PhabricatorUser::getOmnipotentUser(), + $credential); + + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'ImportKeyPair', + array( + 'KeyName' => $this->getAWSKeyPairName(), + 'PublicKeyMaterial' => base64_encode($public_key), + )) + ->resolve(); + } + + $settings = array( + 'ImageId' => $this->getDetail('ami'), + 'MinCount' => 1, + 'MaxCount' => 1, + 'KeyName' => $this->getAWSKeyPairName(), + 'InstanceType' => $this->getDetail('size'), + 'SubnetId' => $this->getDetail('subnet-id'), + ); + + $i = 0; + $security_groups = explode(',', $this->getDetail('security-group-ids')); + foreach ($security_groups as $security_group) { + $settings['SecurityGroupId.'.$i] = $security_group; + $i++; + } + + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'RunInstances', + $settings) + ->resolve(); + + $instance = $result->instancesSet->item[0]; + $instance_id = (string)$instance->instanceId; + + // Allocate the resource and place it into Pending status while + // we wait for the instance to start. + $blueprint = $this->getInstance(); + $resource + ->setName($instance_id) + ->setStatus(DrydockResourceStatus::STATUS_PENDING) + ->setAttributes(array( + 'instance-id' => $instance_id, + 'platform' => $this->getDetail('platform'), + 'protocol' => $protocol, + 'path' => $this->getDetail('storage-path'), + 'credential' => $credential->getID(), + 'winrm-auth' => $winrm_auth_id, + 'aws-status' => 'Instance Requested', + )) + ->save(); + + // Wait until the instance has started. + while (true) { + try { + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DescribeInstances', + array( + 'InstanceId.0' => $instance_id, + )) + ->resolve(); + + $reservation = $result->reservationSet->item[0]; + $instance = $reservation->instancesSet->item[0]; + $instance_state = (string)$instance->instanceState->name; + + if ($instance_state === 'pending') { + sleep(5); + continue; + } else if ($instance_state === 'running') { + break; + } else { + // Instance is shutting down or is otherwise terminated. + throw new Exception( + 'Allocated instance, but ended up in unexpected state \''. + $instance_state.'\'!'); + } + } catch (PhutilAWSException $ex) { + // TODO: This can happen because the instance doesn't exist yet, but + // we should check specifically for that error. + sleep(5); + continue; + } + } + + $resource->setAttribute('aws-status', 'Started in Amazon'); + $resource->save(); + + // Calculate the IP address of the instance. + $address = ''; + if ($this->getDetail('allocate-elastic-ip')) { + $resource->setAttribute('eip-status', 'Allocating Elastic IP'); + $resource->setAttribute('eip-allocated', true); + $resource->save(); + + try { + // Allocate, assign and use a public IP address. + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'AllocateAddress', + array( + 'Domain' => 'vpc', + )) + ->resolve(); + + $public_ip = (string)$result->publicIp; + $allocation_id = (string)$result->allocationId; + + $resource->setAttribute('eip-allocation-id', $allocation_id); + $resource->setAttribute('eip-status', 'Associating Elastic IP'); + $resource->save(); + + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'AssociateAddress', + array( + 'InstanceId' => $instance_id, + 'AllocationId' => $allocation_id, + )) + ->resolve(); + + $association_id = (string)$result->associationId; + + $resource->setAttribute('eip-association-id', $association_id); + $resource->setAttribute('eip-status', 'Associated'); + $resource->save(); + + $address = $public_ip; + } catch (PhutilAWSException $ex) { + // We can't allocate an Elastic IP (probably because we've reached + // the maximum allowed on the account). Terminate the EC2 instance + // we just started and fail the resource allocation. + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'TerminateInstances', + array( + 'InstanceId.0' => $instance_id, + )) + ->resolve(); + + $resource->setAttribute( + 'aws-status', + 'Terminated'); + $resource->save(); + + throw new Exception( + 'Unable to allocate an elastic IP for the new EC2 instance. '. + 'Check your AWS account limits and ensure your limit on '. + 'elastic IP addresses is high enough to complete the '. + 'resource allocation'); + } + } else { + $resource->setAttribute('eip-allocated', false); + + // Use the private IP address. + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DescribeInstances', + array( + 'InstanceId.0' => $instance_id, + )) + ->resolve(); + + $reservation = $result->reservationSet->item[0]; + $instance = $reservation->instancesSet->item[0]; + + $address = (string)$instance->privateIpAddress; + } + + // Update address and port attributes. + $resource->setAttribute('host', $address); + if ($resource->getAttribute('protocol') === 'winrm') { + $resource->setAttribute('port', 5985); + } else { + $resource->setAttribute('port', 22); + } + $resource->save(); + + $protocol_name = ''; + if ($resource->getAttribute('protocol') === 'winrm') { + $protocol_name = 'WinRM'; + } else { + $protocol_name = 'SSH'; + } + + $this->log(pht( + 'Waiting for a successful %s connection', $protocol_name)); + + // Wait until we get a successful connection. + $command = $this->getInterface($resource, $lease, 'command'); + $command->setExecTimeout(60); + if ($resource->getAttribute('protocol') === 'ssh') { + $command->setConnectTimeout(60); + } + + $resource->setAttribute( + 'aws-status', + pht('Waiting for successful %s connection', $protocol_name)); + $resource->save(); + + while (true) { + try { + $command->getExecFuture('echo "test"')->resolvex(); + break; + } catch (Exception $ex) { + + // Make sure the instance hasn't been terminated or shutdown while + // we've been trying to connect. + $result = $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DescribeInstances', + array( + 'InstanceId.0' => $instance_id, + )) + ->resolve(); + + $reservation = $result->reservationSet->item[0]; + $instance = $reservation->instancesSet->item[0]; + $instance_state = (string)$instance->instanceState->name; + + if ($instance_state === 'shutting-down' || + $instance_state === 'terminated') { + + // Deallocate and release the public IP address if we allocated one. + if ($resource->getAttribute('eip-allocated')) { + try { + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DisassociateAddress', + array( + 'AssociationId' => + $resource->getAttribute('eip-association-id'), + )) + ->resolve(); + } catch (PhutilAWSException $ex) {} + + try { + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'ReleaseAddress', + array( + 'AllocationId' => + $resource->getAttribute('eip-allocation-id'), + )) + ->resolve(); + } catch (PhutilAWSException $ex) {} + + $resource->setAttribute( + 'eip-status', + 'Released'); + $resource->save(); + } + + $resource->setAttribute( + 'aws-status', + 'Terminated'); + $resource->save(); + + throw new Exception( + 'Allocated instance, but ended up in unexpected state \''. + $instance_state.'\'!'); + } + + continue; + } + } + + // Update the resource into open status. + $resource->setStatus(DrydockResourceStatus::STATUS_OPEN); + $resource->setAttribute( + 'aws-status', + 'Ready for Use'); + $resource->save(); + return $resource; + } + + protected function executeCloseResource(DrydockResource $resource) { + + // Deallocate and release the public IP address if we allocated one. + if ($resource->getAttribute('eip-allocated')) { + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'DisassociateAddress', + array( + 'AssociationId' => $resource->getAttribute('eip-association-id'), + )) + ->resolve(); + + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'ReleaseAddress', + array( + 'AllocationId' => $resource->getAttribute('eip-allocation-id'), + )) + ->resolve(); + } + + // Terminate the EC2 instance. + $this->getAWSEC2Future() + ->setRawAWSQuery( + 'TerminateInstances', + array( + 'InstanceId.0' => $resource->getAttribute('instance-id'), + )) + ->resolve(); + + } + + protected function executeAcquireLease( + DrydockResource $resource, + DrydockLease $lease) { + + while ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { + // This resource is still being set up by another allocator, wait until + // it is set to open. + sleep(5); + $resource->reload(); + } + + $platform = $resource->getAttribute('platform'); + $path = $resource->getAttribute('path'); + + $lease_id = $lease->getID(); + + // Can't use DIRECTORY_SEPERATOR here because that is relevant to + // the platform we're currently running on, not the platform we are + // remoting to. + $separator = '/'; + if ($platform === 'windows') { + $separator = '\\'; + } + + // Clean up the directory path a little. + $base_path = rtrim($path, '/'); + $base_path = rtrim($base_path, '\\'); + $full_path = $base_path.$separator.$lease_id; + + $cmd = $lease->getInterface('command'); + + $cmd->execx('mkdir %s', $full_path); + + $lease->setAttribute('path', $full_path); + } + + protected function executeReleaseLease( + DrydockResource $resource, + DrydockLease $lease) { + + $cmd = $lease->getInterface('command'); + $path = $lease->getAttribute('path'); + + try { + if ($resource->getAttribute('platform') !== 'windows') { + $cmd->execx('rm -rf %s', $path); + } else { + $cmd->execx('rm -Recurse -Force %s', $path); + } + } catch (Exception $ex) { + // We try to clean up, but sometimes files are locked or still in + // use (this is far more common on Windows). There's nothing we can + // do about this, so we ignore it. + } + } + + public function getType() { + return 'host'; + } + + public function getInterface( + DrydockResource $resource, + DrydockLease $lease, + $type) { + + switch ($type) { + case 'command': + if ($resource->getAttribute('protocol') === 'ssh') { + return id(new DrydockSSHCommandInterface()) + ->setConfiguration(array( + 'host' => $resource->getAttribute('host'), + 'port' => $resource->getAttribute('port'), + 'credential' => $resource->getAttribute('credential'), + 'platform' => $resource->getAttribute('platform'), + )) + ->setWorkingDirectory($lease->getAttribute('path')); + } else if ($resource->getAttribute('protocol') === 'winrm') { + return id(new DrydockWinRMCommandInterface()) + ->setConfiguration(array( + 'host' => $resource->getAttribute('host'), + 'port' => $resource->getAttribute('port'), + 'credential' => $resource->getAttribute('winrm-auth'), + 'platform' => $resource->getAttribute('platform'), + )) + ->setWorkingDirectory($lease->getAttribute('path')); + } else { + throw new Exception('Unsupported protocol for remoting'); + } + case 'filesystem': + return id(new DrydockSFTPFilesystemInterface()) + ->setConfiguration(array( + 'host' => $resource->getAttribute('host'), + 'port' => $resource->getAttribute('port'), + 'credential' => $resource->getAttribute('credential'), + )); + } + + throw new Exception("No interface of type '{$type}'."); + } + + public function getFieldSpecifications() { + return array( + 'amazon' => array( + 'name' => pht('Amazon Configuration'), + 'type' => 'header', + ), + 'region' => array( + 'name' => pht('Region'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s', 'us-west-1'), + ), + 'ami' => array( + 'name' => pht('AMI (Amazon Image)'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s', 'ami-7fd3ae4f'), + ), + 'keypair' => array( + 'name' => pht('Key Pair'), + 'type' => 'credential', + 'required' => true, + 'credential.provides' + => PassphraseSSHPrivateKeyCredentialType::PROVIDES_TYPE, + 'caption' => pht( + 'Only the public key component is transmitted to Amazon.'), + ), + 'winrm-auth' => array( + 'name' => pht('WinRM Credentials'), + 'type' => 'credential', + 'credential.provides' + => PassphrasePasswordCredentialType::PROVIDES_TYPE, + 'caption' => pht( + 'This is only required if the platform is set to "windows".'), + ), + 'size' => array( + 'name' => pht('Instance Size'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s', 't2.micro'), + ), + 'platform' => array( + 'name' => pht('Platform Name'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s or %s', 'windows', 'linux'), + ), + 'subnet-id' => array( + 'name' => pht('VPC Subnet'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s', 'subnet-2a67439b'), + ), + 'security-group-ids' => array( + 'name' => pht('VPC Security Groups'), + 'type' => 'text', + 'required' => true, + 'caption' => pht('e.g. %s', 'sg-3fa3491f,sg-bg18dea2'), + ), + 'storage-path' => array( + 'name' => pht('Storage Path'), + 'type' => 'text', + 'required' => true, + 'caption' => pht( + 'A writable location on the instance where new directories / files '. + 'can be created and data can be stored in.'), + ), + 'allocate-elastic-ip' => array( + 'name' => pht('Allocate Public IP'), + 'type' => 'bool', + 'caption' => pht( + 'If Phabricator is running in the same subnet as the allocated '. + 'machines, then you do not need to turn this option on. If '. + 'phabricator is hosted externally to Amazon EC2, then enable this '. + 'option to automatically allocate and assign elastic IP addresses '. + 'to instances so that Phabricator can SSH to them from the '. + 'internet (instances are still only accessible by SSH key pairs)'), + ), + ) + parent::getFieldSpecifications(); + } + +} diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -347,6 +347,10 @@ return true; } + public function canAllocateResourceForLease(DrydockLease $lease) { + return true; + } + abstract protected function executeAllocateResource( DrydockResource $resource, DrydockLease $lease); diff --git a/src/applications/drydock/worker/DrydockAllocatorWorker.php b/src/applications/drydock/worker/DrydockAllocatorWorker.php --- a/src/applications/drydock/worker/DrydockAllocatorWorker.php +++ b/src/applications/drydock/worker/DrydockAllocatorWorker.php @@ -177,6 +177,12 @@ unset($blueprints[$key]); continue; } + + if ($candidate_blueprint->getType() !== + $lease->getResourceType()) { + unset($blueprints[$key]); + continue; + } } $this->logToDrydock( @@ -200,6 +206,11 @@ unset($blueprints[$key]); continue; } + + if (!$candidate_blueprint->canAllocateResourceForLease($lease)) { + unset($blueprints[$key]); + continue; + } } $this->logToDrydock(