diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -709,6 +709,7 @@ 'DiffusionRepositoryRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryRemarkupRule.php', 'DiffusionRepositorySymbolsController' => 'applications/diffusion/controller/DiffusionRepositorySymbolsController.php', 'DiffusionRepositoryTag' => 'applications/diffusion/data/DiffusionRepositoryTag.php', + 'DiffusionRepositoryTestAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php', 'DiffusionRequest' => 'applications/diffusion/request/DiffusionRequest.php', 'DiffusionResolveRefsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionResolveRefsConduitAPIMethod.php', 'DiffusionResolveUserQuery' => 'applications/diffusion/query/DiffusionResolveUserQuery.php', @@ -909,6 +910,7 @@ 'DrydockSlotLock' => 'applications/drydock/storage/DrydockSlotLock.php', 'DrydockSlotLockException' => 'applications/drydock/exception/DrydockSlotLockException.php', 'DrydockSlotLockFailureLogType' => 'applications/drydock/logtype/DrydockSlotLockFailureLogType.php', + 'DrydockTestRepositoryOperation' => 'applications/drydock/operation/DrydockTestRepositoryOperation.php', 'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php', 'DrydockWorker' => 'applications/drydock/worker/DrydockWorker.php', 'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php', @@ -4465,6 +4467,7 @@ 'DiffusionRepositoryRemarkupRule' => 'PhabricatorObjectRemarkupRule', 'DiffusionRepositorySymbolsController' => 'DiffusionRepositoryEditController', 'DiffusionRepositoryTag' => 'Phobject', + 'DiffusionRepositoryTestAutomationController' => 'DiffusionRepositoryEditController', 'DiffusionRequest' => 'Phobject', 'DiffusionResolveRefsConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 'DiffusionResolveUserQuery' => 'Phobject', @@ -4705,6 +4708,7 @@ 'DrydockSlotLock' => 'DrydockDAO', 'DrydockSlotLockException' => 'Exception', 'DrydockSlotLockFailureLogType' => 'DrydockLogType', + 'DrydockTestRepositoryOperation' => 'DrydockRepositoryOperationType', 'DrydockWebrootInterface' => 'DrydockInterface', 'DrydockWorker' => 'PhabricatorWorker', 'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation', diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -103,6 +103,7 @@ 'symbol/' => 'DiffusionRepositorySymbolsController', 'staging/' => 'DiffusionRepositoryEditStagingController', 'automation/' => 'DiffusionRepositoryEditAutomationController', + 'testautomation/' => 'DiffusionRepositoryTestAutomationController', ), 'pathtree/(?P.*)' => 'DiffusionPathTreeController', 'mirror/' => array( diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php --- a/src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php @@ -4,7 +4,7 @@ extends DiffusionRepositoryEditController { protected function processDiffusionRequest(AphrontRequest $request) { - $viewer = $request->getUser(); + $viewer = $this->getViewer(); $drequest = $this->diffusionRequest; $repository = $drequest->getRepository(); diff --git a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php --- a/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php +++ b/src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php @@ -688,6 +688,19 @@ $this->getRepositoryControllerURI($repository, 'edit/automation/')); $view->addAction($edit); + $can_test = $repository->canPerformAutomation(); + + $test = id(new PhabricatorActionView()) + ->setIcon('fa-gamepad') + ->setName(pht('Test Configuration')) + ->setWorkflow(true) + ->setDisabled(!$can_test) + ->setHref( + $this->getRepositoryControllerURI( + $repository, + 'edit/testautomation/')); + $view->addAction($test); + return $view; } diff --git a/src/applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php b/src/applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php new file mode 100644 --- /dev/null +++ b/src/applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php @@ -0,0 +1,78 @@ +getViewer(); + $drequest = $this->diffusionRequest; + $repository = $drequest->getRepository(); + + $repository = id(new PhabricatorRepositoryQuery()) + ->setViewer($viewer) + ->requireCapabilities( + array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + )) + ->withIDs(array($repository->getID())) + ->executeOne(); + if (!$repository) { + return new Aphront404Response(); + } + + $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/'); + + if (!$repository->canPerformAutomation()) { + return $this->newDialog() + ->setTitle(pht('Automation Not Configured')) + ->appendParagraph( + pht( + 'You can not run a configuration test for this repository '. + 'because you have not configured repository automation yet. '. + 'Configure it first, then test the configuration.')) + ->addCancelButton($edit_uri); + } + + if ($request->isFormPost()) { + $op = new DrydockTestRepositoryOperation(); + + $operation = DrydockRepositoryOperation::initializeNewOperation($op) + ->setAuthorPHID($viewer->getPHID()) + ->setObjectPHID($repository->getPHID()) + ->setRepositoryPHID($repository->getPHID()) + ->setRepositoryTarget('none:') + ->save(); + + $operation->scheduleUpdate(); + + $operation_id = $operation->getID(); + $operation_uri = "/drydock/operation/{$operation_id}/"; + + return id(new AphrontRedirectResponse()) + ->setURI($operation_uri); + } + + return $this->newDialog() + ->setTitle(pht('Test Automation Configuration')) + ->appendParagraph( + pht( + 'This configuration test will build a working copy of the '. + 'repository and perform some basic validation. If it works, '. + 'your configuration is substantially correct.')) + ->appendParagraph( + pht( + 'The test will not perform any writes against the repository, so '. + 'write operations may still fail even if the test passes. This '. + 'test covers building and reading working copies, but not writing '. + 'to them.')) + ->appendParagraph( + pht( + 'If you run into write failures despite passing this test, '. + 'it suggests that your setup is nearly correct but authentication '. + 'is probably not fully configured.')) + ->addCancelButton($edit_uri) + ->addSubmitButton(pht('Start Test')); + } + +} diff --git a/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php b/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php --- a/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php +++ b/src/applications/drydock/controller/DrydockRepositoryOperationViewController.php @@ -46,10 +46,15 @@ ->setHeader($header) ->addPropertyList($properties); + $status_view = id(new DrydockRepositoryOperationStatusView()) + ->setUser($viewer) + ->setOperation($operation); + return $this->buildApplicationPage( array( $crumbs, $object_box, + $status_view, ), array( 'title' => $title, diff --git a/src/applications/drydock/operation/DrydockTestRepositoryOperation.php b/src/applications/drydock/operation/DrydockTestRepositoryOperation.php new file mode 100644 --- /dev/null +++ b/src/applications/drydock/operation/DrydockTestRepositoryOperation.php @@ -0,0 +1,53 @@ +getRepository(); + switch ($operation->getOperationState()) { + case DrydockRepositoryOperation::STATE_WAIT: + return pht( + 'Waiting to test configuration for %s...', + $repository->getMonogram()); + case DrydockRepositoryOperation::STATE_WORK: + return pht( + 'Testing configuration for %s. This may take a moment if Drydock '. + 'has to clone the repository for the first time.', + $repository->getMonogram()); + case DrydockRepositoryOperation::STATE_DONE: + return pht( + 'Success! Automation is configured properly and Drydock can '. + 'operate on %s.', + $repository->getMonogram()); + } + } + + public function applyOperation( + DrydockRepositoryOperation $operation, + DrydockInterface $interface) { + $repository = $operation->getRepository(); + + if ($repository->isGit()) { + $interface->execx('git status'); + } else if ($repository->isHg()) { + $interface->execx('hg status'); + } else if ($repository->isSVN()) { + $interface->execx('svn status'); + } else { + throw new PhutilMethodNotImplementedException(); + } + } + +} diff --git a/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php b/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php --- a/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php @@ -165,6 +165,9 @@ 'branch' => $name, ); break; + case 'none': + $spec = array(); + break; default: throw new Exception( pht( diff --git a/src/docs/user/userguide/differential_land.diviner b/src/docs/user/userguide/differential_land.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/userguide/differential_land.diviner @@ -0,0 +1,53 @@ +@title Differential User Guide: Automated Landing +@group userguide + +Configuring Phabricator so you can "Land Revision" from the web UI. + + +Overview +======== + +IMPORTANT: This feature is a prototype and has substantial limitations. + +Phabricator can be configured so that approved revisions may be published +directly from the web interface. This can make publishing changes more +convenient, particularly for open source projects where authors may not have +commit access to the repository. This document explains the workflow and how to +configure it. + +When properly configured, a {nav Land Revision} action will appear in +Differential. This action works like `arc land` on the command line, and +merges and publishes the revision. + +This feature has significant limitations: + + - This feature is a prototype. + - This feature is only supported in Git. + - This feature always lands changes onto `master`. + - This feature does not currently provide chain of custody, and what lands + may be arbitrarily different than what is shown in Differential. + +To be landable, a revision must satisfy these requirements: + + - It must belong to a repository which is tracked in Diffusion + (both hosted and imported repositories will work). + - The repository must have a **Staging Area** configured. + - The repository must have **Repository Automation** configured. For + details, see @{article:Drydock User Guide: Repository Automation}. + - The revision must have been created with `arc diff` and pushed to the + configured staging area at creation time. + - The user clicking the "Land Revision" button must have permission to push + to the repository. + +If these requirements are met, the {nav Land Revision} action should be +available in the UI. + + +Next Steps +========== + +Continue by: + + - configuring repository automation with + @{article:Drydock User Guide: Repository Automation}; or + - returning to the @{article:Differential User Guide}. diff --git a/src/docs/user/userguide/drydock.diviner b/src/docs/user/userguide/drydock.diviner --- a/src/docs/user/userguide/drydock.diviner +++ b/src/docs/user/userguide/drydock.diviner @@ -15,16 +15,19 @@ you will configure Drydock to enable capabilities in other applications: - Harbormaster can use Drydock to host builds. - - In the future, Differential will be able to use Drydock to perform - server-side merges. + - Differential can use Drydock to perform server-side merges. Users will not normally interact with Drydock directly. +If you want to get started with Drydock right away, see +@{article:Drydock User Guide: Quick Start} for specific instructions on +configuring integrations. + What Drydock Does ================= -Drydock manages working copies, build hosts, and other software and hardware +Drydock manages working copies, hosts, and other software and hardware resources that build and deployment processes may require in order to perform useful work. @@ -49,15 +52,202 @@ Drydock solves these scaling problems by providing a central allocation framework for //resources//, which are physical or virtual resources like a -build host or a working copy. Processes which need to share hardware or -software can use Drydock to coordinate creation, access, and destruction of -those resources. +host or a working copy. Processes which need to share hardware or software can +use Drydock to coordinate creation, access, and destruction of those resources. Applications ask Drydock for resources matching a description, and it allocates a corresponding resource by either finding a suitable unused resource or creating a new resource. When work completes, the resource is returned to the resource pool or destroyed. + +Getting Started with Drydock +============================ + +In general, you will interact with Drydock by configuring blueprints, which +tell Drydock how to build resources. You can jump into this topic directly +in @{article:Drydock Blueprints}. + +For help on configuring specific application features: + + - to configure server-side merges from Differential, see + @{article:Differential User Guide: Automated Landing}. + +You should also understand the Drydock security model before deploying it +in a production environment. See @{article:Drydock User Guide: Security}. + +The remainder of this document has some additional high-level discussion about +how Drydock works and why it works that way, which may be helpful in +understanding the application as a whole. + + +Drydock Concepts +================ + +The major concepts in Drydock are **Blueprints**, **Resources**, **Leases**, +and the **Allocator**. + +**Blueprints** are configuration that tells Drydock how to create resources: +where it can put them, how to access them, how many it can make at once, who is +allowed to ask for access to them, how to actually build them, how to clean +them up when they are no longer in use, and so on. + +Drydock starts without any blueprints. You'll add blueprints to configure +Drydock and enable it to satisfy requests for resources. You can learn more +about blueprints in @{article:Drydock Blueprints}. + +**Resources** represent things (like hosts or working copies) that Drydock has +created, is managing the lifecycle for, and can give other applications access +to. + +**Leases** are requests for resources with certain qualities by other +applications. For example, Harbormaster may request a working copy of a +particular repository so it can run unit tests. + +The **Allocator** is where Drydock actually does work. It works roughly like +this: + + - An application creates a lease describing a resource it needs, and + uses this lease to ask Drydock for an appropriate resource. + - Drydock looks at free resources to try to find one it can use to satisfy + the request. If it finds one, it marks the resource as in use and gives + the application details about how to access it. + - If it can't find an appropriate resource that already exists, it looks at + the blueprints it has configured to try to build one. If it can, it creates + a new resource, then gives the application access to it. + - Once the application finishes using the resource, it frees it. Depending + on configuration, Drydock may reuse it, destroy it, or hold onto it and + make a decision later. + +Some minor concepts in Drydock are **Slot Locks** and **Repository Operations**. + +**Slot Locks** are simple optimistic locks that most Drydock blueprints use to +avoid race conditions. Their design is not particularly interesting or novel, +they're just a fairly good fit for most of the locking problems that Drydock +blueprints tend to encounter and Drydock provides APIs to make them easy to +work with. + +**Repository Operations** help other applications coordinate writes to +repositories. Multiple applications perform similar kinds of writes, and these +writes require more sequencing/coordination and user feedback than other +operations. + + +Architecture Overview +===================== + +This section describes some of Drydock's design goals and architectural +choices, so you can understand its strengths and weaknesses and which problem +domains it is well or poorly suited for. + +A typical use case for Drydock is giving another application access to a +working copy in order to run a build or unit test operation. Drydock can +satisfy the request and resume execution of application code in 1-2 seconds +under reasonable conditions and with moderate tradeoffs, and can satisfy a +large number of these requests in parallel. + +**Scalable**: Drydock is designed to scale easily to something in the realm of +thousands of hosts in hundreds of pools, and far beyond that with a little +work. + +Drydock is intended to solve resource management problems at very large scales +and minimzes blocking operations, locks, and artificial sequencing. Drydock is +designed to fully utilize an almost arbitrarily large pool of resources and +improve performance roughly linearly with available hardware. + +Because the application assumes that deployment at this scale and complexity +level is typical, you may need to configure more things and do more work than +you would under the simplifying assumptions of small scale. + +**Heavy Resources**: Drydock assumes that resources are relatively +heavyweight and and require a meaningful amount (a second or more) of work to +build, maintain and tear down. It also assumes that leases will often have +substantial lifespans (seconds or minutes) while performing operations. + +Resources like working copies (which typically take several seconds to create +with a command like `git clone`) and VMs (which typically take several seconds +to spin up) are good fits for Drydock and for the problems it is intended to +solve. + +Lease operations like running unit tests, performing builds, executing merges, +generating documentation and running temporary services (which typically last +at least a few seconds) are also good fits for Drydock. + +In both cases, the general concern with lightweight resources and operations is +that Drydock operation overhead is roughly on the order of a second for many +tasks, so overhead from Drydock will be substantial if resources are built and +torn down in a few milliseconds or lease operations require only a fraction of +a second to execute. + +As a rule of thumb, Drydock may be a poor fit for a problem if operations +typically take less than a second to build, execute, and destroy. + +**Focus on Resource Construction**: Drydock is primarily solving a resource +construction problem: something needs a resource matching some description, so +Drydock finds or builds that resource as quickly as possible. + +Drydock generally prioritizes responding to requests quickly over other +concerns, like minimizing waste or performing complex scheduling. Although you +can make adjustments to some of these behaviors, it generally assumes that +resources are cheap compared to the cost of waiting for resource construction. + +This isn't to say that Drydock is grossly wasteful or has a terrible scheduler, +just that efficient utilization and efficient scheduling aren't the primary +problems the design focuses on. + +This prioritization corresponds to scenarios where resources are something like +hosts or working copies, and operations are something like builds, and the cost +of hosts and storage is small compared to the cost of engineer time spent +waiting on jobs to get scheduled. + +Drydock may be a weak fit for a problem if it is bounded by resource +availability and using resources as efficiently as possible is very important. +Drydock generally assumes you will respond to a resource deficit by making more +resources available (usually very cheap), rather than by paying engineers to +wait for operations to complete (usually very expensive). + +**Isolation Tradeoffs**: Drydock assumes that multiple operations running at +similar levels of trust may be interested in reducing isolation to improve +performance, reduce complexity, or satisfy some other similar goal. It does not +guarantee isolation and assumes most operations will not run in total isolation. + +If this isn't true for your use case, you'll need to be careful in configuring +Drydock to make sure that operations are fully isolated and can not interact. +Complete isolation will reduce the performance of the allocator as it will +generally prevent it from reusing resources, which is one of the major ways it +can improve performance. + +You can find more discussion of these tradeoffs in +@{article:Drydock User Guide: Security}. + +**Agentless**: Drydock does not require an agent or daemon to be installed on +hosts. It interacts with hosts over SSH. + +**Very Abstract**: Drydock's design is //extremely// abstract. Resources have +very little hardcoded behavior. The allocator has essentially zero specialized +knowledge about what it is actually doing. + +One aspect of this abstractness is that Drydock is composable, and solves +complex allocation problems by //asking itself// to build the pieces it needs. +To build a working copy, Drydock first asks itself for a suitable host. It +solves this allocation sub-problem, then resolves the original request. + +This allows new types of resources to build on Drydock's existing knowledge of +resource construction by just saying "build one of these other things you +already know how to build, then apply a few adjustments". This also means that +you can tell Drydock about a new way to build hosts (say, bring up VMs from a +different service provider) and the rest of the pipeline can use these new +hosts interchangeably with the old hosts. + +While this design theoretically makes Drydock more powerful and more flexible +than a less abstract approach, abstraction is frequently a double-edged sword. + +Drydock is almost certainly at the extreme upper end of abstraction for tools +in this space, and the level of abstraction may ultimately match poorly with a +particular problem domain. Alternative approaches may give you more specialized +and useful tools for approaching a given problem. + + Next Steps ========== @@ -65,5 +255,6 @@ - understanding Drydock security concerns with @{article:Drydock User Guide: Security}; or + - learning about blueprints in @{article:Drydock Blueprints}; or - allowing Phabricator to write to repositories with @{article:Drydock User Guide: Repository Automation}. diff --git a/src/docs/user/userguide/drydock_blueprints.diviner b/src/docs/user/userguide/drydock_blueprints.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/userguide/drydock_blueprints.diviner @@ -0,0 +1,80 @@ +@title Drydock Blueprints +@group userguide + +Overview of Drydock blueprint types. + + +Overview +======== + +IMPORTANT: Drydock is not a mature application and may be difficult to +configure and use for now. + +Drydock builds and manages various hardware and software resources, like +hosts and repository working copies. Other applications can use these resources +to perform useful work (like running tests or builds). + +For additional disussion of Drydock, see @{article:Drydock User Guide}. + +Drydock can't create any resources until you configure it. You'll configure +Drydock by creating **Blueprints**. Each blueprint tells Drydock how to build +a specific kind of resource, how many it is allowed to build, where it should +build them, who is authorized to request them, and so on. + +You can create a new blueprint in Drydock from the web UI: + +{nav Drydock > Blueprints > New Blueprint} + +Each blueprint builds resources of a specific type, like hosts or repository +working copies. Detailed topic guides are available for each resource type: + +**Hosts**: Hosts are the building block for most other resources. For details, +see @{article:Drydock Blueprints: Hosts}. + +**Working Copies**: Working copies allow Drydock to perform repository +operations like running tests, performing builds, and handling merges. + + +Authorizing Access +================== + +Before objects in other applications can use a blueprint, the blueprint must +authorize them. + +This mostly serves to prevent users with limited access from executing +operations on trusted hosts. For additional discussion, see +@{article:Drydock User Guide: Security}. + +This also broadly prevents Drydock from surprising you by coming up with a +valid but unintended solution to an allocation problem which runs some +operation on resources that are techincally suitable but not desirable. For +example, you may not want your Android builds running on your iPhone build +tier, even if there's no technical reason they can't. + +You can review active authorizations and pending authorization requests in +the "Active Authorizations" section of the blueprint detail screen. + +To approve an authorization, click it and select {nav Approve Authorization}. +Until you do, the requesting object won't be able to access resources from +the blueprint. + +You can also decline an authorization. This prevents use of resources and +removes it from the authorization approval queue. + + +Disabling Blueprints +==================== + +You can disable a blueprint by selecting {nav Disable Blueprint} from the +blueprint detail screen. + +Disabled blueprints will no longer be used for new allocations. However, +existing resources will continue to function. + + +Next Steps +========== + +Continue by: + + - returning to the @{article:Drydock User Guide}. diff --git a/src/docs/user/userguide/drydock_hosts.diviner b/src/docs/user/userguide/drydock_hosts.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/userguide/drydock_hosts.diviner @@ -0,0 +1,126 @@ +@title Drydock Blueprints: Hosts +@group userguide + +Guide to configuring Drydock host blueprints. + + +Overview +======== + +IMPORTANT: Drydock is not a mature application and may be difficult to +configure and use for now. + +To give Drydock access to machines so it can perform work, you'll configure +**host blueprints**. These blueprints tell Drydock where to find machines (or +how to build machines) and how to connect to them. + +Once Drydock has access to hosts it can use them to build more interesting and +complex types of resources, like repository working copies. + +Drydock currently supports these kinds of host blueprints: + + - **Almanac Hosts**: Gives Drydock access to a predefined list of hosts. + +Drydock may support additional blueprints in the future. + + +Security +======== + +Drydock can be used to run semi-trusted and untrusted code, and you may want +to isolate specific processes or classes of processes from one another. See +@{article:Drydock User Guide: Security} for discussion of security +concerns and guidance on how to make isolation tradeoffs. + + +General Considerations +====================== + +**You must install software on hosts.** Drydock does not currently handle +installing software on hosts. You'll need to make sure any hosts are configured +properly with any software you need, and have tools like `git`, `hg` or `svn` +that may be required to interact with working copies. + +You do **not** need to install PHP, arcanist, libphutil or Phabricator on the +hosts unless you are specifically running `arc` commands. + +**You must configure authentication.** Drydock also does not handle credentials +for VCS operations. If you're interacting with repositories hosted on +Phabricator, the simplest way to set this up is something like this: + + - Create a new bot user in Phabricator. + - In {nav Settings > SSH Public Keys}, add a public key or generate a + keypair. + - Put the private key on your build hosts as `~/.ssh/id_rsa` for whatever + user you're connecting with. + +This will let processes on the host access Phabricator as the bot user, and +use the bot user's permissions to pull and push changes. + +If you're using hosted repositories from an external service, you can follow +similar steps for that service. + +Note that any processes running under the given user account will have access +to the private key, so you should give the bot the smallest acceptable level of +permissions if you're running semi-trusted or untrusted code like unit tests. + +**You must create a `/var/drydock` directory.** This is hard-coded in Drydock +for now, so you need to create it on the hosts. This can be a symlink to +a different location if you prefer. + + +Almanac Hosts +============= + +The **Almanac Hosts** blueprint type gives Drydock access to a predefined list +of hosts which you configure in the Almanac application. This is the simplest +type of blueprint to set up. + +For more information about Almanac, see @{article:Almanac User Guide}. + +For example, suppose you have `build001.mycompany.com` and +`build002.mycompany.com`, and want to configure Drydock to be able to use these +hosts. To do this: + +**Create Almanac Devices**: Create a device record in Almanac for each your +hosts. + +{nav Almanac > Devices > Create Device} + +Enter the device names (like `build001.mycompany.com`). After creating the +devices, use {nav Add Interface} to configure the ports and IP addresses that +Drydock should connect to over SSH (normally, this is port `22`). + +**Create an Almanac Service**: In the Almanac application, create a new service +to define the pool of devices you want to use. + +{nav Almanac > Services > Create Service} + +Choose the service type **Drydock: Resource Pool**. This will allow Drydock +to use the devices that are bound to the service. + +Now, use {nav Add Binding} to bind all of the devices to the service. + +You can add more hosts to the pool later by binding additional devices, and +Drydock will automatically start using them. Likewise, you can remove bindings +to take hosts out of service. + +**Create a Drydock Blueprint**: Now, create a new blueprint in Drydock. + +{nav Drydock > Blueprints > New Blueprint} + +Choose the **Almanac Hosts** blueprint type. + +In **Almanac Services**, select the service you previously created. For +**Credentials**, select an SSH private key you want Drydock to use to connect +to the hosts. + +Drydock should now be able to build resources from these hosts. + + +Next Steps +========== + +Continue by: + + - returning to @{article:Drydock Blueprints}. diff --git a/src/docs/user/userguide/drydock_quick_start.diviner b/src/docs/user/userguide/drydock_quick_start.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/userguide/drydock_quick_start.diviner @@ -0,0 +1,74 @@ +@title Drydock User Guide: Quick Start +@group userguide + +Guide to getting Drydock + +Quick Start: Land Revisions +=========================== + +Quick start guide to getting "Land Revision" working in Differential. For +a more detailed guide, see @{article:Drydock User Guide: Repository Automation}. + +Choose a repository you want to enable "Land Revision" for. We'll call this +**Repository X**. + +You need to configure a staging area for this repository if you haven't +already. You can do this in Diffusion in {nav Edit Repository > Edit Staging}. +We'll call this **Staging Area Y**. + +Choose or create a host you want to run merges on. We'll call this +`automation001`. For example, you might bring up a new host in EC2 and +label it `automation001.mycompany.com`. You can use an existing host if you +prefer. + +Create a user account on the host, or choose an existing user account. This is +the user that merges will execute under: Drydock will connect to it and run a +bunch of `git` commands, then ultimately run `git push`. We'll call this user +`builder`. + +Install `git`, `hg` or `svn` if you haven't already and set up private keys +for `builder` so it can pull and push any repositories you want to operate +on. + +If your repository and/or staging area are hosted in Phabricator, you may want +to create a corresponding bot account so you can add keys and give it +permissions. + +At this point you should be able to `ssh builder@automation001` to connect to +the host, and get a normal shell. You should be able to `git clone ...` from +**Repository X** and from **Staging Area Y**, and `git push` to **Repository +X**. If you can't, configure things so you can. + +Now, create a host blueprint for the host. You can find a more detailed +walkthrough in @{article:Drydock Blueprints: Hosts}. Briefly: + + - Create an Almanac device for the host. This should have the IP address and + port for your host. + - Create an Almanac service bound to the device. This should be a Drydock + resource pool service and have a binding to the IP from the previous step. + - Create a Drydock host blueprint which uses the service from the previous + step. It should be configured with an SSH private key that can be used + to connect to `builder@automation001`. + +Then, create a new working copy blueprint which uses the host blueprint you +just made. You can find a more detailed walkthrough in @{article:Drydock +Blueprints: Working Copies}. Authorize the working copy blueprint to use the +host blueprint. + +Finally, configure repository automation for **Repository X**: +{nav Edit Repository > Edit Automation}. Provide the working copy blueprint +from the previous step. Authorize the repository to use the working copy +blueprint. + +After you save changes, click {nav Test Configuration} to test that things +are working properly. + +The "Land Revision" action should now be available on revisions for this +repository. + +Next Steps +========== + +Continue by: + + - returning to @{article:Drydock User Guide}. diff --git a/src/docs/user/userguide/drydock_repository_automation.diviner b/src/docs/user/userguide/drydock_repository_automation.diviner --- a/src/docs/user/userguide/drydock_repository_automation.diviner +++ b/src/docs/user/userguide/drydock_repository_automation.diviner @@ -7,12 +7,19 @@ Overview ======== -IMPORTANT: This feature is very new and most of the capabilities described +IMPORTANT: This feature is very new and some of the capabilities described in this document are not yet available. This feature as a whole is a prototype. By configuring Drydock and Diffusion appropriately, you can enable **Repository -Automation** for a repository. Once automation is set up, Phabricator will be -able to make changes to the repository. +Automation** for a repository. This will allow Phabricator to make changes +to the repository. + + +Limitations +=========== + + - This feature is a prototype. + - Only Git is supported. Security @@ -29,6 +36,45 @@ @{article:Drydock User Guide: Security}. +Configuring Automation +====================== + +To configure automation, use {nav Edit Repository > Edit Automation} from +Diffusion. + +On the configuration screen, specify one or more working copy blueprints in +Drydock (usually, you'll just use one). Repository automation will use working +copies built by these blueprints to perform merges and push changes. + +For more details on configuring these blueprints, see +@{article:Drydock Blueprints: Working Copies}. + +After selecting one or more blueprints, make sure you authorize the repository +to use them. Automation operations won't be able to proceed until you do. The +UI will remind you if you have unauthorized blueprints selected. + + +Testing Configuration +===================== + +Once the blueprints are configured and authorized, use {nav Test Configuration} +to check that things are configured correctly. This will build a working copy +in Drydock, connect to it, and run a trivial command (like `git status`) to +make sure things work. + +If it's the first time you're doing this, it may take a few moments since it +will need to clone a fresh working copy. + +If the test is successful, your configuration is generally in good shape. If +not, it should give you more details about what went wrong. + +Since the test doesn't actually do a push, it's possible that you may have +everything configured properly //except// write access. In this case, you'll +run into a permission error when you try to actually perform a merge or other +similar write. If you do, adjust permissions or credentials appropriately so +the working copy can be pushed from. + + Next Steps ========== diff --git a/src/docs/user/userguide/drydock_working_copies.diviner b/src/docs/user/userguide/drydock_working_copies.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/userguide/drydock_working_copies.diviner @@ -0,0 +1,39 @@ +@title Drydock Blueprints: Working Copies +@group userguide + +Guide to configuring Drydock working copy blueprints. + + +Overview +======== + +IMPORTANT: Drydock is not a mature application and may be difficult to +configure and use for now. + +To let Drydock build repository working copies in order to run unit tests and +other similar operations, you'll configure **working copy blueprints**. + +Working Copies +============== + +Working copy blueprints rely on host blueprints, so you'll need to configure +a suitable host blueprint first. See @{article:Drydock Blueprints: Hosts}. + +To configure a working copy blueprint, choose the host blueprints it should +use in **Use Blueprints**. + +You can optionally specify a **Limit**. If you do, the blueprint won't be +allowed to create more than this many simultaneous resources. If you leave +it empty, the blueprint will be able to create an unlimited number of +resources. + +After you save the blueprint, make sure you authorize it to use the selected +host blueprints. It won't be able to acquire host resources until you do. + + +Next Steps +========== + +Continue by: + + - returning to @{article:Drydock Blueprints}.