diff --git a/src/applications/harbormaster/conduit/HarbormasterSendMessageConduitAPIMethod.php b/src/applications/harbormaster/conduit/HarbormasterSendMessageConduitAPIMethod.php index dbf1f18cfa..fc83b7ab42 100644 --- a/src/applications/harbormaster/conduit/HarbormasterSendMessageConduitAPIMethod.php +++ b/src/applications/harbormaster/conduit/HarbormasterSendMessageConduitAPIMethod.php @@ -1,273 +1,273 @@ $parameter) { $type = idx($parameter, 'type'); $type = str_replace('|', ' '.pht('or').' ', $type); $description = idx($parameter, 'description'); $rows[] = "| `{$key}` | //{$type}// | {$description} |"; } $unit_table = implode("\n", $rows); $rows = array(); $rows[] = "| {$head_key} | {$head_name} | {$head_desc} |"; $rows[] = '|-------------|--------------|--------------|'; $results = ArcanistUnitTestResult::getAllResultCodes(); foreach ($results as $result_code) { $name = ArcanistUnitTestResult::getResultCodeName($result_code); $description = ArcanistUnitTestResult::getResultCodeDescription( $result_code); $rows[] = "| `{$result_code}` | **{$name}** | {$description} |"; } $result_table = implode("\n", $rows); $rows = array(); $rows[] = "| {$head_key} | {$head_type} | {$head_desc} |"; $rows[] = '|-------------|--------------|--------------|'; $lint_spec = HarbormasterBuildLintMessage::getParameterSpec(); foreach ($lint_spec as $key => $parameter) { $type = idx($parameter, 'type'); $type = str_replace('|', ' '.pht('or').' ', $type); $description = idx($parameter, 'description'); $rows[] = "| `{$key}` | //{$type}// | {$description} |"; } $lint_table = implode("\n", $rows); $rows = array(); $rows[] = "| {$head_key} | {$head_name} |"; $rows[] = '|-------------|--------------|'; $severities = ArcanistLintSeverity::getLintSeverities(); foreach ($severities as $key => $name) { $rows[] = "| `{$key}` | **{$name}** |"; } $severity_table = implode("\n", $rows); $valid_unit = array( array( 'name' => 'PassingTest', 'result' => ArcanistUnitTestResult::RESULT_PASS, ), array( 'name' => 'FailingTest', 'result' => ArcanistUnitTestResult::RESULT_FAIL, ), ); $valid_lint = array( array( 'name' => pht('Syntax Error'), 'code' => 'EXAMPLE1', 'severity' => ArcanistLintSeverity::SEVERITY_ERROR, 'path' => 'path/to/example.c', 'line' => 17, 'char' => 3, ), array( 'name' => pht('Not A Haiku'), 'code' => 'EXAMPLE2', 'severity' => ArcanistLintSeverity::SEVERITY_ERROR, 'path' => 'path/to/source.cpp', 'line' => 23, 'char' => 1, 'description' => pht( 'This function definition is not a haiku.'), ), ); $json = new PhutilJSON(); $valid_unit = $json->encodeAsList($valid_unit); $valid_lint = $json->encodeAsList($valid_lint); return pht( "Send a message about the status of a build target to Harbormaster, ". "notifying the application of build results in an external system.". "\n\n". "Sending Messages\n". "================\n". "If you run external builds, you can use this method to publish build ". "results back into Harbormaster after the external system finishes work ". "or as it makes progress.". "\n\n". "The simplest way to use this method is to call it once after the ". "build finishes with a `pass` or `fail` message. This will record the ". "build result, and continue the next step in the build if the build was ". "waiting for a result.". "\n\n". "When you send a status message about a build target, you can ". "optionally include detailed `lint` or `unit` results alongside the ". "message. See below for details.". "\n\n". "If you want to report intermediate results but a build hasn't ". "completed yet, you can use the `work` message. This message doesn't ". "have any direct effects, but allows you to send additional data to ". "update the progress of the build target. The target will continue ". "waiting for a completion message, but the UI will update to show the ". "progress which has been made.". "\n\n". "Message Types\n". "=============\n". "When you send Harbormaster a message, you must include a `type`, ". "which describes the overall state of the build. For example, use ". "`pass` to tell Harbormaster that a build completed successfully.". "\n\n". "Supported message types are:". "\n\n". "%s". "\n\n". "Unit Results\n". "============\n". "You can report test results alongside a message. The simplest way to ". "do this is to report all the results alongside a `pass` or `fail` ". "message, but you can also send a `work` message to report intermediate ". "results.\n\n". "To provide unit test results, pass a list of results in the `unit` ". "parameter. Each result should be a dictionary with these keys:". "\n\n". "%s". "\n\n". "The `result` parameter recognizes these test results:". "\n\n". "%s". "\n\n". "This is a simple, valid value for the `unit` parameter. It reports ". "one passing test and one failing test:\n\n". "\n\n". "```lang=json\n". "%s". "```". "\n\n". "Lint Results\n". "============\n". "Like unit test results, you can report lint results alongside a ". "message. The `lint` parameter should contain results as a list of ". "dictionaries with these keys:". "\n\n". "%s". "\n\n". "The `severity` parameter recognizes these severity levels:". "\n\n". "%s". "\n\n". "This is a simple, valid value for the `lint` parameter. It reports one ". "error and one warning:". "\n\n". "```lang=json\n". "%s". "```". "\n\n", $message_table, $unit_table, $result_table, $valid_unit, $lint_table, $severity_table, $valid_lint); } protected function defineParamTypes() { $messages = HarbormasterMessageType::getAllMessages(); $type_const = $this->formatStringConstants($messages); return array( 'buildTargetPHID' => 'required phid', 'type' => 'required '.$type_const, 'unit' => 'optional list', 'lint' => 'optional list', ); } protected function defineReturnType() { return 'void'; } protected function execute(ConduitAPIRequest $request) { $viewer = $request->getUser(); $build_target_phid = $request->getValue('buildTargetPHID'); $message_type = $request->getValue('type'); $build_target = id(new HarbormasterBuildTargetQuery()) ->setViewer($viewer) ->withPHIDs(array($build_target_phid)) ->executeOne(); if (!$build_target) { throw new Exception(pht('No such build target!')); } $save = array(); $lint_messages = $request->getValue('lint', array()); foreach ($lint_messages as $lint) { $save[] = HarbormasterBuildLintMessage::newFromDictionary( $build_target, $lint); } $unit_messages = $request->getValue('unit', array()); foreach ($unit_messages as $unit) { $save[] = HarbormasterBuildUnitMessage::newFromDictionary( $build_target, $unit); } $save[] = HarbormasterBuildMessage::initializeNewMessage($viewer) - ->setBuildTargetPHID($build_target->getPHID()) + ->setReceiverPHID($build_target->getPHID()) ->setType($message_type); $build_target->openTransaction(); foreach ($save as $object) { $object->save(); } $build_target->saveTransaction(); // If the build has completely paused because all steps are blocked on // waiting targets, this will resume it. $build = $build_target->getBuild(); PhabricatorWorker::scheduleTask( 'HarbormasterBuildWorker', array( 'buildID' => $build->getID(), ), array( 'objectPHID' => $build->getPHID(), )); return null; } } diff --git a/src/applications/harbormaster/controller/HarbormasterBuildViewController.php b/src/applications/harbormaster/controller/HarbormasterBuildViewController.php index 5b7171a200..4507139d47 100644 --- a/src/applications/harbormaster/controller/HarbormasterBuildViewController.php +++ b/src/applications/harbormaster/controller/HarbormasterBuildViewController.php @@ -1,667 +1,667 @@ getRequest(); $viewer = $request->getUser(); $id = $request->getURIData('id'); $generation = $request->getInt('g'); $build = id(new HarbormasterBuildQuery()) ->setViewer($viewer) ->withIDs(array($id)) ->executeOne(); if (!$build) { return new Aphront404Response(); } require_celerity_resource('harbormaster-css'); $title = pht('Build %d', $id); $page_header = id(new PHUIHeaderView()) ->setHeader($title) ->setUser($viewer) ->setPolicyObject($build) ->setHeaderIcon('fa-cubes'); if ($build->isRestarting()) { $page_header->setStatus( 'fa-exclamation-triangle', 'red', pht('Restarting')); } else if ($build->isPausing()) { $page_header->setStatus( 'fa-exclamation-triangle', 'red', pht('Pausing')); } else if ($build->isResuming()) { $page_header->setStatus( 'fa-exclamation-triangle', 'red', pht('Resuming')); } else if ($build->isAborting()) { $page_header->setStatus( 'fa-exclamation-triangle', 'red', pht('Aborting')); } $curtain = $this->buildCurtainView($build); $properties = $this->buildPropertyList($build); $crumbs = $this->buildApplicationCrumbs(); $this->addBuildableCrumb($crumbs, $build->getBuildable()); $crumbs->addTextCrumb($title); $crumbs->setBorder(true); if ($generation === null || $generation > $build->getBuildGeneration() || $generation < 0) { $generation = $build->getBuildGeneration(); } $build_targets = id(new HarbormasterBuildTargetQuery()) ->setViewer($viewer) ->needBuildSteps(true) ->withBuildPHIDs(array($build->getPHID())) ->withBuildGenerations(array($generation)) ->execute(); if ($build_targets) { $messages = id(new HarbormasterBuildMessageQuery()) ->setViewer($viewer) - ->withBuildTargetPHIDs(mpull($build_targets, 'getPHID')) + ->withReceiverPHIDs(mpull($build_targets, 'getPHID')) ->execute(); - $messages = mgroup($messages, 'getBuildTargetPHID'); + $messages = mgroup($messages, 'getReceiverPHID'); } else { $messages = array(); } if ($build_targets) { $artifacts = id(new HarbormasterBuildArtifactQuery()) ->setViewer($viewer) ->withBuildTargetPHIDs(mpull($build_targets, 'getPHID')) ->execute(); $artifacts = msort($artifacts, 'getArtifactKey'); $artifacts = mgroup($artifacts, 'getBuildTargetPHID'); } else { $artifacts = array(); } $targets = array(); foreach ($build_targets as $build_target) { $header = id(new PHUIHeaderView()) ->setHeader($build_target->getName()) ->setUser($viewer) ->setHeaderIcon('fa-bullseye'); $target_box = id(new PHUIObjectBoxView()) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setHeader($header); $tab_group = new PHUITabGroupView(); $target_box->addTabGroup($tab_group); $property_list = new PHUIPropertyListView(); $target_artifacts = idx($artifacts, $build_target->getPHID(), array()); $links = array(); $type_uri = HarbormasterURIArtifact::ARTIFACTCONST; foreach ($target_artifacts as $artifact) { if ($artifact->getArtifactType() == $type_uri) { $impl = $artifact->getArtifactImplementation(); if ($impl->isExternalLink()) { $links[] = $impl->renderLink(); } } } if ($links) { $links = phutil_implode_html(phutil_tag('br'), $links); $property_list->addProperty( pht('External Link'), $links); } $status_view = new PHUIStatusListView(); $item = new PHUIStatusItemView(); $status = $build_target->getTargetStatus(); $status_name = HarbormasterBuildTarget::getBuildTargetStatusName($status); $icon = HarbormasterBuildTarget::getBuildTargetStatusIcon($status); $color = HarbormasterBuildTarget::getBuildTargetStatusColor($status); $item->setTarget($status_name); $item->setIcon($icon, $color); $status_view->addItem($item); $when = array(); $started = $build_target->getDateStarted(); $now = PhabricatorTime::getNow(); if ($started) { $ended = $build_target->getDateCompleted(); if ($ended) { $when[] = pht( 'Completed at %s', phabricator_datetime($ended, $viewer)); $duration = ($ended - $started); if ($duration) { $when[] = pht( 'Built for %s', phutil_format_relative_time_detailed($duration)); } else { $when[] = pht('Built instantly'); } } else { $when[] = pht( 'Started at %s', phabricator_datetime($started, $viewer)); $duration = ($now - $started); if ($duration) { $when[] = pht( 'Running for %s', phutil_format_relative_time_detailed($duration)); } } } else { $created = $build_target->getDateCreated(); $when[] = pht( 'Queued at %s', phabricator_datetime($started, $viewer)); $duration = ($now - $created); if ($duration) { $when[] = pht( 'Waiting for %s', phutil_format_relative_time_detailed($duration)); } } $property_list->addProperty( pht('When'), phutil_implode_html(" \xC2\xB7 ", $when)); $property_list->addProperty(pht('Status'), $status_view); $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Overview')) ->setKey('overview') ->appendChild($property_list)); $step = $build_target->getBuildStep(); if ($step) { $description = $step->getDescription(); if ($description) { $description = new PHUIRemarkupView($viewer, $description); $property_list->addSectionHeader( pht('Description'), PHUIPropertyListView::ICON_SUMMARY); $property_list->addTextContent($description); } } else { $target_box->setFormErrors( array( pht( 'This build step has since been deleted on the build plan. '. 'Some information may be omitted.'), )); } $details = $build_target->getDetails(); $property_list = new PHUIPropertyListView(); foreach ($details as $key => $value) { $property_list->addProperty($key, $value); } $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Configuration')) ->setKey('configuration') ->appendChild($property_list)); $variables = $build_target->getVariables(); $variables_tab = $this->buildProperties($variables); $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Variables')) ->setKey('variables') ->appendChild($variables_tab)); $artifacts_tab = $this->buildArtifacts($build_target, $target_artifacts); $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Artifacts')) ->setKey('artifacts') ->appendChild($artifacts_tab)); $build_messages = idx($messages, $build_target->getPHID(), array()); $messages_tab = $this->buildMessages($build_messages); $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Messages')) ->setKey('messages') ->appendChild($messages_tab)); $property_list = new PHUIPropertyListView(); $property_list->addProperty( pht('Build Target ID'), $build_target->getID()); $property_list->addProperty( pht('Build Target PHID'), $build_target->getPHID()); $tab_group->addTab( id(new PHUITabView()) ->setName(pht('Metadata')) ->setKey('metadata') ->appendChild($property_list)); $targets[] = $target_box; $targets[] = $this->buildLog($build, $build_target); } $timeline = $this->buildTransactionTimeline( $build, new HarbormasterBuildTransactionQuery()); $timeline->setShouldTerminate(true); $view = id(new PHUITwoColumnView()) ->setHeader($page_header) ->setCurtain($curtain) ->setMainColumn(array( $properties, $targets, $timeline, )); return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) ->appendChild($view); } private function buildArtifacts( HarbormasterBuildTarget $build_target, array $artifacts) { $viewer = $this->getViewer(); $rows = array(); foreach ($artifacts as $artifact) { $impl = $artifact->getArtifactImplementation(); if ($impl) { $summary = $impl->renderArtifactSummary($viewer); $type_name = $impl->getArtifactTypeName(); } else { $summary = pht(''); $type_name = $artifact->getType(); } $rows[] = array( $artifact->getArtifactKey(), $type_name, $summary, ); } $table = id(new AphrontTableView($rows)) ->setNoDataString(pht('This target has no associated artifacts.')) ->setHeaders( array( pht('Key'), pht('Type'), pht('Summary'), )) ->setColumnClasses( array( 'pri', '', 'wide', )); return $table; } private function buildLog( HarbormasterBuild $build, HarbormasterBuildTarget $build_target) { $request = $this->getRequest(); $viewer = $request->getUser(); $limit = $request->getInt('l', 25); $logs = id(new HarbormasterBuildLogQuery()) ->setViewer($viewer) ->withBuildTargetPHIDs(array($build_target->getPHID())) ->execute(); $empty_logs = array(); $log_boxes = array(); foreach ($logs as $log) { $start = 1; $lines = preg_split("/\r\n|\r|\n/", $log->getLogText()); if ($limit !== 0) { $start = count($lines) - $limit; if ($start >= 1) { $lines = array_slice($lines, -$limit, $limit); } else { $start = 1; } } $id = null; $is_empty = false; if (count($lines) === 1 && trim($lines[0]) === '') { // Prevent Harbormaster from showing empty build logs. $id = celerity_generate_unique_node_id(); $empty_logs[] = $id; $is_empty = true; } $log_view = new ShellLogView(); $log_view->setLines($lines); $log_view->setStart($start); $header = id(new PHUIHeaderView()) ->setHeader(pht( 'Build Log %d (%s - %s)', $log->getID(), $log->getLogSource(), $log->getLogType())) ->setSubheader($this->createLogHeader($build, $log)) ->setUser($viewer); $log_box = id(new PHUIObjectBoxView()) ->setHeader($header) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->setForm($log_view); if ($is_empty) { $log_box = phutil_tag( 'div', array( 'style' => 'display: none', 'id' => $id, ), $log_box); } $log_boxes[] = $log_box; } if ($empty_logs) { $hide_id = celerity_generate_unique_node_id(); Javelin::initBehavior('phabricator-reveal-content'); $expand = phutil_tag( 'div', array( 'id' => $hide_id, 'class' => 'harbormaster-empty-logs-are-hidden', ), array( pht( '%s empty logs are hidden.', phutil_count($empty_logs)), ' ', javelin_tag( 'a', array( 'href' => '#', 'sigil' => 'reveal-content', 'meta' => array( 'showIDs' => $empty_logs, 'hideIDs' => array($hide_id), ), ), pht('Show all logs.')), )); array_unshift($log_boxes, $expand); } return $log_boxes; } private function createLogHeader($build, $log) { $request = $this->getRequest(); $limit = $request->getInt('l', 25); $lines_25 = $this->getApplicationURI('/build/'.$build->getID().'/?l=25'); $lines_50 = $this->getApplicationURI('/build/'.$build->getID().'/?l=50'); $lines_100 = $this->getApplicationURI('/build/'.$build->getID().'/?l=100'); $lines_0 = $this->getApplicationURI('/build/'.$build->getID().'/?l=0'); $link_25 = phutil_tag('a', array('href' => $lines_25), pht('25')); $link_50 = phutil_tag('a', array('href' => $lines_50), pht('50')); $link_100 = phutil_tag('a', array('href' => $lines_100), pht('100')); $link_0 = phutil_tag('a', array('href' => $lines_0), pht('Unlimited')); if ($limit === 25) { $link_25 = phutil_tag('strong', array(), $link_25); } else if ($limit === 50) { $link_50 = phutil_tag('strong', array(), $link_50); } else if ($limit === 100) { $link_100 = phutil_tag('strong', array(), $link_100); } else if ($limit === 0) { $link_0 = phutil_tag('strong', array(), $link_0); } return phutil_tag( 'span', array(), array( $link_25, ' - ', $link_50, ' - ', $link_100, ' - ', $link_0, ' Lines', )); } private function buildCurtainView(HarbormasterBuild $build) { $viewer = $this->getViewer(); $id = $build->getID(); $curtain = $this->newCurtainView($build); $can_restart = $build->canRestartBuild() && $build->canIssueCommand( $viewer, HarbormasterBuildCommand::COMMAND_RESTART); $can_pause = $build->canPauseBuild() && $build->canIssueCommand( $viewer, HarbormasterBuildCommand::COMMAND_PAUSE); $can_resume = $build->canResumeBuild() && $build->canIssueCommand( $viewer, HarbormasterBuildCommand::COMMAND_RESUME); $can_abort = $build->canAbortBuild() && $build->canIssueCommand( $viewer, HarbormasterBuildCommand::COMMAND_ABORT); $curtain->addAction( id(new PhabricatorActionView()) ->setName(pht('Restart Build')) ->setIcon('fa-repeat') ->setHref($this->getApplicationURI('/build/restart/'.$id.'/')) ->setDisabled(!$can_restart) ->setWorkflow(true)); if ($build->canResumeBuild()) { $curtain->addAction( id(new PhabricatorActionView()) ->setName(pht('Resume Build')) ->setIcon('fa-play') ->setHref($this->getApplicationURI('/build/resume/'.$id.'/')) ->setDisabled(!$can_resume) ->setWorkflow(true)); } else { $curtain->addAction( id(new PhabricatorActionView()) ->setName(pht('Pause Build')) ->setIcon('fa-pause') ->setHref($this->getApplicationURI('/build/pause/'.$id.'/')) ->setDisabled(!$can_pause) ->setWorkflow(true)); } $curtain->addAction( id(new PhabricatorActionView()) ->setName(pht('Abort Build')) ->setIcon('fa-exclamation-triangle') ->setHref($this->getApplicationURI('/build/abort/'.$id.'/')) ->setDisabled(!$can_abort) ->setWorkflow(true)); return $curtain; } private function buildPropertyList(HarbormasterBuild $build) { $viewer = $this->getViewer(); $properties = id(new PHUIPropertyListView()) ->setUser($viewer); $handles = id(new PhabricatorHandleQuery()) ->setViewer($viewer) ->withPHIDs(array( $build->getBuildablePHID(), $build->getBuildPlanPHID(), )) ->execute(); $properties->addProperty( pht('Buildable'), $handles[$build->getBuildablePHID()]->renderLink()); $properties->addProperty( pht('Build Plan'), $handles[$build->getBuildPlanPHID()]->renderLink()); $properties->addProperty( pht('Restarts'), $build->getBuildGeneration()); $properties->addProperty( pht('Status'), $this->getStatus($build)); return id(new PHUIObjectBoxView()) ->setHeaderText(pht('Properties')) ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY) ->appendChild($properties); } private function getStatus(HarbormasterBuild $build) { $status_view = new PHUIStatusListView(); $item = new PHUIStatusItemView(); if ($build->isPausing()) { $status_name = pht('Pausing'); $icon = PHUIStatusItemView::ICON_RIGHT; $color = 'dark'; } else { $status = $build->getBuildStatus(); $status_name = HarbormasterBuildStatus::getBuildStatusName($status); $icon = HarbormasterBuildStatus::getBuildStatusIcon($status); $color = HarbormasterBuildStatus::getBuildStatusColor($status); } $item->setTarget($status_name); $item->setIcon($icon, $color); $status_view->addItem($item); return $status_view; } private function buildMessages(array $messages) { $viewer = $this->getRequest()->getUser(); if ($messages) { $handles = id(new PhabricatorHandleQuery()) ->setViewer($viewer) ->withPHIDs(mpull($messages, 'getAuthorPHID')) ->execute(); } else { $handles = array(); } $rows = array(); foreach ($messages as $message) { $rows[] = array( $message->getID(), $handles[$message->getAuthorPHID()]->renderLink(), $message->getType(), $message->getIsConsumed() ? pht('Consumed') : null, phabricator_datetime($message->getDateCreated(), $viewer), ); } $table = new AphrontTableView($rows); $table->setNoDataString(pht('No messages for this build target.')); $table->setHeaders( array( pht('ID'), pht('From'), pht('Type'), pht('Consumed'), pht('Received'), )); $table->setColumnClasses( array( '', '', 'wide', '', 'date', )); return $table; } private function buildProperties(array $properties) { ksort($properties); $rows = array(); foreach ($properties as $key => $value) { $rows[] = array( $key, $value, ); } $table = id(new AphrontTableView($rows)) ->setHeaders( array( pht('Key'), pht('Value'), )) ->setColumnClasses( array( 'pri right', 'wide', )); return $table; } } diff --git a/src/applications/harbormaster/engine/HarbormasterBuildEngine.php b/src/applications/harbormaster/engine/HarbormasterBuildEngine.php index 9c4112e810..4079620f2a 100644 --- a/src/applications/harbormaster/engine/HarbormasterBuildEngine.php +++ b/src/applications/harbormaster/engine/HarbormasterBuildEngine.php @@ -1,578 +1,578 @@ forceBuildableUpdate = $force_buildable_update; return $this; } public function shouldForceBuildableUpdate() { return $this->forceBuildableUpdate; } public function queueNewBuildTarget(HarbormasterBuildTarget $target) { $this->newBuildTargets[] = $target; return $this; } public function getNewBuildTargets() { return $this->newBuildTargets; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setBuild(HarbormasterBuild $build) { $this->build = $build; return $this; } public function getBuild() { return $this->build; } public function continueBuild() { $build = $this->getBuild(); $lock_key = 'harbormaster.build:'.$build->getID(); $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15); $build->reload(); $old_status = $build->getBuildStatus(); try { $this->updateBuild($build); } catch (Exception $ex) { // If any exception is raised, the build is marked as a failure and the // exception is re-thrown (this ensures we don't leave builds in an // inconsistent state). $build->setBuildStatus(HarbormasterBuildStatus::STATUS_ERROR); $build->save(); $lock->unlock(); $this->releaseAllArtifacts($build); throw $ex; } $lock->unlock(); // NOTE: We queue new targets after releasing the lock so that in-process // execution via `bin/harbormaster` does not reenter the locked region. foreach ($this->getNewBuildTargets() as $target) { $task = PhabricatorWorker::scheduleTask( 'HarbormasterTargetWorker', array( 'targetID' => $target->getID(), ), array( 'objectPHID' => $target->getPHID(), )); } // If the build changed status, we might need to update the overall status // on the buildable. $new_status = $build->getBuildStatus(); if ($new_status != $old_status || $this->shouldForceBuildableUpdate()) { $this->updateBuildable($build->getBuildable()); } $this->releaseQueuedArtifacts(); // If we are no longer building for any reason, release all artifacts. if (!$build->isBuilding()) { $this->releaseAllArtifacts($build); } } private function updateBuild(HarbormasterBuild $build) { if ($build->isAborting()) { $this->releaseAllArtifacts($build); $build->setBuildStatus(HarbormasterBuildStatus::STATUS_ABORTED); $build->save(); } if (($build->getBuildStatus() == HarbormasterBuildStatus::STATUS_PENDING) || ($build->isRestarting())) { $this->restartBuild($build); $build->setBuildStatus(HarbormasterBuildStatus::STATUS_BUILDING); $build->save(); } if ($build->isResuming()) { $build->setBuildStatus(HarbormasterBuildStatus::STATUS_BUILDING); $build->save(); } if ($build->isPausing() && !$build->isComplete()) { $build->setBuildStatus(HarbormasterBuildStatus::STATUS_PAUSED); $build->save(); } $build->deleteUnprocessedCommands(); if ($build->getBuildStatus() == HarbormasterBuildStatus::STATUS_BUILDING) { $this->updateBuildSteps($build); } } private function restartBuild(HarbormasterBuild $build) { // We're restarting the build, so release all previous artifacts. $this->releaseAllArtifacts($build); // Increment the build generation counter on the build. $build->setBuildGeneration($build->getBuildGeneration() + 1); // Currently running targets should periodically check their build // generation (which won't have changed) against the build's generation. // If it is different, they will automatically stop what they're doing // and abort. // Previously we used to delete targets, logs and artifacts here. Instead // leave them around so users can view previous generations of this build. } private function updateBuildSteps(HarbormasterBuild $build) { $all_targets = id(new HarbormasterBuildTargetQuery()) ->setViewer($this->getViewer()) ->withBuildPHIDs(array($build->getPHID())) ->withBuildGenerations(array($build->getBuildGeneration())) ->execute(); $this->updateWaitingTargets($all_targets); $targets = mgroup($all_targets, 'getBuildStepPHID'); $steps = id(new HarbormasterBuildStepQuery()) ->setViewer($this->getViewer()) ->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID())) ->execute(); $steps = mpull($steps, null, 'getPHID'); // Identify steps which are in various states. $queued = array(); $underway = array(); $waiting = array(); $complete = array(); $failed = array(); foreach ($steps as $step) { $step_targets = idx($targets, $step->getPHID(), array()); if ($step_targets) { $is_queued = false; $is_underway = false; foreach ($step_targets as $target) { if ($target->isUnderway()) { $is_underway = true; break; } } $is_waiting = false; foreach ($step_targets as $target) { if ($target->isWaiting()) { $is_waiting = true; break; } } $is_complete = true; foreach ($step_targets as $target) { if (!$target->isComplete()) { $is_complete = false; break; } } $is_failed = false; foreach ($step_targets as $target) { if ($target->isFailed()) { $is_failed = true; break; } } } else { $is_queued = true; $is_underway = false; $is_waiting = false; $is_complete = false; $is_failed = false; } if ($is_queued) { $queued[$step->getPHID()] = true; } if ($is_underway) { $underway[$step->getPHID()] = true; } if ($is_waiting) { $waiting[$step->getPHID()] = true; } if ($is_complete) { $complete[$step->getPHID()] = true; } if ($is_failed) { $failed[$step->getPHID()] = true; } } // If any step failed, fail the whole build, then bail. if (count($failed)) { $build->setBuildStatus(HarbormasterBuildStatus::STATUS_FAILED); $build->save(); return; } // If every step is complete, we're done with this build. Mark it passed // and bail. if (count($complete) == count($steps)) { $build->setBuildStatus(HarbormasterBuildStatus::STATUS_PASSED); $build->save(); return; } // Release any artifacts which are not inputs to any remaining build // step. We're done with these, so something else is free to use them. $ongoing_phids = array_keys($queued + $waiting + $underway); $ongoing_steps = array_select_keys($steps, $ongoing_phids); $this->releaseUnusedArtifacts($all_targets, $ongoing_steps); // Identify all the steps which are ready to run (because all their // dependencies are complete). $runnable = array(); foreach ($steps as $step) { $dependencies = $step->getStepImplementation()->getDependencies($step); if (isset($queued[$step->getPHID()])) { $can_run = true; foreach ($dependencies as $dependency) { if (empty($complete[$dependency])) { $can_run = false; break; } } if ($can_run) { $runnable[] = $step; } } } if (!$runnable && !$waiting && !$underway) { // This means the build is deadlocked, and the user has configured // circular dependencies. $build->setBuildStatus(HarbormasterBuildStatus::STATUS_DEADLOCKED); $build->save(); return; } foreach ($runnable as $runnable_step) { $target = HarbormasterBuildTarget::initializeNewBuildTarget( $build, $runnable_step, $build->retrieveVariablesFromBuild()); $target->save(); $this->queueNewBuildTarget($target); } } /** * Release any artifacts which aren't used by any running or waiting steps. * * This releases artifacts as soon as they're no longer used. This can be * particularly relevant when a build uses multiple hosts since it returns * hosts to the pool more quickly. * * @param list Targets in the build. * @param list List of running and waiting steps. * @return void */ private function releaseUnusedArtifacts(array $targets, array $steps) { assert_instances_of($targets, 'HarbormasterBuildTarget'); assert_instances_of($steps, 'HarbormasterBuildStep'); if (!$targets || !$steps) { return; } $target_phids = mpull($targets, 'getPHID'); $artifacts = id(new HarbormasterBuildArtifactQuery()) ->setViewer($this->getViewer()) ->withBuildTargetPHIDs($target_phids) ->withIsReleased(false) ->execute(); if (!$artifacts) { return; } // Collect all the artifacts that remaining build steps accept as inputs. $must_keep = array(); foreach ($steps as $step) { $inputs = $step->getStepImplementation()->getArtifactInputs(); foreach ($inputs as $input) { $artifact_key = $input['key']; $must_keep[$artifact_key] = true; } } // Queue unreleased artifacts which no remaining step uses for immediate // release. foreach ($artifacts as $artifact) { $key = $artifact->getArtifactKey(); if (isset($must_keep[$key])) { continue; } $this->artifactReleaseQueue[] = $artifact; } } /** * Process messages which were sent to these targets, kicking applicable * targets out of "Waiting" and into either "Passed" or "Failed". * * @param list List of targets to process. * @return void */ private function updateWaitingTargets(array $targets) { assert_instances_of($targets, 'HarbormasterBuildTarget'); // We only care about messages for targets which are actually in a waiting // state. $waiting_targets = array(); foreach ($targets as $target) { if ($target->isWaiting()) { $waiting_targets[$target->getPHID()] = $target; } } if (!$waiting_targets) { return; } $messages = id(new HarbormasterBuildMessageQuery()) ->setViewer($this->getViewer()) - ->withBuildTargetPHIDs(array_keys($waiting_targets)) + ->withReceiverPHIDs(array_keys($waiting_targets)) ->withConsumed(false) ->execute(); foreach ($messages as $message) { - $target = $waiting_targets[$message->getBuildTargetPHID()]; + $target = $waiting_targets[$message->getReceiverPHID()]; switch ($message->getType()) { case HarbormasterMessageType::MESSAGE_PASS: $new_status = HarbormasterBuildTarget::STATUS_PASSED; break; case HarbormasterMessageType::MESSAGE_FAIL: $new_status = HarbormasterBuildTarget::STATUS_FAILED; break; case HarbormasterMessageType::MESSAGE_WORK: default: $new_status = null; break; } if ($new_status !== null) { $message->setIsConsumed(true); $message->save(); $target->setTargetStatus($new_status); if ($target->isComplete()) { $target->setDateCompleted(PhabricatorTime::getNow()); } $target->save(); } } } /** * Update the overall status of the buildable this build is attached to. * * After a build changes state (for example, passes or fails) it may affect * the overall state of the associated buildable. Compute the new aggregate * state and save it on the buildable. * * @param HarbormasterBuild The buildable to update. * @return void */ private function updateBuildable(HarbormasterBuildable $buildable) { $viewer = $this->getViewer(); $lock_key = 'harbormaster.buildable:'.$buildable->getID(); $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15); $buildable = id(new HarbormasterBuildableQuery()) ->setViewer($viewer) ->withIDs(array($buildable->getID())) ->needBuilds(true) ->executeOne(); $all_pass = true; $any_fail = false; foreach ($buildable->getBuilds() as $build) { if (!$build->isPassed()) { $all_pass = false; } if ($build->isComplete() && !$build->isPassed()) { $any_fail = true; } } if ($any_fail) { $new_status = HarbormasterBuildable::STATUS_FAILED; } else if ($all_pass) { $new_status = HarbormasterBuildable::STATUS_PASSED; } else { $new_status = HarbormasterBuildable::STATUS_BUILDING; } $old_status = $buildable->getBuildableStatus(); $did_update = ($old_status != $new_status); if ($did_update) { $buildable->setBuildableStatus($new_status); $buildable->save(); } $lock->unlock(); // If we changed the buildable status, try to post a transaction to the // object about it. We can safely do this outside of the locked region. // NOTE: We only post transactions for automatic buildables, not for // manual ones: manual builds are test builds, whoever is doing tests // can look at the results themselves, and other users generally don't // care about the outcome. $should_publish = $did_update && $new_status != HarbormasterBuildable::STATUS_BUILDING && !$buildable->getIsManualBuildable(); if (!$should_publish) { return; } $object = id(new PhabricatorObjectQuery()) ->setViewer($viewer) ->withPHIDs(array($buildable->getBuildablePHID())) ->executeOne(); if (!$object) { return; } $publish_phid = $object->getHarbormasterPublishablePHID(); if (!$publish_phid) { return; } if ($publish_phid === $object->getPHID()) { $publish = $object; } else { $publish = id(new PhabricatorObjectQuery()) ->setViewer($viewer) ->withPHIDs(array($publish_phid)) ->executeOne(); if (!$publish) { return; } } if (!($publish instanceof PhabricatorApplicationTransactionInterface)) { return; } $template = $publish->getApplicationTransactionTemplate(); if (!$template) { return; } $template ->setTransactionType(PhabricatorTransactions::TYPE_BUILDABLE) ->setMetadataValue( 'harbormaster:buildablePHID', $buildable->getPHID()) ->setOldValue($old_status) ->setNewValue($new_status); $harbormaster_phid = id(new PhabricatorHarbormasterApplication()) ->getPHID(); $daemon_source = PhabricatorContentSource::newForSource( PhabricatorDaemonContentSource::SOURCECONST); $editor = $publish->getApplicationTransactionEditor() ->setActor($viewer) ->setActingAsPHID($harbormaster_phid) ->setContentSource($daemon_source) ->setContinueOnNoEffect(true) ->setContinueOnMissingFields(true); $editor->applyTransactions( $publish->getApplicationTransactionObject(), array($template)); } private function releaseAllArtifacts(HarbormasterBuild $build) { $targets = id(new HarbormasterBuildTargetQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withBuildPHIDs(array($build->getPHID())) ->withBuildGenerations(array($build->getBuildGeneration())) ->execute(); if (count($targets) === 0) { return; } $target_phids = mpull($targets, 'getPHID'); $artifacts = id(new HarbormasterBuildArtifactQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withBuildTargetPHIDs($target_phids) ->withIsReleased(false) ->execute(); foreach ($artifacts as $artifact) { $artifact->releaseArtifact(); } } private function releaseQueuedArtifacts() { foreach ($this->artifactReleaseQueue as $key => $artifact) { $artifact->releaseArtifact(); unset($this->artifactReleaseQueue[$key]); } } } diff --git a/src/applications/harbormaster/query/HarbormasterBuildMessageQuery.php b/src/applications/harbormaster/query/HarbormasterBuildMessageQuery.php index 749fdd76d4..1b55f2e781 100644 --- a/src/applications/harbormaster/query/HarbormasterBuildMessageQuery.php +++ b/src/applications/harbormaster/query/HarbormasterBuildMessageQuery.php @@ -1,98 +1,92 @@ ids = $ids; return $this; } - public function withBuildTargetPHIDs(array $phids) { - $this->buildTargetPHIDs = $phids; + public function withReceiverPHIDs(array $phids) { + $this->receiverPHIDs = $phids; return $this; } public function withConsumed($consumed) { $this->consumed = $consumed; return $this; } + public function newResultObject() { + return new HarbormasterBuildMessage(); + } + protected function loadPage() { - $table = new HarbormasterBuildMessage(); - $conn_r = $table->establishConnection('r'); - - $data = queryfx_all( - $conn_r, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn_r), - $this->buildOrderClause($conn_r), - $this->buildLimitClause($conn_r)); - - return $table->loadAllFromArray($data); + return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $page) { - $build_target_phids = array_filter(mpull($page, 'getBuildTargetPHID')); - if ($build_target_phids) { - $build_targets = id(new PhabricatorObjectQuery()) + $receiver_phids = array_filter(mpull($page, 'getReceiverPHID')); + if ($receiver_phids) { + $receivers = id(new PhabricatorObjectQuery()) ->setViewer($this->getViewer()) - ->withPHIDs($build_target_phids) + ->withPHIDs($receiver_phids) ->setParentQuery($this) ->execute(); - $build_targets = mpull($build_targets, null, 'getPHID'); + $receivers = mpull($receivers, null, 'getPHID'); } else { - $build_targets = array(); + $receivers = array(); } foreach ($page as $key => $message) { - $build_target_phid = $message->getBuildTargetPHID(); - if (empty($build_targets[$build_target_phid])) { + $receiver_phid = $message->getReceiverPHID(); + + if (empty($receivers[$receiver_phid])) { unset($page[$key]); + $this->didRejectResult($message); continue; } - $message->attachBuildTarget($build_targets[$build_target_phid]); + + $message->attachReceiver($receivers[$receiver_phid]); } return $page; } - protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { - $where = array(); + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); - if ($this->ids) { + if ($this->ids !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'id IN (%Ld)', $this->ids); } - if ($this->buildTargetPHIDs) { + if ($this->receiverPHIDs !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'buildTargetPHID IN (%Ls)', - $this->buildTargetPHIDs); + $this->receiverPHIDs); } if ($this->consumed !== null) { $where[] = qsprintf( - $conn_r, + $conn, 'isConsumed = %d', (int)$this->consumed); } - $where[] = $this->buildPagingClause($conn_r); - - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { return 'PhabricatorHarbormasterApplication'; } } diff --git a/src/applications/harbormaster/storage/HarbormasterBuildMessage.php b/src/applications/harbormaster/storage/HarbormasterBuildMessage.php index 20d138ad02..5c9c92b2fc 100644 --- a/src/applications/harbormaster/storage/HarbormasterBuildMessage.php +++ b/src/applications/harbormaster/storage/HarbormasterBuildMessage.php @@ -1,77 +1,89 @@ getPHID(); if (!$actor_phid) { $actor_phid = id(new PhabricatorHarbormasterApplication())->getPHID(); } return id(new HarbormasterBuildMessage()) ->setAuthorPHID($actor_phid) ->setIsConsumed(0); } protected function getConfiguration() { return array( self::CONFIG_COLUMN_SCHEMA => array( 'type' => 'text16', 'isConsumed' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_buildtarget' => array( 'columns' => array('buildTargetPHID'), ), ), ) + parent::getConfiguration(); } + public function getReceiverPHID() { + return $this->getBuildTargetPHID(); + } + + public function setReceiverPHID($phid) { + return $this->setBuildTargetPHID($phid); + } + + public function getReceiver() { + return $this->assertAttached($this->receiver); + } + public function getBuildTarget() { - return $this->assertAttached($this->buildTarget); + return $this->getReceiver(); } - public function attachBuildTarget(HarbormasterBuildTarget $target) { - $this->buildTarget = $target; + public function attachReceiver($receiver) { + $this->receiver = $receiver; return $this; } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { - return $this->getBuildTarget()->getPolicy($capability); + return $this->getReceiver()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { - return $this->getBuildTarget()->hasAutomaticCapability( + return $this->getReceiver()->hasAutomaticCapability( $capability, $viewer); } public function describeAutomaticCapability($capability) { - return pht('Build messages have the same policies as their targets.'); + return pht('Build messages have the same policies as their receivers.'); } }