diff --git a/src/applications/differential/query/DifferentialDiffQuery.php b/src/applications/differential/query/DifferentialDiffQuery.php index 1616d58ac8..23c016446c 100644 --- a/src/applications/differential/query/DifferentialDiffQuery.php +++ b/src/applications/differential/query/DifferentialDiffQuery.php @@ -1,123 +1,139 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withRevisionIDs(array $revision_ids) { $this->revisionIDs = $revision_ids; return $this; } public function needChangesets($bool) { $this->needChangesets = $bool; return $this; } + public function needProperties($need_properties) { + $this->needProperties = $need_properties; + return $this; + } + + public function newResultObject() { + return new DifferentialDiff(); + } + protected function loadPage() { - $table = new DifferentialDiff(); - $conn_r = $table->establishConnection('r'); - - $data = queryfx_all( - $conn_r, - 'SELECT * FROM %T %Q %Q %Q', - $table->getTableName(), - $this->buildWhereClause($conn_r), - $this->buildOrderClause($conn_r), - $this->buildLimitClause($conn_r)); - - return $table->loadAllFromArray($data); + return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $diffs) { $revision_ids = array_filter(mpull($diffs, 'getRevisionID')); $revisions = array(); if ($revision_ids) { $revisions = id(new DifferentialRevisionQuery()) ->setViewer($this->getViewer()) ->withIDs($revision_ids) ->execute(); } foreach ($diffs as $key => $diff) { if (!$diff->getRevisionID()) { continue; } $revision = idx($revisions, $diff->getRevisionID()); if ($revision) { $diff->attachRevision($revision); continue; } unset($diffs[$key]); } if ($diffs && $this->needChangesets) { $diffs = $this->loadChangesets($diffs); } return $diffs; } + protected function didFilterPage(array $diffs) { + if ($this->needProperties) { + $properties = id(new DifferentialDiffProperty())->loadAllWhere( + 'diffID IN (%Ld)', + mpull($diffs, 'getID')); + + $properties = mgroup($properties, 'getDiffID'); + foreach ($diffs as $diff) { + $map = idx($properties, $diff->getID(), array()); + $map = mpull($map, 'getData', 'getName'); + $diff->attachDiffProperties($map); + } + } + + return $diffs; + } + private function loadChangesets(array $diffs) { id(new DifferentialChangesetQuery()) ->setViewer($this->getViewer()) ->setParentQuery($this) ->withDiffs($diffs) ->needAttachToDiffs(true) ->needHunks(true) ->execute(); return $diffs; } - protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { - $where = array(); + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); if ($this->ids) { $where[] = qsprintf( - $conn_r, + $conn, 'id IN (%Ld)', $this->ids); } if ($this->phids) { $where[] = qsprintf( - $conn_r, + $conn, 'phid IN (%Ls)', $this->phids); } if ($this->revisionIDs) { $where[] = qsprintf( - $conn_r, + $conn, 'revisionID IN (%Ld)', $this->revisionIDs); } - $where[] = $this->buildPagingClause($conn_r); - return $this->formatWhereClause($where); + return $where; } public function getQueryApplicationClass() { return 'PhabricatorDifferentialApplication'; } } diff --git a/src/applications/drydock/operation/DrydockLandRepositoryOperation.php b/src/applications/drydock/operation/DrydockLandRepositoryOperation.php index 77de219115..4e306772a3 100644 --- a/src/applications/drydock/operation/DrydockLandRepositoryOperation.php +++ b/src/applications/drydock/operation/DrydockLandRepositoryOperation.php @@ -1,339 +1,443 @@ getRepositoryTarget(); $repository = $operation->getRepository(); switch ($operation->getOperationState()) { case DrydockRepositoryOperation::STATE_WAIT: return pht( 'Waiting to land revision into %s on %s...', $repository->getMonogram(), $target); case DrydockRepositoryOperation::STATE_WORK: return pht( 'Landing revision into %s on %s...', $repository->getMonogram(), $target); case DrydockRepositoryOperation::STATE_DONE: return pht( 'Revision landed into %s.', $repository->getMonogram()); } } public function getWorkingCopyMerges(DrydockRepositoryOperation $operation) { $repository = $operation->getRepository(); $merges = array(); $object = $operation->getObject(); if ($object instanceof DifferentialRevision) { $diff = $this->loadDiff($operation); $merges[] = array( 'src.uri' => $repository->getStagingURI(), 'src.ref' => $diff->getStagingRef(), ); } else { throw new Exception( pht( 'Invalid or unknown object ("%s") for land operation, expected '. 'Differential Revision.', $operation->getObjectPHID())); } return $merges; } public function applyOperation( DrydockRepositoryOperation $operation, DrydockInterface $interface) { $viewer = $this->getViewer(); $repository = $operation->getRepository(); $cmd = array(); $arg = array(); $object = $operation->getObject(); if ($object instanceof DifferentialRevision) { $revision = $object; $diff = $this->loadDiff($operation); $dict = $diff->getDiffAuthorshipDict(); $author_name = idx($dict, 'authorName'); $author_email = idx($dict, 'authorEmail'); $api_method = 'differential.getcommitmessage'; $api_params = array( 'revision_id' => $revision->getID(), ); $commit_message = id(new ConduitCall($api_method, $api_params)) ->setUser($viewer) ->execute(); } else { throw new Exception( pht( 'Invalid or unknown object ("%s") for land operation, expected '. 'Differential Revision.', $operation->getObjectPHID())); } $target = $operation->getRepositoryTarget(); list($type, $name) = explode(':', $target, 2); switch ($type) { case 'branch': $push_dst = 'refs/heads/'.$name; break; default: throw new Exception( pht( 'Unknown repository operation target type "%s" (in target "%s").', $type, $target)); } $committer_info = $this->getCommitterInfo($operation); // NOTE: We're doing this commit with "-F -" so we don't run into trouble // with enormous commit messages which might otherwise exceed the maximum // size of a command. $future = $interface->getExecFuture( 'git -c user.name=%s -c user.email=%s commit --author %s -F - --', $committer_info['name'], $committer_info['email'], "{$author_name} <{$author_email}>"); $future->write($commit_message); try { $future->resolvex(); } catch (CommandException $ex) { $display_command = csprintf('git commit'); // TODO: One reason this can fail is if the changes have already been // merged. We could try to detect that. $error = DrydockCommandError::newFromCommandException($ex) ->setPhase(self::PHASE_COMMIT) ->setDisplayCommand($display_command); $operation->setCommandError($error->toDictionary()); throw $ex; } try { $interface->execx( 'git push origin -- %s:%s', 'HEAD', $push_dst); } catch (CommandException $ex) { $display_command = csprintf( 'git push origin %R:%R', 'HEAD', $push_dst); $error = DrydockCommandError::newFromCommandException($ex) ->setPhase(self::PHASE_PUSH) ->setDisplayCommand($display_command); $operation->setCommandError($error->toDictionary()); throw $ex; } } private function getCommitterInfo(DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); $committer_name = null; $author_phid = $operation->getAuthorPHID(); $object = id(new PhabricatorObjectQuery()) ->setViewer($viewer) ->withPHIDs(array($author_phid)) ->executeOne(); if ($object) { if ($object instanceof PhabricatorUser) { $committer_name = $object->getUsername(); } } if (!strlen($committer_name)) { $committer_name = pht('autocommitter'); } // TODO: Probably let users choose a VCS email address in settings. For // now just make something up so we don't leak anyone's stuff. return array( 'name' => $committer_name, 'email' => 'autocommitter@example.com', ); } private function loadDiff(DrydockRepositoryOperation $operation) { $viewer = $this->getViewer(); $revision = $operation->getObject(); $diff_phid = $operation->getProperty('differential.diffPHID'); $diff = id(new DifferentialDiffQuery()) ->setViewer($viewer) ->withPHIDs(array($diff_phid)) ->executeOne(); if (!$diff) { throw new Exception( pht( 'Unable to load diff "%s".', $diff_phid)); } $diff_revid = $diff->getRevisionID(); $revision_id = $revision->getID(); if ($diff_revid != $revision_id) { throw new Exception( pht( 'Diff ("%s") has wrong revision ID ("%s", expected "%s").', $diff_phid, $diff_revid, $revision_id)); } return $diff; } public function getBarrierToLanding( PhabricatorUser $viewer, DifferentialRevision $revision) { $repository = $revision->getRepository(); if (!$repository) { return array( 'title' => pht('No Repository'), 'body' => pht( 'This revision is not associated with a known repository. Only '. 'revisions associated with a tracked repository can be landed '. 'automatically.'), ); } if (!$repository->canPerformAutomation()) { return array( 'title' => pht('No Repository Automation'), 'body' => pht( 'The repository this revision is associated with ("%s") is not '. 'configured to support automation. Configure automation for the '. 'repository to enable revisions to be landed automatically.', $repository->getMonogram()), ); } + // Check if this diff was pushed to a staging area. + $diff = id(new DifferentialDiffQuery()) + ->setViewer($viewer) + ->withIDs(array($revision->getActiveDiff()->getID())) + ->needProperties(true) + ->executeOne(); + + // Older diffs won't have this property. They may still have been pushed. + // At least for now, assume staging changes are present if the property + // is missing. This should smooth the transition to the more formal + // approach. + $has_staging = $diff->hasDiffProperty('arc.staging'); + if ($has_staging) { + $staging = $diff->getProperty('arc.staging'); + if (!is_array($staging)) { + $staging = array(); + } + $status = idx($staging, 'status'); + if ($status != ArcanistDiffWorkflow::STAGING_PUSHED) { + return $this->getBarrierToLandingFromStagingStatus($status); + } + } + // TODO: At some point we should allow installs to give "land reviewed // code" permission to more users than "push any commit", because it is // a much less powerful operation. For now, just require push so this // doesn't do anything users can't do on their own. $can_push = PhabricatorPolicyFilter::hasCapability( $viewer, $repository, DiffusionPushCapability::CAPABILITY); if (!$can_push) { return array( 'title' => pht('Unable to Push'), 'body' => pht( 'You do not have permission to push to the repository this '. 'revision is associated with ("%s"), so you can not land it.', $repository->getMonogram()), ); } $status_accepted = ArcanistDifferentialRevisionStatus::ACCEPTED; if ($revision->getStatus() != $status_accepted) { switch ($revision->getStatus()) { case ArcanistDifferentialRevisionStatus::CLOSED: return array( 'title' => pht('Revision Closed'), 'body' => pht( 'This revision has already been closed. Only open, accepted '. 'revisions may land.'), ); case ArcanistDifferentialRevisionStatus::ABANDONED: return array( 'title' => pht('Revision Abandoned'), 'body' => pht( 'This revision has been abandoned. Only accepted revisions '. 'may land.'), ); default: return array( 'title' => pht('Revision Not Accepted'), 'body' => pht( 'This revision is still under review. Only revisions which '. 'have been accepted may land.'), ); } } // Check for other operations. Eventually this should probably be more // general (e.g., it's OK to land to multiple different branches // simultaneously) but just put this in as a sanity check for now. $other_operations = id(new DrydockRepositoryOperationQuery()) ->setViewer($viewer) ->withObjectPHIDs(array($revision->getPHID())) ->withOperationTypes( array( $this->getOperationConstant(), )) ->withOperationStates( array( DrydockRepositoryOperation::STATE_WAIT, DrydockRepositoryOperation::STATE_WORK, DrydockRepositoryOperation::STATE_DONE, )) ->execute(); if ($other_operations) { $any_done = false; foreach ($other_operations as $operation) { if ($operation->isDone()) { $any_done = true; break; } } if ($any_done) { return array( 'title' => pht('Already Complete'), 'body' => pht('This revision has already landed.'), ); } else { return array( 'title' => pht('Already In Flight'), 'body' => pht('This revision is already landing.'), ); } } return null; } + private function getBarrierToLandingFromStagingStatus($status) { + switch ($status) { + case ArcanistDiffWorkflow::STAGING_USER_SKIP: + return array( + 'title' => pht('Staging Area Skipped'), + 'body' => pht( + 'The diff author used the %s flag to skip pushing this change to '. + 'staging. Changes must be pushed to staging before they can be '. + 'landed from the web.', + phutil_tag('tt', array(), '--skip-staging')), + ); + case ArcanistDiffWorkflow::STAGING_DIFF_RAW: + return array( + 'title' => pht('Raw Diff Source'), + 'body' => pht( + 'The diff was generated from a raw input source, so the change '. + 'could not be pushed to staging. Changes must be pushed to '. + 'staging before they can be landed from the web.'), + ); + case ArcanistDiffWorkflow::STAGING_REPOSITORY_UNKNOWN: + return array( + 'title' => pht('Unknown Repository'), + 'body' => pht( + 'When the diff was generated, the client was not able to '. + 'determine which repository it belonged to, so the change '. + 'was not pushed to staging. Changes must be pushed to staging '. + 'before they can be landed from the web.'), + ); + case ArcanistDiffWorkflow::STAGING_REPOSITORY_UNAVAILABLE: + return array( + 'title' => pht('Staging Unavailable'), + 'body' => pht( + 'When this diff was generated, the server was running an older '. + 'version of Phabricator which did not support staging areas, so '. + 'the change was not pushed to staging. Changes must be pushed '. + 'to staging before they can be landed from the web.'), + ); + case ArcanistDiffWorkflow::STAGING_REPOSITORY_UNSUPPORTED: + return array( + 'title' => pht('Repository Unsupported'), + 'body' => pht( + 'When this diff was generated, the server was running an older '. + 'version of Phabricator which did not support staging areas for '. + 'this version control system, so the chagne was not pushed to '. + 'staging. Changes must be pushed to staging before they can be '. + 'landed from the web.'), + ); + + case ArcanistDiffWorkflow::STAGING_REPOSITORY_UNCONFIGURED: + return array( + 'title' => pht('Repository Unconfigured'), + 'body' => pht( + 'When this diff was generated, the repository was not configured '. + 'with a staging area, so the change was not pushed to staging. '. + 'Changes must be pushed to staging before they can be landed '. + 'from the web.'), + ); + case ArcanistDiffWorkflow::STAGING_CLIENT_UNSUPPORTED: + return array( + 'title' => pht('Client Support Unavailable'), + 'body' => pht( + 'When this diff was generated, the client did not support '. + 'staging areas for this version control system, so the change '. + 'was not pushed to staging. Changes must be pushed to staging '. + 'before they can be landed from the web. Updating the client '. + 'may resolve this issue.'), + ); + default: + return array( + 'title' => pht('Unknown Error'), + 'body' => pht( + 'When this diff was generated, it was not pushed to staging for '. + 'an unknown reason (the status code was "%s"). Changes must be '. + 'pushed to staging before they can be landed from the web. '. + 'The server may be running an out-of-date version of Phabricator, '. + 'and updating may provide more information about this error.', + $status), + ); + } + } + } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php index 40090d0ac3..290b288c6c 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -1,304 +1,306 @@ rope = new PhutilRope(); } public function __destruct() { if ($this->isOpen) { $this->closeBuildLog(); } } public static function initializeNewBuildLog( HarbormasterBuildTarget $build_target) { return id(new HarbormasterBuildLog()) ->setBuildTargetPHID($build_target->getPHID()) ->setDuration(null) ->setLive(0); } public function openBuildLog() { if ($this->isOpen) { throw new Exception(pht('This build log is already open!')); } $this->isOpen = true; return $this ->setLive(1) ->save(); } public function closeBuildLog() { if (!$this->isOpen) { throw new Exception(pht('This build log is not open!')); } if ($this->canCompressLog()) { $this->compressLog(); } $start = $this->getDateCreated(); $now = PhabricatorTime::getNow(); return $this ->setDuration($now - $start) ->setLive(0) ->save(); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_COLUMN_SCHEMA => array( // T6203/NULLABILITY // It seems like these should be non-nullable? All logs should have a // source, etc. 'logSource' => 'text255?', 'logType' => 'text255?', 'duration' => 'uint32?', 'live' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_buildtarget' => array( 'columns' => array('buildTargetPHID'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( HarbormasterBuildLogPHIDType::TYPECONST); } public function attachBuildTarget(HarbormasterBuildTarget $build_target) { $this->buildTarget = $build_target; return $this; } public function getBuildTarget() { return $this->assertAttached($this->buildTarget); } public function getName() { return pht('Build Log'); } public function append($content) { if (!$this->getLive()) { throw new PhutilInvalidStateException('openBuildLog'); } $content = (string)$content; $this->rope->append($content); $this->flush(); + + return $this; } private function flush() { // TODO: Maybe don't flush more than a couple of times per second. If a // caller writes a single character over and over again, we'll currently // spend a lot of time flushing that. $chunk_table = id(new HarbormasterBuildLogChunk())->getTableName(); $chunk_limit = self::CHUNK_BYTE_LIMIT; $encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT; $rope = $this->rope; while (true) { $length = $rope->getByteLength(); if (!$length) { break; } $conn_w = $this->establishConnection('w'); $last = $this->loadLastChunkInfo(); $can_append = ($last) && ($last['encoding'] == $encoding_text) && ($last['size'] < $chunk_limit); if ($can_append) { $append_id = $last['id']; $prefix_size = $last['size']; } else { $append_id = null; $prefix_size = 0; } $data_limit = ($chunk_limit - $prefix_size); $append_data = $rope->getPrefixBytes($data_limit); $data_size = strlen($append_data); if ($append_id) { queryfx( $conn_w, 'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d', $chunk_table, $append_data, $prefix_size + $data_size, $append_id); } else { $this->writeChunk($encoding_text, $data_size, $append_data); } $rope->removeBytesFromHead($data_size); } } public function newChunkIterator() { return id(new HarbormasterBuildLogChunkIterator($this)) ->setPageSize(32); } private function loadLastChunkInfo() { $chunk_table = new HarbormasterBuildLogChunk(); $conn_w = $chunk_table->establishConnection('w'); return queryfx_one( $conn_w, 'SELECT id, size, encoding FROM %T WHERE logID = %d ORDER BY id DESC LIMIT 1', $chunk_table->getTableName(), $this->getID()); } public function getLogText() { // TODO: Remove this method since it won't scale for big logs. $all_chunks = $this->newChunkIterator(); $full_text = array(); foreach ($all_chunks as $chunk) { $full_text[] = $chunk->getChunkDisplayText(); } return implode('', $full_text); } private function canCompressLog() { return function_exists('gzdeflate'); } public function compressLog() { $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP); } public function decompressLog() { $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT); } private function processLog($mode) { $chunks = $this->newChunkIterator(); // NOTE: Because we're going to insert new chunks, we need to stop the // iterator once it hits the final chunk which currently exists. Otherwise, // it may start consuming chunks we just wrote and run forever. $last = $this->loadLastChunkInfo(); if ($last) { $chunks->setRange(null, $last['id']); } $byte_limit = self::CHUNK_BYTE_LIMIT; $rope = new PhutilRope(); $this->openTransaction(); foreach ($chunks as $chunk) { $rope->append($chunk->getChunkDisplayText()); $chunk->delete(); while ($rope->getByteLength() > $byte_limit) { $this->writeEncodedChunk($rope, $byte_limit, $mode); } } while ($rope->getByteLength()) { $this->writeEncodedChunk($rope, $byte_limit, $mode); } $this->saveTransaction(); } private function writeEncodedChunk(PhutilRope $rope, $length, $mode) { $data = $rope->getPrefixBytes($length); $size = strlen($data); switch ($mode) { case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT: // Do nothing. break; case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP: $data = gzdeflate($data); if ($data === false) { throw new Exception(pht('Failed to gzdeflate() log data!')); } break; default: throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); } $this->writeChunk($mode, $size, $data); $rope->removeBytesFromHead($size); } private function writeChunk($encoding, $raw_size, $data) { return id(new HarbormasterBuildLogChunk()) ->setLogID($this->getID()) ->setEncoding($encoding) ->setSize($raw_size) ->setChunk($data) ->save(); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getBuildTarget()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getBuildTarget()->hasAutomaticCapability( $capability, $viewer); } public function describeAutomaticCapability($capability) { return pht( "Users must be able to see a build target to view it's build log."); } }