diff --git a/resources/sql/autopatches/20180222.log.01.filephid.sql b/resources/sql/autopatches/20180222.log.01.filephid.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20180222.log.01.filephid.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlog + ADD filePHID VARBINARY(64); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1309,6 +1309,7 @@ 'HarbormasterLogWorker' => 'applications/harbormaster/worker/HarbormasterLogWorker.php', 'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php', 'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php', + 'HarbormasterManagementRebuildLogWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php', 'HarbormasterManagementRestartWorkflow' => 'applications/harbormaster/management/HarbormasterManagementRestartWorkflow.php', 'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php', 'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php', @@ -6614,6 +6615,7 @@ 'HarbormasterLogWorker' => 'HarbormasterWorker', 'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow', + 'HarbormasterManagementRebuildLogWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementRestartWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow', diff --git a/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php new file mode 100644 --- /dev/null +++ b/src/applications/harbormaster/management/HarbormasterManagementRebuildLogWorkflow.php @@ -0,0 +1,54 @@ +setName('rebuild-log') + ->setExamples('**rebuild-log** --id __id__ [__options__]') + ->setSynopsis( + pht( + 'Rebuild the file and summary for a log. This is primarily '. + 'intended to make it easier to develop new log summarizers.')) + ->setArguments( + array( + array( + 'name' => 'id', + 'param' => 'id', + 'help' => pht('Log to rebuild.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $log_id = $args->getArg('id'); + if (!$log_id) { + throw new PhutilArgumentUsageException( + pht('Choose a build log to rebuild with "--id".')); + } + + $log = id(new HarbormasterBuildLogQuery()) + ->setViewer($viewer) + ->withIDs(array($log_id)) + ->executeOne(); + if (!$log) { + throw new PhutilArgumentUsageException( + pht( + 'Unable to load build log "%s".', + $log_id)); + } + + PhabricatorWorker::setRunAllTasksInProcess(true); + $log->scheduleRebuild(true); + + echo tsprintf( + "%s\n", + pht('Done.')); + + return 0; + } + +} diff --git a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php --- a/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php +++ b/src/applications/harbormaster/management/HarbormasterManagementWriteLogWorkflow.php @@ -7,14 +7,16 @@ $this ->setName('write-log') ->setExamples('**write-log** --target __id__ [__options__]') - ->setSynopsis(pht('Write a new Harbormaster build log.')) + ->setSynopsis( + pht( + 'Write a new Harbormaster build log. This is primarily intended '. + 'to make development and testing easier.')) ->setArguments( array( array( 'name' => 'target', 'param' => 'id', - 'help' => pht( - 'Build Target ID to attach the log to.'), + 'help' => pht('Build Target ID to attach the log to.'), ), )); } diff --git a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php --- a/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php +++ b/src/applications/harbormaster/phid/HarbormasterBuildLogPHIDType.php @@ -31,6 +31,8 @@ foreach ($handles as $phid => $handle) { $build_log = $objects[$phid]; + + $handle->setName(pht('Build Log %d', $build_log->getID())); } } diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -9,12 +9,13 @@ protected $logType; protected $duration; protected $live; + protected $filePHID; private $buildTarget = self::ATTACHABLE; private $rope; private $isOpen; - const CHUNK_BYTE_LIMIT = 102400; + const CHUNK_BYTE_LIMIT = 1048576; public function __construct() { $this->rope = new PhutilRope(); @@ -60,19 +61,23 @@ ->setLive(0) ->save(); + $this->scheduleRebuild(false); + + return $this; + } + + public function scheduleRebuild($force) { PhabricatorWorker::scheduleTask( 'HarbormasterLogWorker', array( 'logPHID' => $this->getPHID(), + 'force' => $force, ), array( 'objectPHID' => $this->getPHID(), )); - - return $this; } - protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, @@ -85,6 +90,7 @@ 'duration' => 'uint32?', 'live' => 'bool', + 'filePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_buildtarget' => array( @@ -180,7 +186,7 @@ public function newChunkIterator() { return id(new HarbormasterBuildLogChunkIterator($this)) - ->setPageSize(32); + ->setPageSize(8); } private function loadLastChunkInfo() { diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php @@ -5,6 +5,7 @@ private $log; private $cursor; + private $asString; private $min = 0; private $max = PHP_INT_MAX; @@ -27,6 +28,11 @@ return $this; } + public function setAsString($as_string) { + $this->asString = $as_string; + return $this; + } + protected function loadPage() { if ($this->cursor > $this->max) { return array(); @@ -43,7 +49,11 @@ $this->cursor = last($results)->getID() + 1; } - return $results; + if ($this->asString) { + return mpull($results, 'getChunkDisplayText'); + } else { + return $results; + } } } diff --git a/src/applications/harbormaster/worker/HarbormasterLogWorker.php b/src/applications/harbormaster/worker/HarbormasterLogWorker.php --- a/src/applications/harbormaster/worker/HarbormasterLogWorker.php +++ b/src/applications/harbormaster/worker/HarbormasterLogWorker.php @@ -8,15 +8,6 @@ $data = $this->getTaskData(); $log_phid = idx($data, 'logPHID'); - $log = id(new HarbormasterBuildLogQuery()) - ->setViewer($viewer) - ->withPHIDs(array($log_phid)) - ->executeOne(); - if (!$log) { - throw new PhabricatorWorkerPermanentFailureException( - pht('Invalid build log PHID "%s".', $log_phid)); - } - $phid_key = PhabricatorHash::digestToLength($log_phid, 14); $lock_key = "build.log({$phid_key})"; $lock = PhabricatorGlobalLock::newLock($lock_key); @@ -29,6 +20,25 @@ $caught = null; try { + $log = id(new HarbormasterBuildLogQuery()) + ->setViewer($viewer) + ->withPHIDs(array($log_phid)) + ->executeOne(); + if (!$log) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Invalid build log PHID "%s".', + $log_phid)); + } + + if ($log->getLive()) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Log "%s" is still live. Logs can not be finalized until '. + 'they have closed.', + $log_phid)); + } + $this->finalizeBuildLog($log); } catch (Exception $ex) { $caught = $ex; @@ -42,9 +52,51 @@ } private function finalizeBuildLog(HarbormasterBuildLog $log) { + $viewer = $this->getViewer(); + + $data = $this->getTaskData(); + $is_force = idx($data, 'force'); + if ($log->canCompressLog()) { $log->compressLog(); } + + if ($is_force) { + $file_phid = $log->getFilePHID(); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + id(new PhabricatorDestructionEngine()) + ->destroyObject($file); + } + $log + ->setFilePHID(null) + ->save(); + } + } + + if (!$log->getFilePHID()) { + $iterator = $log->newChunkIterator() + ->setAsString(true); + + $source = id(new PhabricatorIteratorFileUploadSource()) + ->setName('harbormaster-log-'.$log->getID().'.log') + ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE) + ->setMIMEType('application/octet-stream') + ->setIterator($iterator); + + $file = $source->uploadFile(); + + $file->attachToObject($log->getPHID()); + + $log + ->setFilePHID($file->getPHID()) + ->save(); + } + } }