diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1114,6 +1114,7 @@ 'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php', 'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php', 'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php', + 'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php', 'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php', 'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php', 'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php', @@ -5287,6 +5288,7 @@ 'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation', 'HarbormasterLintMessagesController' => 'HarbormasterController', 'HarbormasterLintPropertyView' => 'AphrontView', + 'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow', 'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow', diff --git a/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php new file mode 100644 --- /dev/null +++ b/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php @@ -0,0 +1,150 @@ +setName('archive-logs') + ->setExamples('**archive-logs** [__options__] --mode __mode__') + ->setSynopsis(pht('Compress, decompress, store or destroy build logs.')) + ->setArguments( + array( + array( + 'name' => 'mode', + 'param' => 'mode', + 'help' => pht( + 'Use "plain" to remove encoding, or "compress" to compress '. + 'logs.'), + ), + array( + 'name' => 'details', + 'help' => pht( + 'Show more details about operations as they are performed. '. + 'Slow! But also very reassuring!'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $viewer = $this->getViewer(); + + $mode = $args->getArg('mode'); + if (!$mode) { + throw new PhutilArgumentUsageException( + pht('Choose an archival mode with --mode.')); + } + + $valid_modes = array( + 'plain', + 'compress', + ); + + $valid_modes = array_fuse($valid_modes); + if (empty($valid_modes[$mode])) { + throw new PhutilArgumentUsageException( + pht( + 'Unknown mode "%s". Valid modes are: %s.', + $mode, + implode(', ', $valid_modes))); + } + + $log_table = new HarbormasterBuildLog(); + $logs = new LiskMigrationIterator($log_table); + + $show_details = $args->getArg('details'); + + if ($show_details) { + $total_old = 0; + $total_new = 0; + } + + foreach ($logs as $log) { + echo tsprintf( + "%s\n", + pht('Processing Harbormaster build log #%d...', $log->getID())); + + if ($show_details) { + $old_stats = $this->computeDetails($log); + } + + switch ($mode) { + case 'plain': + $log->decompressLog(); + break; + case 'compress': + $log->compressLog(); + break; + } + + if ($show_details) { + $new_stats = $this->computeDetails($log); + $this->printStats($old_stats, $new_stats); + + $total_old += $old_stats['bytes']; + $total_new += $new_stats['bytes']; + } + } + + if ($show_details) { + echo tsprintf( + "%s\n", + pht( + 'Done. Total byte size of affected logs: %s -> %s.', + new PhutilNumber($total_old), + new PhutilNumber($total_new))); + } + + return 0; + } + + private function computeDetails(HarbormasterBuildLog $log) { + $bytes = 0; + $chunks = 0; + $hash = hash_init('sha1'); + + foreach ($log->newChunkIterator() as $chunk) { + $bytes += strlen($chunk->getChunk()); + $chunks++; + hash_update($hash, $chunk->getChunkDisplayText()); + } + + return array( + 'bytes' => $bytes, + 'chunks' => $chunks, + 'hash' => hash_final($hash), + ); + } + + private function printStats(array $old_stats, array $new_stats) { + echo tsprintf( + " %s\n", + pht( + '%s: %s -> %s', + pht('Stored Bytes'), + new PhutilNumber($old_stats['bytes']), + new PhutilNumber($new_stats['bytes']))); + + echo tsprintf( + " %s\n", + pht( + '%s: %s -> %s', + pht('Stored Chunks'), + new PhutilNumber($old_stats['chunks']), + new PhutilNumber($new_stats['chunks']))); + + echo tsprintf( + " %s\n", + pht( + '%s: %s -> %s', + pht('Data Hash'), + $old_stats['hash'], + $new_stats['hash'])); + + if ($old_stats['hash'] !== $new_stats['hash']) { + throw new Exception( + pht('Log data hashes differ! Something is tragically wrong!')); + } + } + +} diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -52,9 +52,9 @@ throw new Exception(pht('This build log is not open!')); } - // TODO: Encode the log contents in a gzipped format. - - $this->reload(); + if ($this->canCompressLog()) { + $this->compressLog(); + } $start = $this->getDateCreated(); $now = PhabricatorTime::getNow(); @@ -135,20 +135,15 @@ } $conn_w = $this->establishConnection('w'); - $tail = queryfx_one( - $conn_w, - 'SELECT id, size, encoding FROM %T WHERE logID = %d - ORDER BY id DESC LIMIT 1', - $chunk_table, - $this->getID()); + $last = $this->loadLastChunkInfo(); $can_append = - ($tail) && - ($tail['encoding'] == $encoding_text) && - ($tail['size'] < $chunk_limit); + ($last) && + ($last['encoding'] == $encoding_text) && + ($last['size'] < $chunk_limit); if ($can_append) { - $append_id = $tail['id']; - $prefix_size = $tail['size']; + $append_id = $last['id']; + $prefix_size = $last['size']; } else { $append_id = null; $prefix_size = 0; @@ -167,23 +162,28 @@ $prefix_size + $data_size, $append_id); } else { - queryfx( - $conn_w, - 'INSERT INTO %T (logID, encoding, size, chunk) - VALUES (%d, %s, %d, %B)', - $chunk_table, - $this->getID(), - $encoding_text, - $data_size, - $append_data); + $this->writeChunk($encoding_text, $data_size, $append_data); } - $rope->removeBytesFromHead(strlen($append_data)); + $rope->removeBytesFromHead($data_size); } } public function newChunkIterator() { - return new HarbormasterBuildLogChunkIterator($this); + return id(new HarbormasterBuildLogChunkIterator($this)) + ->setPageSize(32); + } + + private function loadLastChunkInfo() { + $chunk_table = new HarbormasterBuildLogChunk(); + $conn_w = $chunk_table->establishConnection('w'); + + return queryfx_one( + $conn_w, + 'SELECT id, size, encoding FROM %T WHERE logID = %d + ORDER BY id DESC LIMIT 1', + $chunk_table->getTableName(), + $this->getID()); } public function getLogText() { @@ -199,6 +199,82 @@ return implode('', $full_text); } + private function canCompressLog() { + return function_exists('gzdeflate'); + } + + public function compressLog() { + $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP); + } + + public function decompressLog() { + $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT); + } + + private function processLog($mode) { + $chunks = $this->newChunkIterator(); + + // NOTE: Because we're going to insert new chunks, we need to stop the + // iterator once it hits the final chunk which currently exists. Otherwise, + // it may start consuming chunks we just wrote and run forever. + $last = $this->loadLastChunkInfo(); + if ($last) { + $chunks->setRange(null, $last['id']); + } + + $byte_limit = self::CHUNK_BYTE_LIMIT; + $rope = new PhutilRope(); + + $this->openTransaction(); + + foreach ($chunks as $chunk) { + $rope->append($chunk->getChunkDisplayText()); + $chunk->delete(); + + while ($rope->getByteLength() > $byte_limit) { + $this->writeEncodedChunk($rope, $byte_limit, $mode); + } + } + + while ($rope->getByteLength()) { + $this->writeEncodedChunk($rope, $byte_limit, $mode); + } + + $this->saveTransaction(); + } + + private function writeEncodedChunk(PhutilRope $rope, $length, $mode) { + $data = $rope->getPrefixBytes($length); + $size = strlen($data); + + switch ($mode) { + case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT: + // Do nothing. + break; + case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP: + $data = gzdeflate($data); + if ($data === false) { + throw new Exception(pht('Failed to gzdeflate() log data!')); + } + break; + default: + throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); + } + + $this->writeChunk($mode, $size, $data); + + $rope->removeBytesFromHead($size); + } + + private function writeChunk($encoding, $raw_size, $data) { + return id(new HarbormasterBuildLogChunk()) + ->setLogID($this->getID()) + ->setEncoding($encoding) + ->setSize($raw_size) + ->setChunk($data) + ->save(); + } + /* -( PhabricatorPolicyInterface )----------------------------------------- */ diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php @@ -8,15 +8,15 @@ protected $size; protected $chunk; - - /** - * The log is encoded as plain text. - */ const CHUNK_ENCODING_TEXT = 'text'; + const CHUNK_ENCODING_GZIP = 'gzip'; protected function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_BINARY => array( + 'chunk' => true, + ), self::CONFIG_COLUMN_SCHEMA => array( 'logID' => 'id', 'encoding' => 'text32', @@ -43,6 +43,12 @@ case self::CHUNK_ENCODING_TEXT: // Do nothing, data is already plaintext. break; + case self::CHUNK_ENCODING_GZIP: + $data = gzinflate($data); + if ($data === false) { + throw new Exception(pht('Unable to inflate log chunk!')); + } + break; default: throw new Exception( pht('Unknown log chunk encoding ("%s")!', $encoding)); diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php @@ -6,27 +6,41 @@ private $log; private $cursor; + private $min = 0; + private $max = PHP_INT_MAX; + public function __construct(HarbormasterBuildLog $log) { $this->log = $log; } protected function didRewind() { - $this->cursor = 0; + $this->cursor = $this->min; } public function key() { return $this->current()->getID(); } + public function setRange($min, $max) { + $this->min = (int)$min; + $this->max = (int)$max; + return $this; + } + protected function loadPage() { + if ($this->cursor > $this->max) { + return array(); + } + $results = id(new HarbormasterBuildLogChunk())->loadAllWhere( - 'logID = %d AND id > %d ORDER BY id ASC LIMIT %d', + 'logID = %d AND id >= %d AND id <= %d ORDER BY id ASC LIMIT %d', $this->log->getID(), $this->cursor, + $this->max, $this->getPageSize()); if ($results) { - $this->cursor = last($results)->getID(); + $this->cursor = last($results)->getID() + 1; } return $results;