Page MenuHomePhabricator
Files F15555544

D15380.id37077.diff
No OneTemporary
Actions

D15380.id37077.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1114,6 +1114,7 @@
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php',
'HarbormasterLintMessagesController' => 'applications/harbormaster/controller/HarbormasterLintMessagesController.php',
'HarbormasterLintPropertyView' => 'applications/harbormaster/view/HarbormasterLintPropertyView.php',
+ 'HarbormasterManagementArchiveLogsWorkflow' => 'applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php',
'HarbormasterManagementBuildWorkflow' => 'applications/harbormaster/management/HarbormasterManagementBuildWorkflow.php',
'HarbormasterManagementUpdateWorkflow' => 'applications/harbormaster/management/HarbormasterManagementUpdateWorkflow.php',
'HarbormasterManagementWorkflow' => 'applications/harbormaster/management/HarbormasterManagementWorkflow.php',
@@ -5286,6 +5287,7 @@
'HarbormasterLeaseWorkingCopyBuildStepImplementation' => 'HarbormasterBuildStepImplementation',
'HarbormasterLintMessagesController' => 'HarbormasterController',
'HarbormasterLintPropertyView' => 'AphrontView',
+ 'HarbormasterManagementArchiveLogsWorkflow' => 'HarbormasterManagementWorkflow',
'HarbormasterManagementBuildWorkflow' => 'HarbormasterManagementWorkflow',
'HarbormasterManagementUpdateWorkflow' => 'HarbormasterManagementWorkflow',
'HarbormasterManagementWorkflow' => 'PhabricatorManagementWorkflow',
diff --git a/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php b/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php
new file mode 100644
--- /dev/null
+++ b/src/applications/harbormaster/management/HarbormasterManagementArchiveLogsWorkflow.php
@@ -0,0 +1,150 @@
+<?php
+
+final class HarbormasterManagementArchiveLogsWorkflow
+ extends HarbormasterManagementWorkflow {
+
+ protected function didConstruct() {
+ $this
+ ->setName('archive-logs')
+ ->setExamples('**archive-logs** [__options__] --mode __mode__')
+ ->setSynopsis(pht('Compress, decompress, store or destroy build logs.'))
+ ->setArguments(
+ array(
+ array(
+ 'name' => 'mode',
+ 'param' => 'mode',
+ 'help' => pht(
+ 'Use "plain" to remove encoding, or "compress" to compress '.
+ 'logs.'),
+ ),
+ array(
+ 'name' => 'details',
+ 'help' => pht(
+ 'Show more details about operations as they are performed. '.
+ 'Slow! But also very reassuring!'),
+ ),
+ ));
+ }
+
+ public function execute(PhutilArgumentParser $args) {
+ $viewer = $this->getViewer();
+
+ $mode = $args->getArg('mode');
+ if (!$mode) {
+ throw new PhutilArgumentUsageException(
+ pht('Choose an archival mode with --mode.'));
+ }
+
+ $valid_modes = array(
+ 'plain',
+ 'compress',
+ );
+
+ $valid_modes = array_fuse($valid_modes);
+ if (empty($valid_modes[$mode])) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Unknown mode "%s". Valid modes are: %s.',
+ $mode,
+ implode(', ', $valid_modes)));
+ }
+
+ $log_table = new HarbormasterBuildLog();
+ $logs = new LiskMigrationIterator($log_table);
+
+ $show_details = $args->getArg('details');
+
+ if ($show_details) {
+ $total_old = 0;
+ $total_new = 0;
+ }
+
+ foreach ($logs as $log) {
+ echo tsprintf(
+ "%s\n",
+ pht('Processing Harbormaster build log #%d...', $log->getID()));
+
+ if ($show_details) {
+ $old_stats = $this->computeDetails($log);
+ }
+
+ switch ($mode) {
+ case 'plain':
+ $log->decompressLog();
+ break;
+ case 'compress':
+ $log->compressLog();
+ break;
+ }
+
+ if ($show_details) {
+ $new_stats = $this->computeDetails($log);
+ $this->printStats($old_stats, $new_stats);
+
+ $total_old += $old_stats['bytes'];
+ $total_new += $new_stats['bytes'];
+ }
+ }
+
+ if ($show_details) {
+ echo tsprintf(
+ "%s\n",
+ pht(
+ 'Done. Total byte size of affected logs: %s -> %s.',
+ new PhutilNumber($total_old),
+ new PhutilNumber($total_new)));
+ }
+
+ return 0;
+ }
+
+ private function computeDetails(HarbormasterBuildLog $log) {
+ $bytes = 0;
+ $chunks = 0;
+ $hash = hash_init('sha1');
+
+ foreach ($log->newChunkIterator() as $chunk) {
+ $bytes += strlen($chunk->getChunk());
+ $chunks++;
+ hash_update($hash, $chunk->getChunkDisplayText());
+ }
+
+ return array(
+ 'bytes' => $bytes,
+ 'chunks' => $chunks,
+ 'hash' => hash_final($hash),
+ );
+ }
+
+ private function printStats(array $old_stats, array $new_stats) {
+ echo tsprintf(
+ " %s\n",
+ pht(
+ '%s: %s -> %s',
+ pht('Stored Bytes'),
+ new PhutilNumber($old_stats['bytes']),
+ new PhutilNumber($new_stats['bytes'])));
+
+ echo tsprintf(
+ " %s\n",
+ pht(
+ '%s: %s -> %s',
+ pht('Stored Chunks'),
+ new PhutilNumber($old_stats['chunks']),
+ new PhutilNumber($new_stats['chunks'])));
+
+ echo tsprintf(
+ " %s\n",
+ pht(
+ '%s: %s -> %s',
+ pht('Data Hash'),
+ $old_stats['hash'],
+ $new_stats['hash']));
+
+ if ($old_stats['hash'] !== $new_stats['hash']) {
+ throw new Exception(
+ pht('Log data hashes differ! Something is tragically wrong!'));
+ }
+ }
+
+}
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -52,9 +52,9 @@
throw new Exception(pht('This build log is not open!'));
}
- // TODO: Encode the log contents in a gzipped format.
-
- $this->reload();
+ if ($this->canCompressLog()) {
+ $this->compressLog();
+ }
$start = $this->getDateCreated();
$now = PhabricatorTime::getNow();
@@ -135,20 +135,15 @@
}
$conn_w = $this->establishConnection('w');
- $tail = queryfx_one(
- $conn_w,
- 'SELECT id, size, encoding FROM %T WHERE logID = %d
- ORDER BY id DESC LIMIT 1',
- $chunk_table,
- $this->getID());
+ $last = $this->loadLastChunkInfo();
$can_append =
- ($tail) &&
- ($tail['encoding'] == $encoding_text) &&
- ($tail['size'] < $chunk_limit);
+ ($last) &&
+ ($last['encoding'] == $encoding_text) &&
+ ($last['size'] < $chunk_limit);
if ($can_append) {
- $append_id = $tail['id'];
- $prefix_size = $tail['size'];
+ $append_id = $last['id'];
+ $prefix_size = $last['size'];
} else {
$append_id = null;
$prefix_size = 0;
@@ -167,23 +162,28 @@
$prefix_size + $data_size,
$append_id);
} else {
- queryfx(
- $conn_w,
- 'INSERT INTO %T (logID, encoding, size, chunk)
- VALUES (%d, %s, %d, %B)',
- $chunk_table,
- $this->getID(),
- $encoding_text,
- $data_size,
- $append_data);
+ $this->writeChunk($encoding_text, $data_size, $append_data);
}
- $rope->removeBytesFromHead(strlen($append_data));
+ $rope->removeBytesFromHead($data_size);
}
}
public function newChunkIterator() {
- return new HarbormasterBuildLogChunkIterator($this);
+ return id(new HarbormasterBuildLogChunkIterator($this))
+ ->setPageSize(32);
+ }
+
+ private function loadLastChunkInfo() {
+ $chunk_table = new HarbormasterBuildLogChunk();
+ $conn_w = $chunk_table->establishConnection('w');
+
+ return queryfx_one(
+ $conn_w,
+ 'SELECT id, size, encoding FROM %T WHERE logID = %d
+ ORDER BY id DESC LIMIT 1',
+ $chunk_table->getTableName(),
+ $this->getID());
}
public function getLogText() {
@@ -199,6 +199,82 @@
return implode('', $full_text);
}
+ private function canCompressLog() {
+ return function_exists('gzdeflate');
+ }
+
+ public function compressLog() {
+ $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP);
+ }
+
+ public function decompressLog() {
+ $this->processLog(HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT);
+ }
+
+ private function processLog($mode) {
+ $chunks = $this->newChunkIterator();
+
+ // NOTE: Because we're going to insert new chunks, we need to stop the
+ // iterator once it hits the final chunk which currently exists. Otherwise,
+ // it may start consuming chunks we just wrote and run forever.
+ $last = $this->loadLastChunkInfo();
+ if ($last) {
+ $chunks->setRange(null, $last['id']);
+ }
+
+ $byte_limit = self::CHUNK_BYTE_LIMIT;
+ $rope = new PhutilRope();
+
+ $this->openTransaction();
+
+ foreach ($chunks as $chunk) {
+ $rope->append($chunk->getChunkDisplayText());
+ $chunk->delete();
+
+ while ($rope->getByteLength() > $byte_limit) {
+ $this->writeEncodedChunk($rope, $byte_limit, $mode);
+ }
+ }
+
+ while ($rope->getByteLength()) {
+ $this->writeEncodedChunk($rope, $byte_limit, $mode);
+ }
+
+ $this->saveTransaction();
+ }
+
+ private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
+ $data = $rope->getPrefixBytes($length);
+ $size = strlen($data);
+
+ switch ($mode) {
+ case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT:
+ // Do nothing.
+ break;
+ case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP:
+ $data = gzdeflate($data);
+ if ($data === false) {
+ throw new Exception(pht('Failed to gzdeflate() log data!'));
+ }
+ break;
+ default:
+ throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
+ }
+
+ $this->writeChunk($mode, $size, $data);
+
+ $rope->removeBytesFromHead($size);
+ }
+
+ private function writeChunk($encoding, $raw_size, $data) {
+ return id(new HarbormasterBuildLogChunk())
+ ->setLogID($this->getID())
+ ->setEncoding($encoding)
+ ->setSize($raw_size)
+ ->setChunk($data)
+ ->save();
+ }
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
@@ -8,15 +8,15 @@
protected $size;
protected $chunk;
-
- /**
- * The log is encoded as plain text.
- */
const CHUNK_ENCODING_TEXT = 'text';
+ const CHUNK_ENCODING_GZIP = 'gzip';
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
+ self::CONFIG_BINARY => array(
+ 'chunk' => true,
+ ),
self::CONFIG_COLUMN_SCHEMA => array(
'logID' => 'id',
'encoding' => 'text32',
@@ -43,6 +43,12 @@
case self::CHUNK_ENCODING_TEXT:
// Do nothing, data is already plaintext.
break;
+ case self::CHUNK_ENCODING_GZIP:
+ $data = gzinflate($data);
+ if ($data === false) {
+ throw new Exception(pht('Unable to inflate log chunk!'));
+ }
+ break;
default:
throw new Exception(
pht('Unknown log chunk encoding ("%s")!', $encoding));
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunkIterator.php
@@ -6,27 +6,41 @@
private $log;
private $cursor;
+ private $min = 0;
+ private $max = PHP_INT_MAX;
+
public function __construct(HarbormasterBuildLog $log) {
$this->log = $log;
}
protected function didRewind() {
- $this->cursor = 0;
+ $this->cursor = $this->min;
}
public function key() {
return $this->current()->getID();
}
+ public function setRange($min, $max) {
+ $this->min = (int)$min;
+ $this->max = (int)$max;
+ return $this;
+ }
+
protected function loadPage() {
+ if ($this->cursor > $this->max) {
+ return array();
+ }
+
$results = id(new HarbormasterBuildLogChunk())->loadAllWhere(
- 'logID = %d AND id > %d ORDER BY id ASC LIMIT %d',
+ 'logID = %d AND id >= %d AND id <= %d ORDER BY id ASC LIMIT %d',
$this->log->getID(),
$this->cursor,
+ $this->max,
$this->getPageSize());
if ($results) {
- $this->cursor = last($results)->getID();
+ $this->cursor = last($results)->getID() + 1;
}
return $results;

File Metadata

Mime Type
text/plain
Expires
Tue, Apr 29, 11:59 AM (1 w, 3 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7715832
Default Alt Text
D15380.id37077.diff (13 KB)

Event Timeline

Phabricator · Built by Phacility