Page MenuHomePhabricator
Files F14237636

D19149.id45874.diff
No OneTemporary
Actions

D19149.id45874.diff
View Options

diff --git a/resources/sql/autopatches/20180228.log.01.offset.sql b/resources/sql/autopatches/20180228.log.01.offset.sql
new file mode 100644
--- /dev/null
+++ b/resources/sql/autopatches/20180228.log.01.offset.sql
@@ -0,0 +1,5 @@
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+ ADD headOffset BIGINT UNSIGNED NOT NULL;
+
+ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk
+ ADD tailOffset BIGINT UNSIGNED NOT NULL;
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
@@ -130,7 +130,85 @@
}
public function loadData($offset, $length) {
- return substr($this->getLogText(), $offset, $length);
+ $end = ($offset + $length);
+
+ $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere(
+ 'logID = %d AND headOffset < %d AND tailOffset >= %d
+ ORDER BY headOffset ASC',
+ $this->getID(),
+ $end,
+ $offset);
+
+ // Make sure that whatever we read out of the database is a single
+ // contiguous range which contains all of the requested bytes.
+ $ranges = array();
+ foreach ($chunks as $chunk) {
+ $ranges[] = array(
+ 'head' => $chunk->getHeadOffset(),
+ 'tail' => $chunk->getTailOffset(),
+ );
+ }
+
+ $ranges = isort($ranges, 'head');
+ $ranges = array_values($ranges);
+ $count = count($ranges);
+ for ($ii = 0; $ii < ($count - 1); $ii++) {
+ if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) {
+ $ranges[$ii + 1]['head'] = $ranges[$ii]['head'];
+ unset($ranges[$ii]);
+ }
+ }
+
+ if (count($ranges) !== 1) {
+ $display_ranges = array();
+ foreach ($ranges as $range) {
+ $display_ranges[] = pht(
+ '(%d - %d)',
+ $range['head'],
+ $range['tail']);
+ }
+
+ if (!$display_ranges) {
+ $display_ranges[] = pht('<null>');
+ }
+
+ throw new Exception(
+ pht(
+ 'Attempt to load log bytes (%d - %d) failed: failed to '.
+ 'load a single contiguous range. Actual ranges: %s.',
+ implode('; ', $display_ranges)));
+ }
+
+ $range = head($ranges);
+ if ($range['head'] > $offset || $range['tail'] < $end) {
+ throw new Exception(
+ pht(
+ 'Attempt to load log bytes (%d - %d) failed: the loaded range '.
+ '(%d - %d) does not span the requested range.',
+ $offset,
+ $end,
+ $range['head'],
+ $range['tail']));
+ }
+
+ $parts = array();
+ foreach ($chunks as $chunk) {
+ $parts[] = $chunk->getChunkDisplayText();
+ }
+ $parts = implode('', $parts);
+
+ $chop_head = ($offset - $range['head']);
+ $chop_tail = ($range['tail'] - $end);
+
+ if ($chop_head) {
+ $parts = substr($parts, $chop_head);
+ }
+
+ if ($chop_tail) {
+ $parts = substr($parts, 0, -$chop_tail);
+ }
+
+ return $parts;
}
public function getReadPosition($read_offset) {
@@ -220,17 +298,18 @@
$this->openTransaction();
+ $offset = 0;
foreach ($chunks as $chunk) {
$rope->append($chunk->getChunkDisplayText());
$chunk->delete();
while ($rope->getByteLength() > $byte_limit) {
- $this->writeEncodedChunk($rope, $byte_limit, $mode);
+ $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
}
}
while ($rope->getByteLength()) {
- $this->writeEncodedChunk($rope, $byte_limit, $mode);
+ $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode);
}
$this
@@ -240,7 +319,12 @@
$this->saveTransaction();
}
- private function writeEncodedChunk(PhutilRope $rope, $length, $mode) {
+ private function writeEncodedChunk(
+ PhutilRope $rope,
+ $offset,
+ $length,
+ $mode) {
+
$data = $rope->getPrefixBytes($length);
$size = strlen($data);
@@ -258,15 +342,22 @@
throw new Exception(pht('Unknown chunk encoding "%s"!', $mode));
}
- $this->writeChunk($mode, $size, $data);
+ $this->writeChunk($mode, $offset, $size, $data);
$rope->removeBytesFromHead($size);
+
+ return $size;
}
- private function writeChunk($encoding, $raw_size, $data) {
+ private function writeChunk($encoding, $offset, $raw_size, $data) {
+ $head_offset = $offset;
+ $tail_offset = $offset + $raw_size;
+
return id(new HarbormasterBuildLogChunk())
->setLogID($this->getID())
->setEncoding($encoding)
+ ->setHeadOffset($head_offset)
+ ->setTailOffset($tail_offset)
->setSize($raw_size)
->setChunk($data)
->save();
@@ -397,13 +488,23 @@
if ($append_id) {
queryfx(
$conn_w,
- 'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
+ 'UPDATE %T SET
+ chunk = CONCAT(chunk, %B),
+ size = %d,
+ tailOffset = headOffset + %d,
+ WHERE
+ id = %d',
$chunk_table,
$append_data,
$prefix_size + $data_size,
+ $prefix_size + $data_size,
$append_id);
} else {
- $this->writeChunk($encoding_text, $data_size, $append_data);
+ $this->writeChunk(
+ $encoding_text,
+ $this->getByteLength(),
+ $data_size,
+ $append_data);
}
$this->updateLineMap($append_data);
diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
--- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
+++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php
@@ -5,6 +5,8 @@
protected $logID;
protected $encoding;
+ protected $headOffset;
+ protected $tailOffset;
protected $size;
protected $chunk;
@@ -20,6 +22,8 @@
self::CONFIG_COLUMN_SCHEMA => array(
'logID' => 'id',
'encoding' => 'text32',
+ 'headOffset' => 'uint64',
+ 'tailOffset' => 'uint64',
// T6203/NULLABILITY
// Both the type and nullability of this column are crazily wrong.
@@ -28,8 +32,8 @@
'chunk' => 'bytes',
),
self::CONFIG_KEY_SCHEMA => array(
- 'key_log' => array(
- 'columns' => array('logID'),
+ 'key_offset' => array(
+ 'columns' => array('logID', 'headOffset', 'tailOffset'),
),
),
) + parent::getConfiguration();

File Metadata

Mime Type
text/plain
Expires
Fri, Dec 13, 1:22 PM (8 h, 36 m)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
6872215
Default Alt Text
D19149.id45874.diff (6 KB)

Event Timeline

Phabricator · Built by Phacility