diff --git a/resources/sql/autopatches/20180228.log.01.offset.sql b/resources/sql/autopatches/20180228.log.01.offset.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20180228.log.01.offset.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk + ADD headOffset BIGINT UNSIGNED NOT NULL; + +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk + ADD tailOffset BIGINT UNSIGNED NOT NULL; diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -130,7 +130,85 @@ } public function loadData($offset, $length) { - return substr($this->getLogText(), $offset, $length); + $end = ($offset + $length); + + $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere( + 'logID = %d AND headOffset < %d AND tailOffset >= %d + ORDER BY headOffset ASC', + $this->getID(), + $end, + $offset); + + // Make sure that whatever we read out of the database is a single + // contiguous range which contains all of the requested bytes. + $ranges = array(); + foreach ($chunks as $chunk) { + $ranges[] = array( + 'head' => $chunk->getHeadOffset(), + 'tail' => $chunk->getTailOffset(), + ); + } + + $ranges = isort($ranges, 'head'); + $ranges = array_values($ranges); + $count = count($ranges); + for ($ii = 0; $ii < ($count - 1); $ii++) { + if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) { + $ranges[$ii + 1]['head'] = $ranges[$ii]['head']; + unset($ranges[$ii]); + } + } + + if (count($ranges) !== 1) { + $display_ranges = array(); + foreach ($ranges as $range) { + $display_ranges[] = pht( + '(%d - %d)', + $range['head'], + $range['tail']); + } + + if (!$display_ranges) { + $display_ranges[] = pht(''); + } + + throw new Exception( + pht( + 'Attempt to load log bytes (%d - %d) failed: failed to '. + 'load a single contiguous range. Actual ranges: %s.', + implode('; ', $display_ranges))); + } + + $range = head($ranges); + if ($range['head'] > $offset || $range['tail'] < $end) { + throw new Exception( + pht( + 'Attempt to load log bytes (%d - %d) failed: the loaded range '. + '(%d - %d) does not span the requested range.', + $offset, + $end, + $range['head'], + $range['tail'])); + } + + $parts = array(); + foreach ($chunks as $chunk) { + $parts[] = $chunk->getChunkDisplayText(); + } + $parts = implode('', $parts); + + $chop_head = ($offset - $range['head']); + $chop_tail = ($range['tail'] - $end); + + if ($chop_head) { + $parts = substr($parts, $chop_head); + } + + if ($chop_tail) { + $parts = substr($parts, 0, -$chop_tail); + } + + return $parts; } public function getReadPosition($read_offset) { @@ -220,17 +298,18 @@ $this->openTransaction(); + $offset = 0; foreach ($chunks as $chunk) { $rope->append($chunk->getChunkDisplayText()); $chunk->delete(); while ($rope->getByteLength() > $byte_limit) { - $this->writeEncodedChunk($rope, $byte_limit, $mode); + $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); } } while ($rope->getByteLength()) { - $this->writeEncodedChunk($rope, $byte_limit, $mode); + $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); } $this @@ -240,7 +319,12 @@ $this->saveTransaction(); } - private function writeEncodedChunk(PhutilRope $rope, $length, $mode) { + private function writeEncodedChunk( + PhutilRope $rope, + $offset, + $length, + $mode) { + $data = $rope->getPrefixBytes($length); $size = strlen($data); @@ -258,15 +342,22 @@ throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); } - $this->writeChunk($mode, $size, $data); + $this->writeChunk($mode, $offset, $size, $data); $rope->removeBytesFromHead($size); + + return $size; } - private function writeChunk($encoding, $raw_size, $data) { + private function writeChunk($encoding, $offset, $raw_size, $data) { + $head_offset = $offset; + $tail_offset = $offset + $raw_size; + return id(new HarbormasterBuildLogChunk()) ->setLogID($this->getID()) ->setEncoding($encoding) + ->setHeadOffset($head_offset) + ->setTailOffset($tail_offset) ->setSize($raw_size) ->setChunk($data) ->save(); @@ -397,13 +488,23 @@ if ($append_id) { queryfx( $conn_w, - 'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d', + 'UPDATE %T SET + chunk = CONCAT(chunk, %B), + size = %d, + tailOffset = headOffset + %d, + WHERE + id = %d', $chunk_table, $append_data, $prefix_size + $data_size, + $prefix_size + $data_size, $append_id); } else { - $this->writeChunk($encoding_text, $data_size, $append_data); + $this->writeChunk( + $encoding_text, + $this->getByteLength(), + $data_size, + $append_data); } $this->updateLineMap($append_data); diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php @@ -5,6 +5,8 @@ protected $logID; protected $encoding; + protected $headOffset; + protected $tailOffset; protected $size; protected $chunk; @@ -20,6 +22,8 @@ self::CONFIG_COLUMN_SCHEMA => array( 'logID' => 'id', 'encoding' => 'text32', + 'headOffset' => 'uint64', + 'tailOffset' => 'uint64', // T6203/NULLABILITY // Both the type and nullability of this column are crazily wrong. @@ -28,8 +32,8 @@ 'chunk' => 'bytes', ), self::CONFIG_KEY_SCHEMA => array( - 'key_log' => array( - 'columns' => array('logID'), + 'key_offset' => array( + 'columns' => array('logID', 'headOffset', 'tailOffset'), ), ), ) + parent::getConfiguration();