Changeset View
Changeset View
Standalone View
Standalone View
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
| Show First 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | return queryfx_one( | ||||
| $conn_w, | $conn_w, | ||||
| 'SELECT id, size, encoding FROM %T WHERE logID = %d | 'SELECT id, size, encoding FROM %T WHERE logID = %d | ||||
| ORDER BY id DESC LIMIT 1', | ORDER BY id DESC LIMIT 1', | ||||
| $chunk_table->getTableName(), | $chunk_table->getTableName(), | ||||
| $this->getID()); | $this->getID()); | ||||
| } | } | ||||
| public function loadData($offset, $length) { | public function loadData($offset, $length) { | ||||
| return substr($this->getLogText(), $offset, $length); | $end = ($offset + $length); | ||||
| $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere( | |||||
| 'logID = %d AND headOffset < %d AND tailOffset >= %d | |||||
| ORDER BY headOffset ASC', | |||||
| $this->getID(), | |||||
| $end, | |||||
| $offset); | |||||
| // Make sure that whatever we read out of the database is a single | |||||
| // contiguous range which contains all of the requested bytes. | |||||
| $ranges = array(); | |||||
| foreach ($chunks as $chunk) { | |||||
| $ranges[] = array( | |||||
| 'head' => $chunk->getHeadOffset(), | |||||
| 'tail' => $chunk->getTailOffset(), | |||||
| ); | |||||
| } | |||||
| $ranges = isort($ranges, 'head'); | |||||
| $ranges = array_values($ranges); | |||||
| $count = count($ranges); | |||||
| for ($ii = 0; $ii < ($count - 1); $ii++) { | |||||
| if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) { | |||||
| $ranges[$ii + 1]['head'] = $ranges[$ii]['head']; | |||||
| unset($ranges[$ii]); | |||||
| } | |||||
| } | |||||
| if (count($ranges) !== 1) { | |||||
| $display_ranges = array(); | |||||
| foreach ($ranges as $range) { | |||||
| $display_ranges[] = pht( | |||||
| '(%d - %d)', | |||||
| $range['head'], | |||||
| $range['tail']); | |||||
| } | |||||
| if (!$display_ranges) { | |||||
| $display_ranges[] = pht('<null>'); | |||||
| } | |||||
| throw new Exception( | |||||
| pht( | |||||
| 'Attempt to load log bytes (%d - %d) failed: failed to '. | |||||
| 'load a single contiguous range. Actual ranges: %s.', | |||||
| implode('; ', $display_ranges))); | |||||
| } | |||||
| $range = head($ranges); | |||||
| if ($range['head'] > $offset || $range['tail'] < $end) { | |||||
| throw new Exception( | |||||
| pht( | |||||
| 'Attempt to load log bytes (%d - %d) failed: the loaded range '. | |||||
| '(%d - %d) does not span the requested range.', | |||||
| $offset, | |||||
| $end, | |||||
| $range['head'], | |||||
| $range['tail'])); | |||||
| } | |||||
| $parts = array(); | |||||
| foreach ($chunks as $chunk) { | |||||
| $parts[] = $chunk->getChunkDisplayText(); | |||||
| } | |||||
| $parts = implode('', $parts); | |||||
| $chop_head = ($offset - $range['head']); | |||||
| $chop_tail = ($range['tail'] - $end); | |||||
| if ($chop_head) { | |||||
| $parts = substr($parts, $chop_head); | |||||
| } | |||||
| if ($chop_tail) { | |||||
| $parts = substr($parts, 0, -$chop_tail); | |||||
| } | |||||
| return $parts; | |||||
| } | } | ||||
| public function getReadPosition($read_offset) { | public function getReadPosition($read_offset) { | ||||
| $position = array(0, 0); | $position = array(0, 0); | ||||
| $map = $this->getLineMap(); | $map = $this->getLineMap(); | ||||
| if (!$map) { | if (!$map) { | ||||
| throw new Exception(pht('No line map.')); | throw new Exception(pht('No line map.')); | ||||
| ▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines | if ($last) { | ||||
| $chunks->setRange(null, $last['id']); | $chunks->setRange(null, $last['id']); | ||||
| } | } | ||||
| $byte_limit = self::CHUNK_BYTE_LIMIT; | $byte_limit = self::CHUNK_BYTE_LIMIT; | ||||
| $rope = new PhutilRope(); | $rope = new PhutilRope(); | ||||
| $this->openTransaction(); | $this->openTransaction(); | ||||
| $offset = 0; | |||||
| foreach ($chunks as $chunk) { | foreach ($chunks as $chunk) { | ||||
| $rope->append($chunk->getChunkDisplayText()); | $rope->append($chunk->getChunkDisplayText()); | ||||
| $chunk->delete(); | $chunk->delete(); | ||||
| while ($rope->getByteLength() > $byte_limit) { | while ($rope->getByteLength() > $byte_limit) { | ||||
| $this->writeEncodedChunk($rope, $byte_limit, $mode); | $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); | ||||
| } | } | ||||
| } | } | ||||
| while ($rope->getByteLength()) { | while ($rope->getByteLength()) { | ||||
| $this->writeEncodedChunk($rope, $byte_limit, $mode); | $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); | ||||
| } | } | ||||
| $this | $this | ||||
| ->setChunkFormat($mode) | ->setChunkFormat($mode) | ||||
| ->save(); | ->save(); | ||||
| $this->saveTransaction(); | $this->saveTransaction(); | ||||
| } | } | ||||
| private function writeEncodedChunk(PhutilRope $rope, $length, $mode) { | private function writeEncodedChunk( | ||||
| PhutilRope $rope, | |||||
| $offset, | |||||
| $length, | |||||
| $mode) { | |||||
| $data = $rope->getPrefixBytes($length); | $data = $rope->getPrefixBytes($length); | ||||
| $size = strlen($data); | $size = strlen($data); | ||||
| switch ($mode) { | switch ($mode) { | ||||
| case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT: | case HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT: | ||||
| // Do nothing. | // Do nothing. | ||||
| break; | break; | ||||
| case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP: | case HarbormasterBuildLogChunk::CHUNK_ENCODING_GZIP: | ||||
| $data = gzdeflate($data); | $data = gzdeflate($data); | ||||
| if ($data === false) { | if ($data === false) { | ||||
| throw new Exception(pht('Failed to gzdeflate() log data!')); | throw new Exception(pht('Failed to gzdeflate() log data!')); | ||||
| } | } | ||||
| break; | break; | ||||
| default: | default: | ||||
| throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); | throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); | ||||
| } | } | ||||
| $this->writeChunk($mode, $size, $data); | $this->writeChunk($mode, $offset, $size, $data); | ||||
| $rope->removeBytesFromHead($size); | $rope->removeBytesFromHead($size); | ||||
| return $size; | |||||
| } | } | ||||
| private function writeChunk($encoding, $raw_size, $data) { | private function writeChunk($encoding, $offset, $raw_size, $data) { | ||||
| $head_offset = $offset; | |||||
| $tail_offset = $offset + $raw_size; | |||||
| return id(new HarbormasterBuildLogChunk()) | return id(new HarbormasterBuildLogChunk()) | ||||
| ->setLogID($this->getID()) | ->setLogID($this->getID()) | ||||
| ->setEncoding($encoding) | ->setEncoding($encoding) | ||||
| ->setHeadOffset($head_offset) | |||||
| ->setTailOffset($tail_offset) | |||||
| ->setSize($raw_size) | ->setSize($raw_size) | ||||
| ->setChunk($data) | ->setChunk($data) | ||||
| ->save(); | ->save(); | ||||
| } | } | ||||
| /* -( Writing )------------------------------------------------------------ */ | /* -( Writing )------------------------------------------------------------ */ | ||||
| ▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | while (true) { | ||||
| $data_limit = ($chunk_limit - $prefix_size); | $data_limit = ($chunk_limit - $prefix_size); | ||||
| $append_data = $rope->getPrefixBytes($data_limit); | $append_data = $rope->getPrefixBytes($data_limit); | ||||
| $data_size = strlen($append_data); | $data_size = strlen($append_data); | ||||
| $this->openTransaction(); | $this->openTransaction(); | ||||
| if ($append_id) { | if ($append_id) { | ||||
| queryfx( | queryfx( | ||||
| $conn_w, | $conn_w, | ||||
| 'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d', | 'UPDATE %T SET | ||||
| chunk = CONCAT(chunk, %B), | |||||
| size = %d, | |||||
| tailOffset = headOffset + %d, | |||||
| WHERE | |||||
| id = %d', | |||||
| $chunk_table, | $chunk_table, | ||||
| $append_data, | $append_data, | ||||
| $prefix_size + $data_size, | $prefix_size + $data_size, | ||||
| $prefix_size + $data_size, | |||||
| $append_id); | $append_id); | ||||
| } else { | } else { | ||||
| $this->writeChunk($encoding_text, $data_size, $append_data); | $this->writeChunk( | ||||
| $encoding_text, | |||||
| $this->getByteLength(), | |||||
| $data_size, | |||||
| $append_data); | |||||
| } | } | ||||
| $this->updateLineMap($append_data); | $this->updateLineMap($append_data); | ||||
| $this->save(); | $this->save(); | ||||
| $this->saveTransaction(); | $this->saveTransaction(); | ||||
| $rope->removeBytesFromHead($data_size); | $rope->removeBytesFromHead($data_size); | ||||
| ▲ Show 20 Lines • Show All 202 Lines • Show Last 20 Lines | |||||