diff --git a/src/applications/files/diff/PhabricatorDocumentEngineBlocks.php b/src/applications/files/diff/PhabricatorDocumentEngineBlocks.php index d07f341815..3897af2f02 100644 --- a/src/applications/files/diff/PhabricatorDocumentEngineBlocks.php +++ b/src/applications/files/diff/PhabricatorDocumentEngineBlocks.php @@ -1,251 +1,252 @@ rangeMin = $min; $this->rangeMax = $max; return $this; } public function setRevealedIndexes(array $indexes) { $this->revealedIndexes = $indexes; return $this; } public function getLayoutAvailableRowCount() { if ($this->layoutAvailableRowCount === null) { throw new PhutilInvalidStateException('new...Layout'); } return $this->layoutAvailableRowCount; } public function addMessage($message) { $this->messages[] = $message; return $this; } public function getMessages() { return $this->messages; } public function addBlockList( PhabricatorDocumentRef $ref = null, array $blocks = array()) { assert_instances_of($blocks, 'PhabricatorDocumentEngineBlock'); $this->lists[] = array( 'ref' => $ref, 'blocks' => array_values($blocks), ); return $this; } public function getDocumentRefs() { return ipull($this->lists, 'ref'); } public function newTwoUpLayout() { $rows = array(); $lists = $this->lists; if (count($lists) != 2) { + $this->layoutAvailableRowCount = 0; return array(); } $specs = array(); foreach ($this->lists as $list) { $specs[] = $this->newDiffSpec($list['blocks']); } $old_map = $specs[0]['map']; $new_map = $specs[1]['map']; $old_list = $specs[0]['list']; $new_list = $specs[1]['list']; $changeset = id(new PhabricatorDifferenceEngine()) ->generateChangesetFromFileContent($old_list, $new_list); $hunk_parser = id(new DifferentialHunkParser()) ->parseHunksForLineData($changeset->getHunks()) ->reparseHunksForSpecialAttributes(); $hunk_parser->generateVisibleBlocksMask(2); $mask = $hunk_parser->getVisibleLinesMask(); $old_lines = $hunk_parser->getOldLines(); $new_lines = $hunk_parser->getNewLines(); $rows = array(); $count = count($old_lines); for ($ii = 0; $ii < $count; $ii++) { $old_line = idx($old_lines, $ii); $new_line = idx($new_lines, $ii); $is_visible = !empty($mask[$ii]); if ($old_line) { $old_hash = rtrim($old_line['text'], "\n"); if (!strlen($old_hash)) { // This can happen when one of the sources has no blocks. $old_block = null; } else { $old_block = array_shift($old_map[$old_hash]); $old_block ->setDifferenceType($old_line['type']) ->setIsVisible($is_visible); } } else { $old_block = null; } if ($new_line) { $new_hash = rtrim($new_line['text'], "\n"); if (!strlen($new_hash)) { $new_block = null; } else { $new_block = array_shift($new_map[$new_hash]); $new_block ->setDifferenceType($new_line['type']) ->setIsVisible($is_visible); } } else { $new_block = null; } // If both lists are empty, we may generate a row which has two empty // blocks. if (!$old_block && !$new_block) { continue; } $rows[] = array( $old_block, $new_block, ); } $this->layoutAvailableRowCount = count($rows); $rows = $this->revealIndexes($rows, true); $rows = $this->sliceRows($rows); return $rows; } public function newOneUpLayout() { $rows = array(); $lists = $this->lists; $idx = 0; while (true) { $found_any = false; $row = array(); foreach ($lists as $list) { $blocks = $list['blocks']; $cell = idx($blocks, $idx); if ($cell !== null) { $found_any = true; } if ($cell) { $rows[] = $cell; } } if (!$found_any) { break; } $idx++; } $this->layoutAvailableRowCount = count($rows); $rows = $this->revealIndexes($rows, false); $rows = $this->sliceRows($rows); return $rows; } private function newDiffSpec(array $blocks) { $map = array(); $list = array(); foreach ($blocks as $block) { $hash = $block->getDifferenceHash(); if (!isset($map[$hash])) { $map[$hash] = array(); } $map[$hash][] = $block; $list[] = $hash; } return array( 'map' => $map, 'list' => implode("\n", $list)."\n", ); } private function sliceRows(array $rows) { $min = $this->rangeMin; $max = $this->rangeMax; if ($min === null && $max === null) { return $rows; } if ($max === null) { return array_slice($rows, $min, null, true); } if ($min === null) { $min = 0; } return array_slice($rows, $min, $max - $min, true); } private function revealIndexes(array $rows, $is_vector) { if ($this->revealedIndexes === null) { return $rows; } foreach ($this->revealedIndexes as $index) { if (!isset($rows[$index])) { continue; } if ($is_vector) { foreach ($rows[$index] as $block) { if ($block !== null) { $block->setIsVisible(true); } } } else { $rows[$index]->setIsVisible(true); } } return $rows; } } diff --git a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php index 3e479fdace..753cdf3921 100644 --- a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php +++ b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php @@ -1,755 +1,764 @@ getName(); if (preg_match('/\\.ipynb\z/i', $name)) { return 2000; } return 500; } protected function canRenderDocumentType(PhabricatorDocumentRef $ref) { return $ref->isProbablyJSON(); } public function canDiffDocuments( PhabricatorDocumentRef $uref = null, PhabricatorDocumentRef $vref = null) { return true; } public function newEngineBlocks( PhabricatorDocumentRef $uref = null, PhabricatorDocumentRef $vref = null) { $blocks = new PhabricatorDocumentEngineBlocks(); try { if ($uref) { $u_blocks = $this->newDiffBlocks($uref); } else { $u_blocks = array(); } if ($vref) { $v_blocks = $this->newDiffBlocks($vref); } else { $v_blocks = array(); } $blocks->addBlockList($uref, $u_blocks); $blocks->addBlockList($vref, $v_blocks); } catch (Exception $ex) { + phlog($ex); $blocks->addMessage($ex->getMessage()); } return $blocks; } public function newBlockDiffViews( PhabricatorDocumentRef $uref, PhabricatorDocumentEngineBlock $ublock, PhabricatorDocumentRef $vref, PhabricatorDocumentEngineBlock $vblock) { $ucell = $ublock->getContent(); $vcell = $vblock->getContent(); $utype = idx($ucell, 'cell_type'); $vtype = idx($vcell, 'cell_type'); if ($utype === $vtype) { switch ($utype) { case 'markdown': $usource = idx($ucell, 'source'); - $usource = implode('', $usource); + if (is_array($usource)) { + $usource = implode('', $usource); + } $vsource = idx($vcell, 'source'); - $vsource = implode('', $vsource); + if (is_array($vsource)) { + $vsource = implode('', $vsource); + } $diff = id(new PhutilProseDifferenceEngine()) ->getDiff($usource, $vsource); $u_content = $this->newProseDiffCell($diff, array('=', '-')); $v_content = $this->newProseDiffCell($diff, array('=', '+')); $u_content = $this->newJupyterCell(null, $u_content, null); $v_content = $this->newJupyterCell(null, $v_content, null); $u_content = $this->newCellContainer($u_content); $v_content = $this->newCellContainer($v_content); return id(new PhabricatorDocumentEngineBlockDiff()) ->setOldContent($u_content) ->addOldClass('old') ->setNewContent($v_content) ->addNewClass('new'); case 'code/line': $usource = idx($ucell, 'raw'); $vsource = idx($vcell, 'raw'); $udisplay = idx($ucell, 'display'); $vdisplay = idx($vcell, 'display'); $ulabel = idx($ucell, 'label'); $vlabel = idx($vcell, 'label'); $intraline_segments = ArcanistDiffUtils::generateIntralineDiff( $usource, $vsource); $u_segments = array(); foreach ($intraline_segments[0] as $u_segment) { $u_segments[] = $u_segment; } $v_segments = array(); foreach ($intraline_segments[1] as $v_segment) { $v_segments[] = $v_segment; } $usource = PhabricatorDifferenceEngine::applyIntralineDiff( $udisplay, $u_segments); $vsource = PhabricatorDifferenceEngine::applyIntralineDiff( $vdisplay, $v_segments); $u_content = $this->newCodeLineCell($ucell, $usource); $v_content = $this->newCodeLineCell($vcell, $vsource); $classes = array( 'jupyter-cell-flush', ); $u_content = $this->newJupyterCell($ulabel, $u_content, $classes); $v_content = $this->newJupyterCell($vlabel, $v_content, $classes); $u_content = $this->newCellContainer($u_content); $v_content = $this->newCellContainer($v_content); return id(new PhabricatorDocumentEngineBlockDiff()) ->setOldContent($u_content) ->addOldClass('old') ->setNewContent($v_content) ->addNewClass('new'); } } return parent::newBlockDiffViews($uref, $ublock, $vref, $vblock); } public function newBlockContentView( PhabricatorDocumentRef $ref, PhabricatorDocumentEngineBlock $block) { $viewer = $this->getViewer(); $cell = $block->getContent(); $cell_content = $this->renderJupyterCell($viewer, $cell); return $this->newCellContainer($cell_content); } private function newCellContainer($cell_content) { $notebook_table = phutil_tag( 'table', array( 'class' => 'jupyter-notebook', ), $cell_content); $container = phutil_tag( 'div', array( 'class' => 'document-engine-jupyter document-engine-diff', ), $notebook_table); return $container; } private function newProseDiffCell(PhutilProseDiff $diff, array $mask) { $mask = array_fuse($mask); $result = array(); foreach ($diff->getParts() as $part) { $type = $part['type']; $text = $part['text']; if (!isset($mask[$type])) { continue; } switch ($type) { case '-': $result[] = phutil_tag( 'span', array( 'class' => 'bright', ), $text); break; case '+': $result[] = phutil_tag( 'span', array( 'class' => 'bright', ), $text); break; case '=': $result[] = $text; break; } } return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-markdown', ), $result), ); } private function newDiffBlocks(PhabricatorDocumentRef $ref) { $viewer = $this->getViewer(); $content = $ref->loadData(); $cells = $this->newCells($content, true); $idx = 1; $blocks = array(); foreach ($cells as $cell) { // When the cell is a source code line, we can hash just the raw // input rather than all the cell metadata. switch (idx($cell, 'cell_type')) { case 'code/line': $hash_input = $cell['raw']; break; case 'markdown': - $hash_input = implode('', $cell['source']); + $hash_input = $cell['source']; + if (is_array($hash_input)) { + $hash_input = implode('', $cell['source']); + } break; default: $hash_input = serialize($cell); break; } $hash = PhabricatorHash::digestWithNamedKey( $hash_input, 'document-engine.content-digest'); $blocks[] = id(new PhabricatorDocumentEngineBlock()) ->setBlockKey($idx) ->setDifferenceHash($hash) ->setContent($cell); $idx++; } return $blocks; } protected function newDocumentContent(PhabricatorDocumentRef $ref) { $viewer = $this->getViewer(); $content = $ref->loadData(); try { $cells = $this->newCells($content, false); } catch (Exception $ex) { return $this->newMessage($ex->getMessage()); } $rows = array(); foreach ($cells as $cell) { $rows[] = $this->renderJupyterCell($viewer, $cell); } $notebook_table = phutil_tag( 'table', array( 'class' => 'jupyter-notebook', ), $rows); $container = phutil_tag( 'div', array( 'class' => 'document-engine-jupyter', ), $notebook_table); return $container; } private function newCells($content, $for_diff) { try { $data = phutil_json_decode($content); } catch (PhutilJSONParserException $ex) { throw new Exception( pht( 'This is not a valid JSON document and can not be rendered as '. 'a Jupyter notebook: %s.', $ex->getMessage())); } if (!is_array($data)) { throw new Exception( pht( 'This document does not encode a valid JSON object and can not '. 'be rendered as a Jupyter notebook.')); } $nbformat = idx($data, 'nbformat'); if (!strlen($nbformat)) { throw new Exception( pht( 'This document is missing an "nbformat" field. Jupyter notebooks '. 'must have this field.')); } if ($nbformat !== 4) { throw new Exception( pht( 'This Jupyter notebook uses an unsupported version of the file '. 'format (found version %s, expected version 4).', $nbformat)); } $cells = idx($data, 'cells'); if (!is_array($cells)) { throw new Exception( pht( 'This Jupyter notebook does not specify a list of "cells".')); } if (!$cells) { throw new Exception( pht( 'This Jupyter notebook does not specify any notebook cells.')); } if (!$for_diff) { return $cells; } // If we're extracting cells to build a diff view, split code cells into // individual lines and individual outputs. We want users to be able to // add inline comments to each line and each output block. $results = array(); foreach ($cells as $cell) { $cell_type = idx($cell, 'cell_type'); - if ($cell_type === 'markdown') { $source = $cell['source']; - $source = implode('', $source); + if (is_array($source)) { + $source = implode('', $source); + } // Attempt to split contiguous blocks of markdown into smaller // pieces. $chunks = preg_split( '/\n\n+/', $source); foreach ($chunks as $chunk) { $result = $cell; $result['source'] = array($chunk); $results[] = $result; } continue; } if ($cell_type !== 'code') { $results[] = $cell; continue; } $label = $this->newCellLabel($cell); $lines = idx($cell, 'source'); if (!is_array($lines)) { $lines = array(); } $content = $this->highlightLines($lines); $count = count($lines); for ($ii = 0; $ii < $count; $ii++) { $is_head = ($ii === 0); $is_last = ($ii === ($count - 1)); if ($is_head) { $line_label = $label; } else { $line_label = null; } $results[] = array( 'cell_type' => 'code/line', 'label' => $line_label, 'raw' => $lines[$ii], 'display' => idx($content, $ii), 'head' => $is_head, 'last' => $is_last, ); } $outputs = array(); $output_list = idx($cell, 'outputs'); if (is_array($output_list)) { foreach ($output_list as $output) { $results[] = array( 'cell_type' => 'code/output', 'output' => $output, ); } } } return $results; } private function renderJupyterCell( PhabricatorUser $viewer, array $cell) { list($label, $content) = $this->renderJupyterCellContent($viewer, $cell); $classes = null; switch (idx($cell, 'cell_type')) { case 'code/line': $classes = 'jupyter-cell-flush'; break; } return $this->newJupyterCell( $label, $content, $classes); } private function newJupyterCell($label, $content, $classes) { $label_cell = phutil_tag( 'td', array( 'class' => 'jupyter-label', ), $label); $content_cell = phutil_tag( 'td', array( 'class' => $classes, ), $content); return phutil_tag( 'tr', array(), array( $label_cell, $content_cell, )); } private function renderJupyterCellContent( PhabricatorUser $viewer, array $cell) { $cell_type = idx($cell, 'cell_type'); switch ($cell_type) { case 'markdown': return $this->newMarkdownCell($cell); case 'code': return $this->newCodeCell($cell); case 'code/line': return $this->newCodeLineCell($cell); case 'code/output': return $this->newCodeOutputCell($cell); } $json_content = id(new PhutilJSON()) ->encodeFormatted($cell); return $this->newRawCell($json_content); } private function newRawCell($content) { return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-raw PhabricatorMonospaced', ), $content), ); } private function newMarkdownCell(array $cell) { $content = idx($cell, 'source'); if (!is_array($content)) { $content = array(); } // TODO: This should ideally highlight as Markdown, but the "md" // highlighter in Pygments is painfully slow and not terribly useful. $content = $this->highlightLines($content, 'txt'); return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-markdown', ), $content), ); } private function newCodeCell(array $cell) { $label = $this->newCellLabel($cell); $content = idx($cell, 'source'); if (!is_array($content)) { $content = array(); } $content = $this->highlightLines($content); $outputs = array(); $output_list = idx($cell, 'outputs'); if (is_array($output_list)) { foreach ($output_list as $output) { $outputs[] = $this->newOutput($output); } } return array( $label, array( phutil_tag( 'div', array( 'class' => 'jupyter-cell-code jupyter-cell-code-block '. 'PhabricatorMonospaced remarkup-code', ), array( $content, )), $outputs, ), ); } private function newCodeLineCell(array $cell, $content = null) { $classes = array(); $classes[] = 'PhabricatorMonospaced'; $classes[] = 'remarkup-code'; $classes[] = 'jupyter-cell-code'; $classes[] = 'jupyter-cell-code-line'; if ($cell['head']) { $classes[] = 'jupyter-cell-code-head'; } if ($cell['last']) { $classes[] = 'jupyter-cell-code-last'; } $classes = implode(' ', $classes); if ($content === null) { $content = $cell['display']; } return array( $cell['label'], array( phutil_tag( 'div', array( 'class' => $classes, ), array( $content, )), ), ); } private function newCodeOutputCell(array $cell) { return array( null, $this->newOutput($cell['output']), ); } private function newOutput(array $output) { if (!is_array($output)) { return pht(''); } $classes = array( 'jupyter-output', 'PhabricatorMonospaced', ); $output_name = idx($output, 'name'); switch ($output_name) { case 'stderr': $classes[] = 'jupyter-output-stderr'; break; } $output_type = idx($output, 'output_type'); switch ($output_type) { case 'execute_result': case 'display_data': $data = idx($output, 'data'); $image_formats = array( 'image/png', 'image/jpeg', 'image/jpg', 'image/gif', ); foreach ($image_formats as $image_format) { if (!isset($data[$image_format])) { continue; } $raw_data = $data[$image_format]; if (!is_array($raw_data)) { $raw_data = array($raw_data); } $raw_data = implode('', $raw_data); $content = phutil_tag( 'img', array( 'src' => 'data:'.$image_format.';base64,'.$raw_data, )); break 2; } if (isset($data['text/html'])) { $content = $data['text/html']; $classes[] = 'jupyter-output-html'; break; } if (isset($data['application/javascript'])) { $content = $data['application/javascript']; $classes[] = 'jupyter-output-html'; break; } if (isset($data['text/plain'])) { $content = $data['text/plain']; break; } break; case 'stream': default: $content = idx($output, 'text'); if (!is_array($content)) { $content = array(); } $content = implode('', $content); break; } return phutil_tag( 'div', array( 'class' => implode(' ', $classes), ), $content); } private function newCellLabel(array $cell) { $execution_count = idx($cell, 'execution_count'); if ($execution_count) { $label = 'In ['.$execution_count.']:'; } else { $label = null; } return $label; } private function highlightLines(array $lines, $force_language = null) { if ($force_language === null) { $head = head($lines); $matches = null; if (preg_match('/^%%(.*)$/', $head, $matches)) { $restore = array_shift($lines); $lang = $matches[1]; } else { $restore = null; $lang = 'py'; } } else { $restore = null; $lang = $force_language; } $content = PhabricatorSyntaxHighlighter::highlightWithLanguage( $lang, implode('', $lines)); $content = phutil_split_lines($content); if ($restore !== null) { $language_tag = phutil_tag( 'span', array( 'class' => 'language-tag', ), $restore); array_unshift($content, $language_tag); } return $content; } public function shouldSuggestEngine(PhabricatorDocumentRef $ref) { return true; } }