diff --git a/src/applications/differential/parser/DifferentialChangesetParser.php b/src/applications/differential/parser/DifferentialChangesetParser.php index 3652d18a9a..ad62b2a1ef 100644 --- a/src/applications/differential/parser/DifferentialChangesetParser.php +++ b/src/applications/differential/parser/DifferentialChangesetParser.php @@ -1,1959 +1,1975 @@ rangeStart = $start; $this->rangeEnd = $end; return $this; } public function setMask(array $mask) { $this->mask = $mask; return $this; } public function renderChangeset() { return $this->render($this->rangeStart, $this->rangeEnd, $this->mask); } public function setShowEditAndReplyLinks($bool) { $this->showEditAndReplyLinks = $bool; return $this; } public function getShowEditAndReplyLinks() { return $this->showEditAndReplyLinks; } public function setViewState(PhabricatorChangesetViewState $view_state) { $this->viewState = $view_state; return $this; } public function getViewState() { return $this->viewState; } public function setRenderer(DifferentialChangesetRenderer $renderer) { $this->renderer = $renderer; return $this; } public function getRenderer() { return $this->renderer; } public function setDisableCache($disable_cache) { $this->disableCache = $disable_cache; return $this; } public function getDisableCache() { return $this->disableCache; } public function setCanMarkDone($can_mark_done) { $this->canMarkDone = $can_mark_done; return $this; } public function getCanMarkDone() { return $this->canMarkDone; } public function setObjectOwnerPHID($phid) { $this->objectOwnerPHID = $phid; return $this; } public function getObjectOwnerPHID() { return $this->objectOwnerPHID; } public function setOffsetMode($offset_mode) { $this->offsetMode = $offset_mode; return $this; } public function getOffsetMode() { return $this->offsetMode; } public function setViewer(PhabricatorUser $viewer) { $this->viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } private function newRenderer() { $viewer = $this->getViewer(); $viewstate = $this->getViewstate(); $renderer_key = $viewstate->getRendererKey(); if ($renderer_key === null) { $is_unified = $viewer->compareUserSetting( PhabricatorUnifiedDiffsSetting::SETTINGKEY, PhabricatorUnifiedDiffsSetting::VALUE_ALWAYS_UNIFIED); if ($is_unified) { $renderer_key = '1up'; } else { $renderer_key = $viewstate->getDefaultDeviceRendererKey(); } } switch ($renderer_key) { case '1up': $renderer = new DifferentialChangesetOneUpRenderer(); break; default: $renderer = new DifferentialChangesetTwoUpRenderer(); break; } return $renderer; } const CACHE_VERSION = 14; const CACHE_MAX_SIZE = 8e6; const ATTR_GENERATED = 'attr:generated'; const ATTR_DELETED = 'attr:deleted'; const ATTR_UNCHANGED = 'attr:unchanged'; const ATTR_MOVEAWAY = 'attr:moveaway'; public function setOldLines(array $lines) { $this->old = $lines; return $this; } public function setNewLines(array $lines) { $this->new = $lines; return $this; } public function setSpecialAttributes(array $attributes) { $this->specialAttributes = $attributes; return $this; } public function setIntraLineDiffs(array $diffs) { $this->intra = $diffs; return $this; } public function setDepthOnlyLines(array $lines) { $this->depthOnlyLines = $lines; return $this; } public function getDepthOnlyLines() { return $this->depthOnlyLines; } public function setVisibleLinesMask(array $mask) { $this->visible = $mask; return $this; } public function setLinesOfContext($lines_of_context) { $this->linesOfContext = $lines_of_context; return $this; } public function getLinesOfContext() { return $this->linesOfContext; } /** * Configure which Changeset comments added to the right side of the visible * diff will be attached to. The ID must be the ID of a real Differential * Changeset. * * The complexity here is that we may show an arbitrary side of an arbitrary * changeset as either the left or right part of a diff. This method allows * the left and right halves of the displayed diff to be correctly mapped to * storage changesets. * * @param id The Differential Changeset ID that comments added to the right * side of the visible diff should be attached to. * @param bool If true, attach new comments to the right side of the storage * changeset. Note that this may be false, if the left side of * some storage changeset is being shown as the right side of * a display diff. * @return this */ public function setRightSideCommentMapping($id, $is_new) { $this->rightSideChangesetID = $id; $this->rightSideAttachesToNewFile = $is_new; return $this; } /** * See setRightSideCommentMapping(), but this sets information for the left * side of the display diff. */ public function setLeftSideCommentMapping($id, $is_new) { $this->leftSideChangesetID = $id; $this->leftSideAttachesToNewFile = $is_new; return $this; } public function setOriginals( DifferentialChangeset $left, DifferentialChangeset $right) { $this->originalLeft = $left; $this->originalRight = $right; return $this; } public function diffOriginals() { $engine = new PhabricatorDifferenceEngine(); $changeset = $engine->generateChangesetFromFileContent( implode('', mpull($this->originalLeft->getHunks(), 'getChanges')), implode('', mpull($this->originalRight->getHunks(), 'getChanges'))); $parser = new DifferentialHunkParser(); return $parser->parseHunksForHighlightMasks( $changeset->getHunks(), $this->originalLeft->getHunks(), $this->originalRight->getHunks()); } /** * Set a key for identifying this changeset in the render cache. If set, the * parser will attempt to use the changeset render cache, which can improve * performance for frequently-viewed changesets. * * By default, there is no render cache key and parsers do not use the cache. * This is appropriate for rarely-viewed changesets. * * @param string Key for identifying this changeset in the render cache. * @return this */ public function setRenderCacheKey($key) { $this->renderCacheKey = $key; return $this; } private function getRenderCacheKey() { return $this->renderCacheKey; } public function setChangeset(DifferentialChangeset $changeset) { $this->changeset = $changeset; $this->setFilename($changeset->getFilename()); return $this; } public function setRenderingReference($ref) { $this->renderingReference = $ref; return $this; } private function getRenderingReference() { return $this->renderingReference; } public function getChangeset() { return $this->changeset; } public function setFilename($filename) { $this->filename = $filename; return $this; } public function setHandles(array $handles) { assert_instances_of($handles, 'PhabricatorObjectHandle'); $this->handles = $handles; return $this; } public function setMarkupEngine(PhabricatorMarkupEngine $engine) { $this->markupEngine = $engine; return $this; } public function setCoverage($coverage) { $this->coverage = $coverage; return $this; } private function getCoverage() { return $this->coverage; } public function parseInlineComment( PhabricatorInlineComment $comment) { // Parse only comments which are actually visible. if ($this->isCommentVisibleOnRenderedDiff($comment)) { $this->comments[] = $comment; } return $this; } private function loadCache() { $render_cache_key = $this->getRenderCacheKey(); if (!$render_cache_key) { return false; } $data = null; $changeset = new DifferentialChangeset(); $conn_r = $changeset->establishConnection('r'); $data = queryfx_one( $conn_r, 'SELECT * FROM %T WHERE cacheIndex = %s', DifferentialChangeset::TABLE_CACHE, PhabricatorHash::digestForIndex($render_cache_key)); if (!$data) { return false; } if ($data['cache'][0] == '{') { // This is likely an old-style JSON cache which we will not be able to // deserialize. return false; } $data = unserialize($data['cache']); if (!is_array($data) || !$data) { return false; } foreach (self::getCacheableProperties() as $cache_key) { if (!array_key_exists($cache_key, $data)) { // If we're missing a cache key, assume we're looking at an old cache // and ignore it. return false; } } if ($data['cacheVersion'] !== self::CACHE_VERSION) { return false; } // Someone displays contents of a partially cached shielded file. if (!isset($data['newRender']) && (!$this->isTopLevel || $this->comments)) { return false; } unset($data['cacheVersion'], $data['cacheHost']); $cache_prop = array_select_keys($data, self::getCacheableProperties()); foreach ($cache_prop as $cache_key => $v) { $this->$cache_key = $v; } return true; } protected static function getCacheableProperties() { return array( 'visible', 'new', 'old', 'intra', 'depthOnlyLines', 'newRender', 'oldRender', 'specialAttributes', 'hunkStartLines', 'cacheVersion', 'cacheHost', 'highlightingDisabled', ); } public function saveCache() { if (PhabricatorEnv::isReadOnly()) { return false; } if ($this->highlightErrors) { return false; } $render_cache_key = $this->getRenderCacheKey(); if (!$render_cache_key) { return false; } $cache = array(); foreach (self::getCacheableProperties() as $cache_key) { switch ($cache_key) { case 'cacheVersion': $cache[$cache_key] = self::CACHE_VERSION; break; case 'cacheHost': $cache[$cache_key] = php_uname('n'); break; default: $cache[$cache_key] = $this->$cache_key; break; } } $cache = serialize($cache); // We don't want to waste too much space by a single changeset. if (strlen($cache) > self::CACHE_MAX_SIZE) { return; } $changeset = new DifferentialChangeset(); $conn_w = $changeset->establishConnection('w'); $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); try { queryfx( $conn_w, 'INSERT INTO %T (cacheIndex, cache, dateCreated) VALUES (%s, %B, %d) ON DUPLICATE KEY UPDATE cache = VALUES(cache)', DifferentialChangeset::TABLE_CACHE, PhabricatorHash::digestForIndex($render_cache_key), $cache, PhabricatorTime::getNow()); } catch (AphrontQueryException $ex) { // Ignore these exceptions. A common cause is that the cache is // larger than 'max_allowed_packet', in which case we're better off // not writing it. // TODO: It would be nice to tailor this more narrowly. } unset($unguarded); } private function markGenerated($new_corpus_block = '') { $generated_guess = (strpos($new_corpus_block, '@'.'generated') !== false); if (!$generated_guess) { $generated_path_regexps = PhabricatorEnv::getEnvConfig( 'differential.generated-paths'); foreach ($generated_path_regexps as $regexp) { if (preg_match($regexp, $this->changeset->getFilename())) { $generated_guess = true; break; } } } $event = new PhabricatorEvent( PhabricatorEventType::TYPE_DIFFERENTIAL_WILLMARKGENERATED, array( 'corpus' => $new_corpus_block, 'is_generated' => $generated_guess, ) ); PhutilEventEngine::dispatchEvent($event); $generated = $event->getValue('is_generated'); $attribute = $this->changeset->isGeneratedChangeset(); if ($attribute) { $generated = true; } $this->specialAttributes[self::ATTR_GENERATED] = $generated; } public function isGenerated() { return idx($this->specialAttributes, self::ATTR_GENERATED, false); } public function isDeleted() { return idx($this->specialAttributes, self::ATTR_DELETED, false); } public function isUnchanged() { return idx($this->specialAttributes, self::ATTR_UNCHANGED, false); } public function isMoveAway() { return idx($this->specialAttributes, self::ATTR_MOVEAWAY, false); } private function applyIntraline(&$render, $intra, $corpus) { foreach ($render as $key => $text) { $result = $text; if (isset($intra[$key])) { $result = PhabricatorDifferenceEngine::applyIntralineDiff( $result, $intra[$key]); } $result = $this->adjustRenderedLineForDisplay($result); $render[$key] = $result; } } private function getHighlightFuture($corpus) { $language = $this->getViewState()->getHighlightLanguage(); if (!$language) { $language = $this->highlightEngine->getLanguageFromFilename( $this->filename); if (($language != 'txt') && (strlen($corpus) > self::HIGHLIGHT_BYTE_LIMIT)) { $this->highlightingDisabled = true; $language = 'txt'; } } return $this->highlightEngine->getHighlightFuture( $language, $corpus); } protected function processHighlightedSource($data, $result) { $result_lines = phutil_split_lines($result); foreach ($data as $key => $info) { if (!$info) { unset($result_lines[$key]); } } return $result_lines; } private function tryCacheStuff() { $changeset = $this->getChangeset(); if (!$changeset->hasSourceTextBody()) { // TODO: This isn't really correct (the change is not "generated"), the // intent is just to not render a text body for Subversion directory // changes, etc. $this->markGenerated(); return; } $viewstate = $this->getViewState(); $skip_cache = false; if ($this->disableCache) { $skip_cache = true; } $character_encoding = $viewstate->getCharacterEncoding(); if ($character_encoding !== null) { $skip_cache = true; } $highlight_language = $viewstate->getHighlightLanguage(); if ($highlight_language !== null) { $skip_cache = true; } if ($skip_cache || !$this->loadCache()) { $this->process(); if (!$skip_cache) { $this->saveCache(); } } } private function process() { $changeset = $this->changeset; $hunk_parser = new DifferentialHunkParser(); $hunk_parser->parseHunksForLineData($changeset->getHunks()); $this->realignDiff($changeset, $hunk_parser); $hunk_parser->reparseHunksForSpecialAttributes(); $unchanged = false; if (!$hunk_parser->getHasAnyChanges()) { $filetype = $this->changeset->getFileType(); if ($filetype == DifferentialChangeType::FILE_TEXT || $filetype == DifferentialChangeType::FILE_SYMLINK) { $unchanged = true; } } $moveaway = false; $changetype = $this->changeset->getChangeType(); if ($changetype == DifferentialChangeType::TYPE_MOVE_AWAY) { $moveaway = true; } $this->setSpecialAttributes(array( self::ATTR_UNCHANGED => $unchanged, self::ATTR_DELETED => $hunk_parser->getIsDeleted(), self::ATTR_MOVEAWAY => $moveaway, )); $lines_context = $this->getLinesOfContext(); $hunk_parser->generateIntraLineDiffs(); $hunk_parser->generateVisibleLinesMask($lines_context); $this->setOldLines($hunk_parser->getOldLines()); $this->setNewLines($hunk_parser->getNewLines()); $this->setIntraLineDiffs($hunk_parser->getIntraLineDiffs()); $this->setDepthOnlyLines($hunk_parser->getDepthOnlyLines()); $this->setVisibleLinesMask($hunk_parser->getVisibleLinesMask()); $this->hunkStartLines = $hunk_parser->getHunkStartLines( $changeset->getHunks()); $new_corpus = $hunk_parser->getNewCorpus(); $new_corpus_block = implode('', $new_corpus); $this->markGenerated($new_corpus_block); if ($this->isTopLevel && !$this->comments && ($this->isGenerated() || $this->isUnchanged() || $this->isDeleted())) { return; } $old_corpus = $hunk_parser->getOldCorpus(); $old_corpus_block = implode('', $old_corpus); $old_future = $this->getHighlightFuture($old_corpus_block); $new_future = $this->getHighlightFuture($new_corpus_block); $futures = array( 'old' => $old_future, 'new' => $new_future, ); $corpus_blocks = array( 'old' => $old_corpus_block, 'new' => $new_corpus_block, ); $this->highlightErrors = false; foreach (new FutureIterator($futures) as $key => $future) { try { try { $highlighted = $future->resolve(); } catch (PhutilSyntaxHighlighterException $ex) { $this->highlightErrors = true; $highlighted = id(new PhutilDefaultSyntaxHighlighter()) ->getHighlightFuture($corpus_blocks[$key]) ->resolve(); } switch ($key) { case 'old': $this->oldRender = $this->processHighlightedSource( $this->old, $highlighted); break; case 'new': $this->newRender = $this->processHighlightedSource( $this->new, $highlighted); break; } } catch (Exception $ex) { phlog($ex); throw $ex; } } $this->applyIntraline( $this->oldRender, ipull($this->intra, 0), $old_corpus); $this->applyIntraline( $this->newRender, ipull($this->intra, 1), $new_corpus); } private function shouldRenderPropertyChangeHeader($changeset) { if (!$this->isTopLevel) { // We render properties only at top level; otherwise we get multiple // copies of them when a user clicks "Show More". return false; } return true; } public function render( $range_start = null, $range_len = null, $mask_force = array()) { $viewer = $this->getViewer(); $renderer = $this->getRenderer(); if (!$renderer) { $renderer = $this->newRenderer(); $this->setRenderer($renderer); } // "Top level" renders are initial requests for the whole file, versus // requests for a specific range generated by clicking "show more". We // generate property changes and "shield" UI elements only for toplevel // requests. $this->isTopLevel = (($range_start === null) && ($range_len === null)); $this->highlightEngine = PhabricatorSyntaxHighlighter::newEngine(); $viewstate = $this->getViewState(); $encoding = null; $character_encoding = $viewstate->getCharacterEncoding(); if ($character_encoding) { // We are forcing this changeset to be interpreted with a specific // character encoding, so force all the hunks into that encoding and // propagate it to the renderer. $encoding = $character_encoding; foreach ($this->changeset->getHunks() as $hunk) { $hunk->forceEncoding($character_encoding); } } else { // We're just using the default, so tell the renderer what that is // (by reading the encoding from the first hunk). foreach ($this->changeset->getHunks() as $hunk) { $encoding = $hunk->getDataEncoding(); break; } } $this->tryCacheStuff(); // If we're rendering in an offset mode, treat the range numbers as line // numbers instead of rendering offsets. $offset_mode = $this->getOffsetMode(); if ($offset_mode) { if ($offset_mode == 'new') { $offset_map = $this->new; } else { $offset_map = $this->old; } // NOTE: Inline comments use zero-based lengths. For example, a comment // that starts and ends on line 123 has length 0. Rendering considers // this range to have length 1. Probably both should agree, but that // ship likely sailed long ago. Tweak things here to get the two systems // to agree. See PHI985, where this affected mail rendering of inline // comments left on the final line of a file. $range_end = $this->getOffset($offset_map, $range_start + $range_len); $range_start = $this->getOffset($offset_map, $range_start); $range_len = ($range_end - $range_start) + 1; } $render_pch = $this->shouldRenderPropertyChangeHeader($this->changeset); $rows = max( count($this->old), count($this->new)); $renderer = $this->getRenderer() ->setUser($this->getViewer()) ->setChangeset($this->changeset) ->setRenderPropertyChangeHeader($render_pch) ->setIsTopLevel($this->isTopLevel) ->setOldRender($this->oldRender) ->setNewRender($this->newRender) ->setHunkStartLines($this->hunkStartLines) ->setOldChangesetID($this->leftSideChangesetID) ->setNewChangesetID($this->rightSideChangesetID) ->setOldAttachesToNewFile($this->leftSideAttachesToNewFile) ->setNewAttachesToNewFile($this->rightSideAttachesToNewFile) ->setCodeCoverage($this->getCoverage()) ->setRenderingReference($this->getRenderingReference()) ->setHandles($this->handles) ->setOldLines($this->old) ->setNewLines($this->new) ->setOriginalCharacterEncoding($encoding) ->setShowEditAndReplyLinks($this->getShowEditAndReplyLinks()) ->setCanMarkDone($this->getCanMarkDone()) ->setObjectOwnerPHID($this->getObjectOwnerPHID()) ->setHighlightingDisabled($this->highlightingDisabled) ->setDepthOnlyLines($this->getDepthOnlyLines()); if ($this->markupEngine) { $renderer->setMarkupEngine($this->markupEngine); } list($engine, $old_ref, $new_ref) = $this->newDocumentEngine(); if ($engine) { $engine_blocks = $engine->newEngineBlocks( $old_ref, $new_ref); } else { $engine_blocks = null; } $has_document_engine = ($engine_blocks !== null); // Remove empty comments that don't have any unsaved draft data. PhabricatorInlineComment::loadAndAttachVersionedDrafts( $viewer, $this->comments); foreach ($this->comments as $key => $comment) { if ($comment->isVoidComment($viewer)) { unset($this->comments[$key]); } } // See T13515. Sometimes, we collapse file content by default: for // example, if the file is marked as containing generated code. // If a file has inline comments, that normally means we never collapse // it. However, if the viewer has already collapsed all of the inlines, // it's fine to collapse the file. $expanded_comments = array(); foreach ($this->comments as $comment) { if ($comment->isHidden()) { continue; } $expanded_comments[] = $comment; } $collapsed_count = (count($this->comments) - count($expanded_comments)); $shield_raw = null; $shield_text = null; $shield_type = null; if ($this->isTopLevel && !$expanded_comments && !$has_document_engine) { if ($this->isGenerated()) { $shield_text = pht( 'This file contains generated code, which does not normally '. 'need to be reviewed.'); } else if ($this->isMoveAway()) { // We put an empty shield on these files. Normally, they do not have // any diff content anyway. However, if they come through `arc`, they // may have content. We don't want to show it (it's not useful) and // we bailed out of fully processing it earlier anyway. // We could show a message like "this file was moved", but we show // that as a change header anyway, so it would be redundant. Instead, // just render an empty shield to skip rendering the diff body. $shield_raw = ''; } else if ($this->isUnchanged()) { $type = 'text'; if (!$rows) { // NOTE: Normally, diffs which don't change files do not include // file content (for example, if you "chmod +x" a file and then // run "git show", the file content is not available). Similarly, // if you move a file from A to B without changing it, diffs normally // do not show the file content. In some cases `arc` is able to // synthetically generate content for these diffs, but for raw diffs // we'll never have it so we need to be prepared to not render a link. $type = 'none'; } $shield_type = $type; $type_add = DifferentialChangeType::TYPE_ADD; if ($this->changeset->getChangeType() == $type_add) { // Although the generic message is sort of accurate in a technical // sense, this more-tailored message is less confusing. $shield_text = pht('This is an empty file.'); } else { $shield_text = pht('The contents of this file were not changed.'); } } else if ($this->isDeleted()) { $shield_text = pht('This file was completely deleted.'); } else if ($this->changeset->getAffectedLineCount() > 2500) { $shield_text = pht( 'This file has a very large number of changes (%s lines).', new PhutilNumber($this->changeset->getAffectedLineCount())); } } $shield = null; if ($shield_raw !== null) { $shield = $shield_raw; } else if ($shield_text !== null) { if ($shield_type === null) { $shield_type = 'default'; } // If we have inlines and the shield would normally show the whole file, // downgrade it to show only text around the inlines. if ($collapsed_count) { if ($shield_type === 'text') { $shield_type = 'default'; } $shield_text = array( $shield_text, ' ', pht( 'This file has %d collapsed inline comment(s).', new PhutilNumber($collapsed_count)), ); } $shield = $renderer->renderShield($shield_text, $shield_type); } if ($shield !== null) { return $renderer->renderChangesetTable($shield); } // This request should render the "undershield" headers if it's a top-level // request which made it this far (indicating the changeset has no shield) // or it's a request with no mask information (indicating it's the request // that removes the rendering shield). Possibly, this second class of // request might need to be made more explicit. $is_undershield = (empty($mask_force) || $this->isTopLevel); $renderer->setIsUndershield($is_undershield); $old_comments = array(); $new_comments = array(); $old_mask = array(); $new_mask = array(); $feedback_mask = array(); $lines_context = $this->getLinesOfContext(); if ($this->comments) { // If there are any comments which appear in sections of the file which // we don't have, we're going to move them backwards to the closest // earlier line. Two cases where this may happen are: // // - Porting ghost comments forward into a file which was mostly // deleted. // - Porting ghost comments forward from a full-context diff to a // partial-context diff. list($old_backmap, $new_backmap) = $this->buildLineBackmaps(); foreach ($this->comments as $comment) { $new_side = $this->isCommentOnRightSideWhenDisplayed($comment); $line = $comment->getLineNumber(); // See T13524. Lint inlines from Harbormaster may not have a line // number. if ($line === null) { $back_line = null; } else if ($new_side) { $back_line = idx($new_backmap, $line); } else { $back_line = idx($old_backmap, $line); } if ($back_line != $line) { // TODO: This should probably be cleaner, but just be simple and // obvious for now. $ghost = $comment->getIsGhost(); if ($ghost) { $moved = pht( 'This comment originally appeared on line %s, but that line '. 'does not exist in this version of the diff. It has been '. 'moved backward to the nearest line.', new PhutilNumber($line)); $ghost['reason'] = $ghost['reason']."\n\n".$moved; $comment->setIsGhost($ghost); } $comment->setLineNumber($back_line); $comment->setLineLength(0); } $start = max($comment->getLineNumber() - $lines_context, 0); $end = $comment->getLineNumber() + $comment->getLineLength() + $lines_context; for ($ii = $start; $ii <= $end; $ii++) { if ($new_side) { $new_mask[$ii] = true; } else { $old_mask[$ii] = true; } } } foreach ($this->old as $ii => $old) { if (isset($old['line']) && isset($old_mask[$old['line']])) { $feedback_mask[$ii] = true; } } foreach ($this->new as $ii => $new) { if (isset($new['line']) && isset($new_mask[$new['line']])) { $feedback_mask[$ii] = true; } } $this->comments = id(new PHUIDiffInlineThreader()) ->reorderAndThreadCommments($this->comments); + $old_max_display = 1; + foreach ($this->old as $old) { + if (isset($old['line'])) { + $old_max_display = $old['line']; + } + } + + $new_max_display = 1; + foreach ($this->new as $new) { + if (isset($new['line'])) { + $new_max_display = $new['line']; + } + } + foreach ($this->comments as $comment) { - $final = $comment->getLineNumber() + - $comment->getLineLength(); - $final = max(1, $final); + $display_line = $comment->getLineNumber() + $comment->getLineLength(); + $display_line = max(1, $display_line); + if ($this->isCommentOnRightSideWhenDisplayed($comment)) { - $new_comments[$final][] = $comment; + $display_line = min($new_max_display, $display_line); + $new_comments[$display_line][] = $comment; } else { - $old_comments[$final][] = $comment; + $display_line = min($old_max_display, $display_line); + $old_comments[$display_line][] = $comment; } } } $renderer ->setOldComments($old_comments) ->setNewComments($new_comments); if ($engine_blocks !== null) { $reference = $this->getRenderingReference(); $parts = explode('/', $reference); if (count($parts) == 2) { list($id, $vs) = $parts; } else { $id = $parts[0]; $vs = 0; } // If we don't have an explicit "vs" changeset, it's the left side of // the "id" changeset. if (!$vs) { $vs = $id; } if ($mask_force) { $engine_blocks->setRevealedIndexes(array_keys($mask_force)); } if ($range_start !== null || $range_len !== null) { $range_min = $range_start; if ($range_len === null) { $range_max = null; } else { $range_max = (int)$range_start + (int)$range_len; } $engine_blocks->setRange($range_min, $range_max); } $renderer ->setDocumentEngine($engine) ->setDocumentEngineBlocks($engine_blocks); return $renderer->renderDocumentEngineBlocks( $engine_blocks, (string)$id, (string)$vs); } // If we've made it here with a type of file we don't know how to render, // bail out with a default empty rendering. Normally, we'd expect a // document engine to catch these changes before we make it this far. switch ($this->changeset->getFileType()) { case DifferentialChangeType::FILE_DIRECTORY: case DifferentialChangeType::FILE_BINARY: case DifferentialChangeType::FILE_IMAGE: $output = $renderer->renderChangesetTable(null); return $output; } if ($this->originalLeft && $this->originalRight) { list($highlight_old, $highlight_new) = $this->diffOriginals(); $highlight_old = array_flip($highlight_old); $highlight_new = array_flip($highlight_new); $renderer ->setHighlightOld($highlight_old) ->setHighlightNew($highlight_new); } $renderer ->setOriginalOld($this->originalLeft) ->setOriginalNew($this->originalRight); if ($range_start === null) { $range_start = 0; } if ($range_len === null) { $range_len = $rows; } $range_len = min($range_len, $rows - $range_start); list($gaps, $mask) = $this->calculateGapsAndMask( $mask_force, $feedback_mask, $range_start, $range_len); $renderer ->setGaps($gaps) ->setMask($mask); $html = $renderer->renderTextChange( $range_start, $range_len, $rows); return $renderer->renderChangesetTable($html); } /** * This function calculates a lot of stuff we need to know to display * the diff: * * Gaps - compute gaps in the visible display diff, where we will render * "Show more context" spacers. If a gap is smaller than the context size, * we just display it. Otherwise, we record it into $gaps and will render a * "show more context" element instead of diff text below. A given $gap * is a tuple of $gap_line_number_start and $gap_length. * * Mask - compute the actual lines that need to be shown (because they * are near changes lines, near inline comments, or the request has * explicitly asked for them, i.e. resulting from the user clicking * "show more"). The $mask returned is a sparsely populated dictionary * of $visible_line_number => true. * * @return array($gaps, $mask) */ private function calculateGapsAndMask( $mask_force, $feedback_mask, $range_start, $range_len) { $lines_context = $this->getLinesOfContext(); $gaps = array(); $gap_start = 0; $in_gap = false; $base_mask = $this->visible + $mask_force + $feedback_mask; $base_mask[$range_start + $range_len] = true; for ($ii = $range_start; $ii <= $range_start + $range_len; $ii++) { if (isset($base_mask[$ii])) { if ($in_gap) { $gap_length = $ii - $gap_start; if ($gap_length <= $lines_context) { for ($jj = $gap_start; $jj <= $gap_start + $gap_length; $jj++) { $base_mask[$jj] = true; } } else { $gaps[] = array($gap_start, $gap_length); } $in_gap = false; } } else { if (!$in_gap) { $gap_start = $ii; $in_gap = true; } } } $gaps = array_reverse($gaps); $mask = $base_mask; return array($gaps, $mask); } /** * Determine if an inline comment will appear on the rendered diff, * taking into consideration which halves of which changesets will actually * be shown. * * @param PhabricatorInlineComment Comment to test for visibility. * @return bool True if the comment is visible on the rendered diff. */ private function isCommentVisibleOnRenderedDiff( PhabricatorInlineComment $comment) { $changeset_id = $comment->getChangesetID(); $is_new = $comment->getIsNewFile(); if ($changeset_id == $this->rightSideChangesetID && $is_new == $this->rightSideAttachesToNewFile) { return true; } if ($changeset_id == $this->leftSideChangesetID && $is_new == $this->leftSideAttachesToNewFile) { return true; } return false; } /** * Determine if a comment will appear on the right side of the display diff. * Note that the comment must appear somewhere on the rendered changeset, as * per isCommentVisibleOnRenderedDiff(). * * @param PhabricatorInlineComment Comment to test for display * location. * @return bool True for right, false for left. */ private function isCommentOnRightSideWhenDisplayed( PhabricatorInlineComment $comment) { if (!$this->isCommentVisibleOnRenderedDiff($comment)) { throw new Exception(pht('Comment is not visible on changeset!')); } $changeset_id = $comment->getChangesetID(); $is_new = $comment->getIsNewFile(); if ($changeset_id == $this->rightSideChangesetID && $is_new == $this->rightSideAttachesToNewFile) { return true; } return false; } /** * Parse the 'range' specification that this class and the client-side JS * emit to indicate that a user clicked "Show more..." on a diff. Generally, * use is something like this: * * $spec = $request->getStr('range'); * $parsed = DifferentialChangesetParser::parseRangeSpecification($spec); * list($start, $end, $mask) = $parsed; * $parser->render($start, $end, $mask); * * @param string Range specification, indicating the range of the diff that * should be rendered. * @return tuple List of suitable for passing to * @{method:render}. */ public static function parseRangeSpecification($spec) { $range_s = null; $range_e = null; $mask = array(); if ($spec) { $match = null; if (preg_match('@^(\d+)-(\d+)(?:/(\d+)-(\d+))?$@', $spec, $match)) { $range_s = (int)$match[1]; $range_e = (int)$match[2]; if (count($match) > 3) { $start = (int)$match[3]; $len = (int)$match[4]; for ($ii = $start; $ii < $start + $len; $ii++) { $mask[$ii] = true; } } } } return array($range_s, $range_e, $mask); } /** * Render "modified coverage" information; test coverage on modified lines. * This synthesizes diff information with unit test information into a useful * indicator of how well tested a change is. */ public function renderModifiedCoverage() { $na = phutil_tag('em', array(), '-'); $coverage = $this->getCoverage(); if (!$coverage) { return $na; } $covered = 0; $not_covered = 0; foreach ($this->new as $k => $new) { if ($new === null) { continue; } if (!$new['line']) { continue; } if (!$new['type']) { continue; } if (empty($coverage[$new['line'] - 1])) { continue; } switch ($coverage[$new['line'] - 1]) { case 'C': $covered++; break; case 'U': $not_covered++; break; } } if (!$covered && !$not_covered) { return $na; } return sprintf('%d%%', 100 * ($covered / ($covered + $not_covered))); } /** * Build maps from lines comments appear on to actual lines. */ private function buildLineBackmaps() { $old_back = array(); $new_back = array(); foreach ($this->old as $ii => $old) { if ($old === null) { continue; } $old_back[$old['line']] = $old['line']; } foreach ($this->new as $ii => $new) { if ($new === null) { continue; } $new_back[$new['line']] = $new['line']; } $max_old_line = 0; $max_new_line = 0; foreach ($this->comments as $comment) { if ($this->isCommentOnRightSideWhenDisplayed($comment)) { $max_new_line = max($max_new_line, $comment->getLineNumber()); } else { $max_old_line = max($max_old_line, $comment->getLineNumber()); } } $cursor = 1; for ($ii = 1; $ii <= $max_old_line; $ii++) { if (empty($old_back[$ii])) { $old_back[$ii] = $cursor; } else { $cursor = $old_back[$ii]; } } $cursor = 1; for ($ii = 1; $ii <= $max_new_line; $ii++) { if (empty($new_back[$ii])) { $new_back[$ii] = $cursor; } else { $cursor = $new_back[$ii]; } } return array($old_back, $new_back); } private function getOffset(array $map, $line) { if (!$map) { return null; } $line = (int)$line; foreach ($map as $key => $spec) { if ($spec && isset($spec['line'])) { if ((int)$spec['line'] >= $line) { return $key; } } } return $key; } private function realignDiff( DifferentialChangeset $changeset, DifferentialHunkParser $hunk_parser) { // Normalizing and realigning the diff depends on rediffing the files, and // we currently need complete representations of both files to do anything // reasonable. If we only have parts of the files, skip realignment. // We have more than one hunk, so we're definitely missing part of the file. $hunks = $changeset->getHunks(); if (count($hunks) !== 1) { return null; } // The first hunk doesn't start at the beginning of the file, so we're // missing some context. $first_hunk = head($hunks); if ($first_hunk->getOldOffset() != 1 || $first_hunk->getNewOffset() != 1) { return null; } $old_file = $changeset->makeOldFile(); $new_file = $changeset->makeNewFile(); if ($old_file === $new_file) { // If the old and new files are exactly identical, the synthetic // diff below will give us nonsense and whitespace modes are // irrelevant anyway. This occurs when you, e.g., copy a file onto // itself in Subversion (see T271). return null; } $engine = id(new PhabricatorDifferenceEngine()) ->setNormalize(true); $normalized_changeset = $engine->generateChangesetFromFileContent( $old_file, $new_file); $type_parser = new DifferentialHunkParser(); $type_parser->parseHunksForLineData($normalized_changeset->getHunks()); $hunk_parser->setNormalized(true); $hunk_parser->setOldLineTypeMap($type_parser->getOldLineTypeMap()); $hunk_parser->setNewLineTypeMap($type_parser->getNewLineTypeMap()); } private function adjustRenderedLineForDisplay($line) { // IMPORTANT: We're using "str_replace()" against raw HTML here, which can // easily become unsafe. The input HTML has already had syntax highlighting // and intraline diff highlighting applied, so it's full of "" tags. static $search; static $replace; if ($search === null) { $rules = $this->newSuspiciousCharacterRules(); $map = array(); foreach ($rules as $key => $spec) { $tag = phutil_tag( 'span', array( 'data-copy-text' => $key, 'class' => $spec['class'], 'title' => $spec['title'], ), $spec['replacement']); $map[$key] = phutil_string_cast($tag); } $search = array_keys($map); $replace = array_values($map); } $is_html = false; if ($line instanceof PhutilSafeHTML) { $is_html = true; $line = hsprintf('%s', $line); } $line = phutil_string_cast($line); // TODO: This should be flexible, eventually. $tab_width = 2; $line = self::replaceTabsWithSpaces($line, $tab_width); $line = str_replace($search, $replace, $line); if ($is_html) { $line = phutil_safe_html($line); } return $line; } private function newSuspiciousCharacterRules() { // The "title" attributes are cached in the database, so they're // intentionally not wrapped in "pht(...)". $rules = array( "\xE2\x80\x8B" => array( 'title' => 'ZWS', 'class' => 'suspicious-character', 'replacement' => '!', ), "\xC2\xA0" => array( 'title' => 'NBSP', 'class' => 'suspicious-character', 'replacement' => '!', ), "\x7F" => array( 'title' => 'DEL (0x7F)', 'class' => 'suspicious-character', 'replacement' => "\xE2\x90\xA1", ), ); // Unicode defines special pictures for the control characters in the // range between "0x00" and "0x1F". $control = array( 'NULL', 'SOH', 'STX', 'ETX', 'EOT', 'ENQ', 'ACK', 'BEL', 'BS', null, // "\t" Tab null, // "\n" New Line 'VT', 'FF', null, // "\r" Carriage Return, 'SO', 'SI', 'DLE', 'DC1', 'DC2', 'DC3', 'DC4', 'NAK', 'SYN', 'ETB', 'CAN', 'EM', 'SUB', 'ESC', 'FS', 'GS', 'RS', 'US', ); foreach ($control as $idx => $label) { if ($label === null) { continue; } $rules[chr($idx)] = array( 'title' => sprintf('%s (0x%02X)', $label, $idx), 'class' => 'suspicious-character', 'replacement' => "\xE2\x90".chr(0x80 + $idx), ); } return $rules; } public static function replaceTabsWithSpaces($line, $tab_width) { static $tags = array(); if (empty($tags[$tab_width])) { for ($ii = 1; $ii <= $tab_width; $ii++) { $tag = phutil_tag( 'span', array( 'data-copy-text' => "\t", ), str_repeat(' ', $ii)); $tag = phutil_string_cast($tag); $tags[$ii] = $tag; } } // Expand all prefix tabs until we encounter any non-tab character. This // is cheap and often immediately produces the correct result with no // further work (and, particularly, no need to handle any unicode cases). $len = strlen($line); $head = 0; for ($head = 0; $head < $len; $head++) { $char = $line[$head]; if ($char !== "\t") { break; } } if ($head) { if (empty($tags[$tab_width * $head])) { $tags[$tab_width * $head] = str_repeat($tags[$tab_width], $head); } $prefix = $tags[$tab_width * $head]; $line = substr($line, $head); } else { $prefix = ''; } // If we have no remaining tabs elsewhere in the string after taking care // of all the prefix tabs, we're done. if (strpos($line, "\t") === false) { return $prefix.$line; } $len = strlen($line); // If the line is particularly long, don't try to do anything special with // it. Use a faster approximation of the correct tabstop expansion instead. // This usually still arrives at the right result. if ($len > 256) { return $prefix.str_replace("\t", $tags[$tab_width], $line); } $in_tag = false; $pos = 0; // See PHI1210. If the line only has single-byte characters, we don't need // to vectorize it and can avoid an expensive UTF8 call. $fast_path = preg_match('/^[\x01-\x7F]*\z/', $line); if ($fast_path) { $replace = array(); for ($ii = 0; $ii < $len; $ii++) { $char = $line[$ii]; if ($char === '>') { $in_tag = false; continue; } if ($in_tag) { continue; } if ($char === '<') { $in_tag = true; continue; } if ($char === "\t") { $count = $tab_width - ($pos % $tab_width); $pos += $count; $replace[$ii] = $tags[$count]; continue; } $pos++; } if ($replace) { // Apply replacements starting at the end of the string so they // don't mess up the offsets for following replacements. $replace = array_reverse($replace, true); foreach ($replace as $replace_pos => $replacement) { $line = substr_replace($line, $replacement, $replace_pos, 1); } } } else { $line = phutil_utf8v_combined($line); foreach ($line as $key => $char) { if ($char === '>') { $in_tag = false; continue; } if ($in_tag) { continue; } if ($char === '<') { $in_tag = true; continue; } if ($char === "\t") { $count = $tab_width - ($pos % $tab_width); $pos += $count; $line[$key] = $tags[$count]; continue; } $pos++; } $line = implode('', $line); } return $prefix.$line; } private function newDocumentEngine() { $changeset = $this->changeset; $viewer = $this->getViewer(); list($old_file, $new_file) = $this->loadFileObjectsForChangeset(); $no_old = !$changeset->hasOldState(); $no_new = !$changeset->hasNewState(); if ($no_old) { $old_ref = null; } else { $old_ref = id(new PhabricatorDocumentRef()) ->setName($changeset->getOldFile()); if ($old_file) { $old_ref->setFile($old_file); } else { $old_data = $this->getRawDocumentEngineData($this->old); $old_ref->setData($old_data); } } if ($no_new) { $new_ref = null; } else { $new_ref = id(new PhabricatorDocumentRef()) ->setName($changeset->getFilename()); if ($new_file) { $new_ref->setFile($new_file); } else { $new_data = $this->getRawDocumentEngineData($this->new); $new_ref->setData($new_data); } } $old_engines = null; if ($old_ref) { $old_engines = PhabricatorDocumentEngine::getEnginesForRef( $viewer, $old_ref); } $new_engines = null; if ($new_ref) { $new_engines = PhabricatorDocumentEngine::getEnginesForRef( $viewer, $new_ref); } if ($new_engines !== null && $old_engines !== null) { $shared_engines = array_intersect_key($new_engines, $old_engines); $default_engine = head_key($new_engines); } else if ($new_engines !== null) { $shared_engines = $new_engines; $default_engine = head_key($shared_engines); } else if ($old_engines !== null) { $shared_engines = $old_engines; $default_engine = head_key($shared_engines); } else { return null; } foreach ($shared_engines as $key => $shared_engine) { if (!$shared_engine->canDiffDocuments($old_ref, $new_ref)) { unset($shared_engines[$key]); } } $this->availableDocumentEngines = $shared_engines; $viewstate = $this->getViewState(); $engine_key = $viewstate->getDocumentEngineKey(); if (strlen($engine_key)) { if (isset($shared_engines[$engine_key])) { $document_engine = $shared_engines[$engine_key]; } else { $document_engine = null; } } else { // If we aren't rendering with a specific engine, only use a default // engine if the best engine for the new file is a shared engine which // can diff files. If we're less picky (for example, by accepting any // shared engine) we can end up with silly behavior (like ".json" files // rendering as Jupyter documents). if (isset($shared_engines[$default_engine])) { $document_engine = $shared_engines[$default_engine]; } else { $document_engine = null; } } if ($document_engine) { return array( $document_engine, $old_ref, $new_ref); } return null; } private function loadFileObjectsForChangeset() { $changeset = $this->changeset; $viewer = $this->getViewer(); $old_phid = $changeset->getOldFileObjectPHID(); $new_phid = $changeset->getNewFileObjectPHID(); $old_file = null; $new_file = null; if ($old_phid || $new_phid) { $file_phids = array(); if ($old_phid) { $file_phids[] = $old_phid; } if ($new_phid) { $file_phids[] = $new_phid; } $files = id(new PhabricatorFileQuery()) ->setViewer($viewer) ->withPHIDs($file_phids) ->execute(); $files = mpull($files, null, 'getPHID'); if ($old_phid) { $old_file = idx($files, $old_phid); if (!$old_file) { throw new Exception( pht( 'Failed to load file data for changeset ("%s").', $old_phid)); } $changeset->attachOldFileObject($old_file); } if ($new_phid) { $new_file = idx($files, $new_phid); if (!$new_file) { throw new Exception( pht( 'Failed to load file data for changeset ("%s").', $new_phid)); } $changeset->attachNewFileObject($new_file); } } return array($old_file, $new_file); } public function newChangesetResponse() { // NOTE: This has to happen first because it has side effects. Yuck. $rendered_changeset = $this->renderChangeset(); $renderer = $this->getRenderer(); $renderer_key = $renderer->getRendererKey(); $viewstate = $this->getViewState(); $undo_templates = $renderer->renderUndoTemplates(); foreach ($undo_templates as $key => $undo_template) { $undo_templates[$key] = hsprintf('%s', $undo_template); } $document_engine = $renderer->getDocumentEngine(); if ($document_engine) { $document_engine_key = $document_engine->getDocumentEngineKey(); } else { $document_engine_key = null; } $available_keys = array(); $engines = $this->availableDocumentEngines; if (!$engines) { $engines = array(); } $available_keys = mpull($engines, 'getDocumentEngineKey'); // TODO: Always include "source" as a usable engine to default to // the buitin rendering. This is kind of a hack and does not actually // use the source engine. The source engine isn't a diff engine, so // selecting it causes us to fall through and render with builtin // behavior. For now, overall behavir is reasonable. $available_keys[] = PhabricatorSourceDocumentEngine::ENGINEKEY; $available_keys = array_fuse($available_keys); $available_keys = array_values($available_keys); $state = array( 'undoTemplates' => $undo_templates, 'rendererKey' => $renderer_key, 'highlight' => $viewstate->getHighlightLanguage(), 'characterEncoding' => $viewstate->getCharacterEncoding(), 'requestDocumentEngineKey' => $viewstate->getDocumentEngineKey(), 'responseDocumentEngineKey' => $document_engine_key, 'availableDocumentEngineKeys' => $available_keys, 'isHidden' => $viewstate->getHidden(), ); return id(new PhabricatorChangesetResponse()) ->setRenderedChangeset($rendered_changeset) ->setChangesetState($state); } private function getRawDocumentEngineData(array $lines) { $text = array(); foreach ($lines as $line) { if ($line === null) { continue; } // If this is a "No newline at end of file." annotation, don't hand it // off to the DocumentEngine. if ($line['type'] === '\\') { continue; } $text[] = $line['text']; } return implode('', $text); } } diff --git a/src/applications/differential/render/DifferentialChangesetOneUpRenderer.php b/src/applications/differential/render/DifferentialChangesetOneUpRenderer.php index 186924e359..bcdbf4dd46 100644 --- a/src/applications/differential/render/DifferentialChangesetOneUpRenderer.php +++ b/src/applications/differential/render/DifferentialChangesetOneUpRenderer.php @@ -1,526 +1,530 @@ simpleMode = $simple_mode; return $this; } public function getSimpleMode() { return $this->simpleMode; } public function isOneUpRenderer() { return true; } protected function getRendererTableClass() { return 'diff-1up'; } public function getRendererKey() { return '1up'; } protected function renderColgroup() { return phutil_tag('colgroup', array(), array( phutil_tag('col', array('class' => 'num')), phutil_tag('col', array('class' => 'num')), phutil_tag('col', array('class' => 'copy')), phutil_tag('col', array('class' => 'unified')), )); } public function renderTextChange( $range_start, $range_len, $rows) { $primitives = $this->buildPrimitives($range_start, $range_len); return $this->renderPrimitives($primitives, $rows); } protected function renderPrimitives(array $primitives, $rows) { list($left_prefix, $right_prefix) = $this->getLineIDPrefixes(); $is_simple = $this->getSimpleMode(); $no_copy = phutil_tag('td', array('class' => 'copy')); $no_coverage = null; $column_width = 4; $aural_minus = javelin_tag( 'span', array( 'aural' => true, 'data-aural' => true, ), '- '); $aural_plus = javelin_tag( 'span', array( 'aural' => true, 'data-aural' => true, ), '+ '); $out = array(); foreach ($primitives as $k => $p) { $type = $p['type']; switch ($type) { case 'old': case 'new': case 'old-file': case 'new-file': $is_old = ($type == 'old' || $type == 'old-file'); $cells = array(); if ($is_old) { if ($p['htype']) { - if (empty($p['oline'])) { + if ($p['htype'] === '\\') { + $class = 'comment'; + } else if (empty($p['oline'])) { $class = 'left old old-full'; } else { $class = 'left old'; } $aural = $aural_minus; } else { $class = 'left'; $aural = null; } if ($type == 'old-file') { $class = "{$class} differential-old-image"; } if ($left_prefix) { $left_id = $left_prefix.$p['line']; } else { $left_id = null; } $line = $p['line']; $cells[] = phutil_tag( 'td', array( 'id' => $left_id, 'class' => $class.' n', 'data-n' => $line, )); $render = $p['render']; if ($aural !== null) { $render = array($aural, $render); } $cells[] = phutil_tag( 'td', array( 'class' => $class.' n', )); $cells[] = $no_copy; $cells[] = phutil_tag('td', array('class' => $class), $render); $cells[] = $no_coverage; } else { if ($p['htype']) { - if (empty($p['oline'])) { + if ($p['htype'] === '\\') { + $class = 'comment'; + } else if (empty($p['oline'])) { $class = 'right new new-full'; } else { $class = 'right new'; } $cells[] = phutil_tag( 'td', array( 'class' => $class.' n', )); $aural = $aural_plus; } else { $class = 'right'; if ($left_prefix) { $left_id = $left_prefix.$p['oline']; } else { $left_id = null; } $oline = $p['oline']; $cells[] = phutil_tag( 'td', array( 'id' => $left_id, 'class' => 'n', 'data-n' => $oline, )); $aural = null; } if ($type == 'new-file') { $class = "{$class} differential-new-image"; } if ($right_prefix) { $right_id = $right_prefix.$p['line']; } else { $right_id = null; } $line = $p['line']; $cells[] = phutil_tag( 'td', array( 'id' => $right_id, 'class' => $class.' n', 'data-n' => $line, )); $render = $p['render']; if ($aural !== null) { $render = array($aural, $render); } $cells[] = $no_copy; $cells[] = phutil_tag( 'td', array( 'class' => $class, 'data-copy-mode' => 'copy-unified', ), $render); $cells[] = $no_coverage; } // In simple mode, only render the text. This is used to render // "Edit Suggestions" in inline comments. if ($is_simple) { $cells = array($cells[3]); } $out[] = phutil_tag('tr', array(), $cells); break; case 'inline': $inline = $this->buildInlineComment( $p['comment'], $p['right']); $out[] = $this->getRowScaffoldForInline($inline); break; case 'no-context': $out[] = phutil_tag( 'tr', array(), phutil_tag( 'td', array( 'class' => 'show-more', 'colspan' => $column_width, ), pht('Context not available.'))); break; case 'context': $top = $p['top']; $len = $p['len']; $links = $this->renderShowContextLinks($top, $len, $rows); $out[] = javelin_tag( 'tr', array( 'sigil' => 'context-target', ), phutil_tag( 'td', array( 'class' => 'show-more', 'colspan' => $column_width, ), $links)); break; default: $out[] = hsprintf('%s', $type); break; } } $result = null; if ($out) { if ($is_simple) { $result = $this->newSimpleTable($out); } else { $result = $this->wrapChangeInTable(phutil_implode_html('', $out)); } } return $result; } public function renderDocumentEngineBlocks( PhabricatorDocumentEngineBlocks $block_list, $old_changeset_key, $new_changeset_key) { $engine = $this->getDocumentEngine(); $layout = $block_list->newTwoUpLayout(); $old_comments = $this->getOldComments(); $new_comments = $this->getNewComments(); $unchanged = array(); foreach ($layout as $key => $row) { list($old, $new) = $row; if (!$old) { continue; } if (!$new) { continue; } if ($old->getDifferenceType() !== null) { continue; } if ($new->getDifferenceType() !== null) { continue; } $unchanged[$key] = true; } $rows = array(); $count = count($layout); for ($ii = 0; $ii < $count;) { $start = $ii; for ($jj = $ii; $jj < $count; $jj++) { list($old, $new) = $layout[$jj]; if (empty($unchanged[$jj])) { break; } $rows[] = array( 'type' => 'unchanged', 'layoutKey' => $jj, ); } $ii = $jj; for ($jj = $ii; $jj < $count; $jj++) { list($old, $new) = $layout[$jj]; if (!empty($unchanged[$jj])) { break; } $rows[] = array( 'type' => 'old', 'layoutKey' => $jj, ); } for ($jj = $ii; $jj < $count; $jj++) { list($old, $new) = $layout[$jj]; if (!empty($unchanged[$jj])) { break; } $rows[] = array( 'type' => 'new', 'layoutKey' => $jj, ); } $ii = $jj; // We always expect to consume at least one row when iterating through // the loop and make progress. If we don't, bail out to avoid spinning // to death. if ($ii === $start) { throw new Exception( pht( 'Failed to make progress during 1up diff layout.')); } } $old_ref = null; $new_ref = null; $refs = $block_list->getDocumentRefs(); if ($refs) { list($old_ref, $new_ref) = $refs; } $view = array(); foreach ($rows as $row) { $row_type = $row['type']; $layout_key = $row['layoutKey']; $row_layout = $layout[$layout_key]; list($old, $new) = $row_layout; if ($old) { $old_key = $old->getBlockKey(); } else { $old_key = null; } if ($new) { $new_key = $new->getBlockKey(); } else { $new_key = null; } $cells = array(); $cell_classes = array(); if ($row_type === 'unchanged') { $cell_content = $engine->newBlockContentView( $old_ref, $old); } else if ($old && $new) { $block_diff = $engine->newBlockDiffViews( $old_ref, $old, $new_ref, $new); // TODO: We're currently double-rendering this: once when building // the old row, and once when building the new one. In both cases, // we throw away the other half of the output. We could cache this // to improve performance. if ($row_type === 'old') { $cell_content = $block_diff->getOldContent(); $cell_classes = $block_diff->getOldClasses(); } else { $cell_content = $block_diff->getNewContent(); $cell_classes = $block_diff->getNewClasses(); } } else if ($row_type === 'old') { if (!$old_ref || !$old) { continue; } $cell_content = $engine->newBlockContentView( $old_ref, $old); $cell_classes[] = 'old'; $cell_classes[] = 'old-full'; $new_key = null; } else if ($row_type === 'new') { if (!$new_ref || !$new) { continue; } $cell_content = $engine->newBlockContentView( $new_ref, $new); $cell_classes[] = 'new'; $cell_classes[] = 'new-full'; $old_key = null; } if ($old_key === null) { $old_id = null; } else { $old_id = "C{$old_changeset_key}OL{$old_key}"; } if ($new_key === null) { $new_id = null; } else { $new_id = "C{$new_changeset_key}NL{$new_key}"; } $cells[] = phutil_tag( 'td', array( 'id' => $old_id, 'data-n' => $old_key, 'class' => 'n', )); $cells[] = phutil_tag( 'td', array( 'id' => $new_id, 'data-n' => $new_key, 'class' => 'n', )); $cells[] = phutil_tag( 'td', array( 'class' => 'copy', )); $cell_classes[] = 'diff-flush'; $cell_classes = implode(' ', $cell_classes); $cells[] = phutil_tag( 'td', array( 'class' => $cell_classes, 'data-copy-mode' => 'copy-unified', ), $cell_content); $view[] = phutil_tag( 'tr', array(), $cells); if ($old_key !== null) { $old_inlines = idx($old_comments, $old_key, array()); foreach ($old_inlines as $inline) { $inline = $this->buildInlineComment( $inline, $on_right = false); $view[] = $this->getRowScaffoldForInline($inline); } } if ($new_key !== null) { $new_inlines = idx($new_comments, $new_key, array()); foreach ($new_inlines as $inline) { $inline = $this->buildInlineComment( $inline, $on_right = true); $view[] = $this->getRowScaffoldForInline($inline); } } } $output = $this->wrapChangeInTable($view); return $this->renderChangesetTable($output); } public function getRowScaffoldForInline(PHUIDiffInlineCommentView $view) { return id(new PHUIDiffOneUpInlineCommentRowScaffold()) ->addInlineView($view); } private function newSimpleTable($content) { return phutil_tag( 'table', array( 'class' => 'diff-1up-simple-table', ), $content); } } diff --git a/src/applications/differential/render/DifferentialChangesetRenderer.php b/src/applications/differential/render/DifferentialChangesetRenderer.php index d493c0e886..dcae3b979b 100644 --- a/src/applications/differential/render/DifferentialChangesetRenderer.php +++ b/src/applications/differential/render/DifferentialChangesetRenderer.php @@ -1,767 +1,771 @@ showEditAndReplyLinks = $bool; return $this; } public function getShowEditAndReplyLinks() { return $this->showEditAndReplyLinks; } public function setHighlightingDisabled($highlighting_disabled) { $this->highlightingDisabled = $highlighting_disabled; return $this; } public function getHighlightingDisabled() { return $this->highlightingDisabled; } public function setOriginalCharacterEncoding($original_character_encoding) { $this->originalCharacterEncoding = $original_character_encoding; return $this; } public function getOriginalCharacterEncoding() { return $this->originalCharacterEncoding; } public function setIsUndershield($is_undershield) { $this->isUndershield = $is_undershield; return $this; } public function getIsUndershield() { return $this->isUndershield; } public function setMask($mask) { $this->mask = $mask; return $this; } protected function getMask() { return $this->mask; } public function setGaps($gaps) { $this->gaps = $gaps; return $this; } protected function getGaps() { return $this->gaps; } public function setDepthOnlyLines(array $lines) { $this->depthOnlyLines = $lines; return $this; } public function getDepthOnlyLines() { return $this->depthOnlyLines; } public function attachOldFile(PhabricatorFile $old = null) { $this->oldFile = $old; return $this; } public function getOldFile() { if ($this->oldFile === false) { throw new PhabricatorDataNotAttachedException($this); } return $this->oldFile; } public function hasOldFile() { return (bool)$this->oldFile; } public function attachNewFile(PhabricatorFile $new = null) { $this->newFile = $new; return $this; } public function getNewFile() { if ($this->newFile === false) { throw new PhabricatorDataNotAttachedException($this); } return $this->newFile; } public function hasNewFile() { return (bool)$this->newFile; } public function setOriginalNew($original_new) { $this->originalNew = $original_new; return $this; } protected function getOriginalNew() { return $this->originalNew; } public function setOriginalOld($original_old) { $this->originalOld = $original_old; return $this; } protected function getOriginalOld() { return $this->originalOld; } public function setNewRender($new_render) { $this->newRender = $new_render; return $this; } protected function getNewRender() { return $this->newRender; } public function setOldRender($old_render) { $this->oldRender = $old_render; return $this; } protected function getOldRender() { return $this->oldRender; } public function setMarkupEngine(PhabricatorMarkupEngine $markup_engine) { $this->markupEngine = $markup_engine; return $this; } public function getMarkupEngine() { return $this->markupEngine; } public function setHandles(array $handles) { assert_instances_of($handles, 'PhabricatorObjectHandle'); $this->handles = $handles; return $this; } protected function getHandles() { return $this->handles; } public function setCodeCoverage($code_coverage) { $this->codeCoverage = $code_coverage; return $this; } protected function getCodeCoverage() { return $this->codeCoverage; } public function setHighlightNew($highlight_new) { $this->highlightNew = $highlight_new; return $this; } protected function getHighlightNew() { return $this->highlightNew; } public function setHighlightOld($highlight_old) { $this->highlightOld = $highlight_old; return $this; } protected function getHighlightOld() { return $this->highlightOld; } public function setNewAttachesToNewFile($attaches) { $this->newAttachesToNewFile = $attaches; return $this; } protected function getNewAttachesToNewFile() { return $this->newAttachesToNewFile; } public function setOldAttachesToNewFile($attaches) { $this->oldAttachesToNewFile = $attaches; return $this; } protected function getOldAttachesToNewFile() { return $this->oldAttachesToNewFile; } public function setNewChangesetID($new_changeset_id) { $this->newChangesetID = $new_changeset_id; return $this; } protected function getNewChangesetID() { return $this->newChangesetID; } public function setOldChangesetID($old_changeset_id) { $this->oldChangesetID = $old_changeset_id; return $this; } protected function getOldChangesetID() { return $this->oldChangesetID; } public function setDocumentEngine(PhabricatorDocumentEngine $engine) { $this->documentEngine = $engine; return $this; } public function getDocumentEngine() { return $this->documentEngine; } public function setDocumentEngineBlocks( PhabricatorDocumentEngineBlocks $blocks) { $this->documentEngineBlocks = $blocks; return $this; } public function getDocumentEngineBlocks() { return $this->documentEngineBlocks; } public function setNewComments(array $new_comments) { foreach ($new_comments as $line_number => $comments) { assert_instances_of($comments, 'PhabricatorInlineComment'); } $this->newComments = $new_comments; return $this; } protected function getNewComments() { return $this->newComments; } public function setOldComments(array $old_comments) { foreach ($old_comments as $line_number => $comments) { assert_instances_of($comments, 'PhabricatorInlineComment'); } $this->oldComments = $old_comments; return $this; } protected function getOldComments() { return $this->oldComments; } public function setNewLines(array $new_lines) { $this->newLines = $new_lines; return $this; } protected function getNewLines() { return $this->newLines; } public function setOldLines(array $old_lines) { $this->oldLines = $old_lines; return $this; } protected function getOldLines() { return $this->oldLines; } public function setHunkStartLines(array $hunk_start_lines) { $this->hunkStartLines = $hunk_start_lines; return $this; } protected function getHunkStartLines() { return $this->hunkStartLines; } public function setUser(PhabricatorUser $user) { $this->user = $user; return $this; } protected function getUser() { return $this->user; } public function setChangeset(DifferentialChangeset $changeset) { $this->changeset = $changeset; return $this; } protected function getChangeset() { return $this->changeset; } public function setRenderingReference($rendering_reference) { $this->renderingReference = $rendering_reference; return $this; } protected function getRenderingReference() { return $this->renderingReference; } public function setRenderPropertyChangeHeader($should_render) { $this->renderPropertyChangeHeader = $should_render; return $this; } private function shouldRenderPropertyChangeHeader() { return $this->renderPropertyChangeHeader; } public function setIsTopLevel($is) { $this->isTopLevel = $is; return $this; } private function getIsTopLevel() { return $this->isTopLevel; } public function setCanMarkDone($can_mark_done) { $this->canMarkDone = $can_mark_done; return $this; } public function getCanMarkDone() { return $this->canMarkDone; } public function setObjectOwnerPHID($phid) { $this->objectOwnerPHID = $phid; return $this; } public function getObjectOwnerPHID() { return $this->objectOwnerPHID; } final public function renderChangesetTable($content) { $props = null; if ($this->shouldRenderPropertyChangeHeader()) { $props = $this->renderPropertyChangeHeader(); } $notice = null; if ($this->getIsTopLevel()) { $force = (!$content && !$props); // If we have DocumentEngine messages about the blocks, assume they // explain why there's no content. $blocks = $this->getDocumentEngineBlocks(); if ($blocks) { if ($blocks->getMessages()) { $force = false; } } $notice = $this->renderChangeTypeHeader($force); } $undershield = null; if ($this->getIsUndershield()) { $undershield = $this->renderUndershieldHeader(); } $result = array( $notice, $props, $undershield, $content, ); return hsprintf('%s', $result); } abstract public function isOneUpRenderer(); abstract public function renderTextChange( $range_start, $range_len, $rows); public function renderDocumentEngineBlocks( PhabricatorDocumentEngineBlocks $blocks, $old_changeset_key, $new_changeset_key) { return null; } abstract protected function renderChangeTypeHeader($force); abstract protected function renderUndershieldHeader(); protected function didRenderChangesetTableContents($contents) { return $contents; } /** * Render a "shield" over the diff, with a message like "This file is * generated and does not need to be reviewed." or "This file was completely * deleted." This UI element hides unimportant text so the reviewer doesn't * need to scroll past it. * * The shield includes a link to view the underlying content. This link * may force certain rendering modes when the link is clicked: * * - `"default"`: Render the diff normally, as though it was not * shielded. This is the default and appropriate if the underlying * diff is a normal change, but was hidden for reasons of not being * important (e.g., generated code). * - `"text"`: Force the text to be shown. This is probably only relevant * when a file is not changed. * - `"none"`: Don't show the link (e.g., text not available). * * @param string Message explaining why the diff is hidden. * @param string|null Force mode, see above. * @return string Shield markup. */ abstract public function renderShield($message, $force = 'default'); abstract protected function renderPropertyChangeHeader(); protected function buildPrimitives($range_start, $range_len) { $primitives = array(); $hunk_starts = $this->getHunkStartLines(); $mask = $this->getMask(); $gaps = $this->getGaps(); $old = $this->getOldLines(); $new = $this->getNewLines(); $old_render = $this->getOldRender(); $new_render = $this->getNewRender(); $old_comments = $this->getOldComments(); $new_comments = $this->getNewComments(); $size = count($old); for ($ii = $range_start; $ii < $range_start + $range_len; $ii++) { if (empty($mask[$ii])) { list($top, $len) = array_pop($gaps); $primitives[] = array( 'type' => 'context', 'top' => $top, 'len' => $len, ); $ii += ($len - 1); continue; } $ospec = array( 'type' => 'old', 'htype' => null, 'cursor' => $ii, 'line' => null, 'oline' => null, 'render' => null, ); $nspec = array( 'type' => 'new', 'htype' => null, 'cursor' => $ii, 'line' => null, 'oline' => null, 'render' => null, 'copy' => null, 'coverage' => null, ); if (isset($old[$ii])) { $ospec['line'] = (int)$old[$ii]['line']; $nspec['oline'] = (int)$old[$ii]['line']; $ospec['htype'] = $old[$ii]['type']; if (isset($old_render[$ii])) { $ospec['render'] = $old_render[$ii]; + } else if ($ospec['htype'] === '\\') { + $ospec['render'] = $old[$ii]['text']; } } if (isset($new[$ii])) { $nspec['line'] = (int)$new[$ii]['line']; $ospec['oline'] = (int)$new[$ii]['line']; $nspec['htype'] = $new[$ii]['type']; if (isset($new_render[$ii])) { $nspec['render'] = $new_render[$ii]; + } else if ($nspec['htype'] === '\\') { + $nspec['render'] = $new[$ii]['text']; } } if (isset($hunk_starts[$ospec['line']])) { $primitives[] = array( 'type' => 'no-context', ); } $primitives[] = $ospec; $primitives[] = $nspec; if ($ospec['line'] !== null && isset($old_comments[$ospec['line']])) { foreach ($old_comments[$ospec['line']] as $comment) { $primitives[] = array( 'type' => 'inline', 'comment' => $comment, 'right' => false, ); } } if ($nspec['line'] !== null && isset($new_comments[$nspec['line']])) { foreach ($new_comments[$nspec['line']] as $comment) { $primitives[] = array( 'type' => 'inline', 'comment' => $comment, 'right' => true, ); } } if ($hunk_starts && ($ii == $size - 1)) { $primitives[] = array( 'type' => 'no-context', ); } } if ($this->isOneUpRenderer()) { $primitives = $this->processPrimitivesForOneUp($primitives); } return $primitives; } private function processPrimitivesForOneUp(array $primitives) { // Primitives come out of buildPrimitives() in two-up format, because it // is the most general, flexible format. To put them into one-up format, // we need to filter and reorder them. In particular: // // - We discard unchanged lines in the old file; in one-up format, we // render them only once. // - We group contiguous blocks of old-modified and new-modified lines, so // they render in "block of old, block of new" order instead of // alternating old and new lines. $out = array(); $old_buf = array(); $new_buf = array(); foreach ($primitives as $primitive) { $type = $primitive['type']; if ($type == 'old') { if (!$primitive['htype']) { // This is a line which appears in both the old file and the new // file, or the spacer corresponding to a line added in the new file. // Ignore it when rendering a one-up diff. continue; } $old_buf[] = $primitive; } else if ($type == 'new') { if ($primitive['line'] === null) { // This is an empty spacer corresponding to a line removed from the // old file. Ignore it when rendering a one-up diff. continue; } if (!$primitive['htype']) { // If this line is the same in both versions of the file, put it in // the old line buffer. This makes sure inlines on old, unchanged // lines end up in the right place. // First, we need to flush the line buffers if they're not empty. if ($old_buf) { $out[] = $old_buf; $old_buf = array(); } if ($new_buf) { $out[] = $new_buf; $new_buf = array(); } $old_buf[] = $primitive; } else { $new_buf[] = $primitive; } } else if ($type == 'context' || $type == 'no-context') { $out[] = $old_buf; $out[] = $new_buf; $old_buf = array(); $new_buf = array(); $out[] = array($primitive); } else if ($type == 'inline') { // If this inline is on the left side, put it after the old lines. if (!$primitive['right']) { $out[] = $old_buf; $out[] = array($primitive); $old_buf = array(); } else { $out[] = $old_buf; $out[] = $new_buf; $out[] = array($primitive); $old_buf = array(); $new_buf = array(); } } else { throw new Exception(pht("Unknown primitive type '%s'!", $primitive)); } } $out[] = $old_buf; $out[] = $new_buf; $out = array_mergev($out); return $out; } protected function getChangesetProperties($changeset) { $old = $changeset->getOldProperties(); $new = $changeset->getNewProperties(); // If a property has been changed, but is not present on one side of the // change and has an uninteresting default value on the other, remove it. // This most commonly happens when a change adds or removes a file: the // side of the change with the file has a "100644" filemode in Git. $defaults = array( 'unix:filemode' => '100644', ); foreach ($defaults as $default_key => $default_value) { $old_value = idx($old, $default_key, $default_value); $new_value = idx($new, $default_key, $default_value); $old_default = ($old_value === $default_value); $new_default = ($new_value === $default_value); if ($old_default && $new_default) { unset($old[$default_key]); unset($new[$default_key]); } } $metadata = $changeset->getMetadata(); if ($this->hasOldFile()) { $file = $this->getOldFile(); if ($file->getImageWidth()) { $dimensions = $file->getImageWidth().'x'.$file->getImageHeight(); $old['file:dimensions'] = $dimensions; } $old['file:mimetype'] = $file->getMimeType(); $old['file:size'] = phutil_format_bytes($file->getByteSize()); } else { $old['file:mimetype'] = idx($metadata, 'old:file:mime-type'); $size = idx($metadata, 'old:file:size'); if ($size !== null) { $old['file:size'] = phutil_format_bytes($size); } } if ($this->hasNewFile()) { $file = $this->getNewFile(); if ($file->getImageWidth()) { $dimensions = $file->getImageWidth().'x'.$file->getImageHeight(); $new['file:dimensions'] = $dimensions; } $new['file:mimetype'] = $file->getMimeType(); $new['file:size'] = phutil_format_bytes($file->getByteSize()); } else { $new['file:mimetype'] = idx($metadata, 'new:file:mime-type'); $size = idx($metadata, 'new:file:size'); if ($size !== null) { $new['file:size'] = phutil_format_bytes($size); } } return array($old, $new); } public function renderUndoTemplates() { $views = array( 'l' => id(new PHUIDiffInlineCommentUndoView())->setIsOnRight(false), 'r' => id(new PHUIDiffInlineCommentUndoView())->setIsOnRight(true), ); foreach ($views as $key => $view) { $scaffold = $this->getRowScaffoldForInline($view); $views[$key] = id(new PHUIDiffInlineCommentTableScaffold()) ->addRowScaffold($scaffold); } return $views; } final protected function getScopeEngine() { if ($this->scopeEngine === false) { $hunk_starts = $this->getHunkStartLines(); // If this change is missing context, don't try to identify scopes, since // we won't really be able to get anywhere. $has_multiple_hunks = (count($hunk_starts) > 1); $has_offset_hunks = false; if ($hunk_starts) { $has_offset_hunks = (head_key($hunk_starts) != 1); } $missing_context = ($has_multiple_hunks || $has_offset_hunks); if ($missing_context) { $scope_engine = null; } else { $line_map = $this->getNewLineTextMap(); $scope_engine = id(new PhabricatorDiffScopeEngine()) ->setLineTextMap($line_map); } $this->scopeEngine = $scope_engine; } return $this->scopeEngine; } private function getNewLineTextMap() { $new = $this->getNewLines(); $text_map = array(); foreach ($new as $new_line) { if (!isset($new_line['line'])) { continue; } $text_map[$new_line['line']] = $new_line['text']; } return $text_map; } } diff --git a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php index 753cdf3921..20474b047b 100644 --- a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php +++ b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php @@ -1,764 +1,748 @@ getName(); if (preg_match('/\\.ipynb\z/i', $name)) { return 2000; } return 500; } protected function canRenderDocumentType(PhabricatorDocumentRef $ref) { return $ref->isProbablyJSON(); } public function canDiffDocuments( PhabricatorDocumentRef $uref = null, PhabricatorDocumentRef $vref = null) { return true; } public function newEngineBlocks( PhabricatorDocumentRef $uref = null, PhabricatorDocumentRef $vref = null) { $blocks = new PhabricatorDocumentEngineBlocks(); try { if ($uref) { $u_blocks = $this->newDiffBlocks($uref); } else { $u_blocks = array(); } if ($vref) { $v_blocks = $this->newDiffBlocks($vref); } else { $v_blocks = array(); } $blocks->addBlockList($uref, $u_blocks); $blocks->addBlockList($vref, $v_blocks); } catch (Exception $ex) { phlog($ex); $blocks->addMessage($ex->getMessage()); } return $blocks; } public function newBlockDiffViews( PhabricatorDocumentRef $uref, PhabricatorDocumentEngineBlock $ublock, PhabricatorDocumentRef $vref, PhabricatorDocumentEngineBlock $vblock) { $ucell = $ublock->getContent(); $vcell = $vblock->getContent(); $utype = idx($ucell, 'cell_type'); $vtype = idx($vcell, 'cell_type'); if ($utype === $vtype) { switch ($utype) { case 'markdown': - $usource = idx($ucell, 'source'); - if (is_array($usource)) { - $usource = implode('', $usource); - } - - $vsource = idx($vcell, 'source'); - if (is_array($vsource)) { - $vsource = implode('', $vsource); - } + $usource = $this->readString($ucell, 'source'); + $vsource = $this->readString($vcell, 'source'); $diff = id(new PhutilProseDifferenceEngine()) ->getDiff($usource, $vsource); $u_content = $this->newProseDiffCell($diff, array('=', '-')); $v_content = $this->newProseDiffCell($diff, array('=', '+')); $u_content = $this->newJupyterCell(null, $u_content, null); $v_content = $this->newJupyterCell(null, $v_content, null); $u_content = $this->newCellContainer($u_content); $v_content = $this->newCellContainer($v_content); return id(new PhabricatorDocumentEngineBlockDiff()) ->setOldContent($u_content) ->addOldClass('old') ->setNewContent($v_content) ->addNewClass('new'); case 'code/line': $usource = idx($ucell, 'raw'); $vsource = idx($vcell, 'raw'); $udisplay = idx($ucell, 'display'); $vdisplay = idx($vcell, 'display'); - $ulabel = idx($ucell, 'label'); - $vlabel = idx($vcell, 'label'); $intraline_segments = ArcanistDiffUtils::generateIntralineDiff( $usource, $vsource); $u_segments = array(); foreach ($intraline_segments[0] as $u_segment) { $u_segments[] = $u_segment; } $v_segments = array(); foreach ($intraline_segments[1] as $v_segment) { $v_segments[] = $v_segment; } $usource = PhabricatorDifferenceEngine::applyIntralineDiff( $udisplay, $u_segments); $vsource = PhabricatorDifferenceEngine::applyIntralineDiff( $vdisplay, $v_segments); - $u_content = $this->newCodeLineCell($ucell, $usource); - $v_content = $this->newCodeLineCell($vcell, $vsource); + list($u_label, $u_content) = $this->newCodeLineCell($ucell, $usource); + list($v_label, $v_content) = $this->newCodeLineCell($vcell, $vsource); $classes = array( 'jupyter-cell-flush', ); - $u_content = $this->newJupyterCell($ulabel, $u_content, $classes); - $v_content = $this->newJupyterCell($vlabel, $v_content, $classes); + $u_content = $this->newJupyterCell($u_label, $u_content, $classes); + $v_content = $this->newJupyterCell($v_label, $v_content, $classes); $u_content = $this->newCellContainer($u_content); $v_content = $this->newCellContainer($v_content); return id(new PhabricatorDocumentEngineBlockDiff()) ->setOldContent($u_content) ->addOldClass('old') ->setNewContent($v_content) ->addNewClass('new'); } } return parent::newBlockDiffViews($uref, $ublock, $vref, $vblock); } public function newBlockContentView( PhabricatorDocumentRef $ref, PhabricatorDocumentEngineBlock $block) { $viewer = $this->getViewer(); $cell = $block->getContent(); $cell_content = $this->renderJupyterCell($viewer, $cell); return $this->newCellContainer($cell_content); } private function newCellContainer($cell_content) { $notebook_table = phutil_tag( 'table', array( 'class' => 'jupyter-notebook', ), $cell_content); $container = phutil_tag( 'div', array( 'class' => 'document-engine-jupyter document-engine-diff', ), $notebook_table); return $container; } private function newProseDiffCell(PhutilProseDiff $diff, array $mask) { $mask = array_fuse($mask); $result = array(); foreach ($diff->getParts() as $part) { $type = $part['type']; $text = $part['text']; if (!isset($mask[$type])) { continue; } switch ($type) { case '-': $result[] = phutil_tag( 'span', array( 'class' => 'bright', ), $text); break; case '+': $result[] = phutil_tag( 'span', array( 'class' => 'bright', ), $text); break; case '=': $result[] = $text; break; } } return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-markdown', ), $result), ); } private function newDiffBlocks(PhabricatorDocumentRef $ref) { $viewer = $this->getViewer(); $content = $ref->loadData(); $cells = $this->newCells($content, true); $idx = 1; $blocks = array(); foreach ($cells as $cell) { // When the cell is a source code line, we can hash just the raw // input rather than all the cell metadata. switch (idx($cell, 'cell_type')) { case 'code/line': $hash_input = $cell['raw']; break; case 'markdown': - $hash_input = $cell['source']; - if (is_array($hash_input)) { - $hash_input = implode('', $cell['source']); - } + $hash_input = $this->readString($cell, 'source'); break; default: $hash_input = serialize($cell); break; } $hash = PhabricatorHash::digestWithNamedKey( $hash_input, 'document-engine.content-digest'); $blocks[] = id(new PhabricatorDocumentEngineBlock()) ->setBlockKey($idx) ->setDifferenceHash($hash) ->setContent($cell); $idx++; } return $blocks; } protected function newDocumentContent(PhabricatorDocumentRef $ref) { $viewer = $this->getViewer(); $content = $ref->loadData(); try { $cells = $this->newCells($content, false); } catch (Exception $ex) { return $this->newMessage($ex->getMessage()); } $rows = array(); foreach ($cells as $cell) { $rows[] = $this->renderJupyterCell($viewer, $cell); } $notebook_table = phutil_tag( 'table', array( 'class' => 'jupyter-notebook', ), $rows); $container = phutil_tag( 'div', array( 'class' => 'document-engine-jupyter', ), $notebook_table); return $container; } private function newCells($content, $for_diff) { try { $data = phutil_json_decode($content); } catch (PhutilJSONParserException $ex) { throw new Exception( pht( 'This is not a valid JSON document and can not be rendered as '. 'a Jupyter notebook: %s.', $ex->getMessage())); } if (!is_array($data)) { throw new Exception( pht( 'This document does not encode a valid JSON object and can not '. 'be rendered as a Jupyter notebook.')); } - $nbformat = idx($data, 'nbformat'); if (!strlen($nbformat)) { throw new Exception( pht( 'This document is missing an "nbformat" field. Jupyter notebooks '. 'must have this field.')); } if ($nbformat !== 4) { throw new Exception( pht( 'This Jupyter notebook uses an unsupported version of the file '. 'format (found version %s, expected version 4).', $nbformat)); } $cells = idx($data, 'cells'); if (!is_array($cells)) { throw new Exception( pht( 'This Jupyter notebook does not specify a list of "cells".')); } if (!$cells) { throw new Exception( pht( 'This Jupyter notebook does not specify any notebook cells.')); } if (!$for_diff) { return $cells; } // If we're extracting cells to build a diff view, split code cells into // individual lines and individual outputs. We want users to be able to // add inline comments to each line and each output block. $results = array(); foreach ($cells as $cell) { $cell_type = idx($cell, 'cell_type'); if ($cell_type === 'markdown') { - $source = $cell['source']; - if (is_array($source)) { - $source = implode('', $source); - } + $source = $this->readString($cell, 'source'); // Attempt to split contiguous blocks of markdown into smaller // pieces. $chunks = preg_split( '/\n\n+/', $source); foreach ($chunks as $chunk) { $result = $cell; $result['source'] = array($chunk); $results[] = $result; } continue; } if ($cell_type !== 'code') { $results[] = $cell; continue; } $label = $this->newCellLabel($cell); - $lines = idx($cell, 'source'); - if (!is_array($lines)) { - $lines = array(); - } - + $lines = $this->readStringList($cell, 'source'); $content = $this->highlightLines($lines); $count = count($lines); for ($ii = 0; $ii < $count; $ii++) { $is_head = ($ii === 0); $is_last = ($ii === ($count - 1)); if ($is_head) { $line_label = $label; } else { $line_label = null; } $results[] = array( 'cell_type' => 'code/line', 'label' => $line_label, 'raw' => $lines[$ii], 'display' => idx($content, $ii), 'head' => $is_head, 'last' => $is_last, ); } $outputs = array(); $output_list = idx($cell, 'outputs'); if (is_array($output_list)) { foreach ($output_list as $output) { $results[] = array( 'cell_type' => 'code/output', 'output' => $output, ); } } } return $results; } private function renderJupyterCell( PhabricatorUser $viewer, array $cell) { list($label, $content) = $this->renderJupyterCellContent($viewer, $cell); $classes = null; switch (idx($cell, 'cell_type')) { case 'code/line': $classes = 'jupyter-cell-flush'; break; } return $this->newJupyterCell( $label, $content, $classes); } private function newJupyterCell($label, $content, $classes) { $label_cell = phutil_tag( 'td', array( 'class' => 'jupyter-label', ), $label); $content_cell = phutil_tag( 'td', array( 'class' => $classes, ), $content); return phutil_tag( 'tr', array(), array( $label_cell, $content_cell, )); } private function renderJupyterCellContent( PhabricatorUser $viewer, array $cell) { $cell_type = idx($cell, 'cell_type'); switch ($cell_type) { case 'markdown': return $this->newMarkdownCell($cell); case 'code': return $this->newCodeCell($cell); case 'code/line': return $this->newCodeLineCell($cell); case 'code/output': return $this->newCodeOutputCell($cell); } $json_content = id(new PhutilJSON()) ->encodeFormatted($cell); return $this->newRawCell($json_content); } private function newRawCell($content) { return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-raw PhabricatorMonospaced', ), $content), ); } private function newMarkdownCell(array $cell) { - $content = idx($cell, 'source'); - if (!is_array($content)) { - $content = array(); - } + $content = $this->readStringList($cell, 'source'); // TODO: This should ideally highlight as Markdown, but the "md" // highlighter in Pygments is painfully slow and not terribly useful. $content = $this->highlightLines($content, 'txt'); return array( null, phutil_tag( 'div', array( 'class' => 'jupyter-cell-markdown', ), $content), ); } private function newCodeCell(array $cell) { $label = $this->newCellLabel($cell); - $content = idx($cell, 'source'); - if (!is_array($content)) { - $content = array(); - } - + $content = $this->readStringList($cell, 'source'); $content = $this->highlightLines($content); $outputs = array(); $output_list = idx($cell, 'outputs'); if (is_array($output_list)) { foreach ($output_list as $output) { $outputs[] = $this->newOutput($output); } } return array( $label, array( phutil_tag( 'div', array( 'class' => 'jupyter-cell-code jupyter-cell-code-block '. 'PhabricatorMonospaced remarkup-code', ), array( $content, )), $outputs, ), ); } private function newCodeLineCell(array $cell, $content = null) { $classes = array(); $classes[] = 'PhabricatorMonospaced'; $classes[] = 'remarkup-code'; $classes[] = 'jupyter-cell-code'; $classes[] = 'jupyter-cell-code-line'; if ($cell['head']) { $classes[] = 'jupyter-cell-code-head'; } if ($cell['last']) { $classes[] = 'jupyter-cell-code-last'; } $classes = implode(' ', $classes); if ($content === null) { $content = $cell['display']; } return array( $cell['label'], array( phutil_tag( 'div', array( 'class' => $classes, ), array( $content, )), ), ); } private function newCodeOutputCell(array $cell) { return array( null, $this->newOutput($cell['output']), ); } private function newOutput(array $output) { if (!is_array($output)) { return pht(''); } $classes = array( 'jupyter-output', 'PhabricatorMonospaced', ); $output_name = idx($output, 'name'); switch ($output_name) { case 'stderr': $classes[] = 'jupyter-output-stderr'; break; } $output_type = idx($output, 'output_type'); switch ($output_type) { case 'execute_result': case 'display_data': $data = idx($output, 'data'); $image_formats = array( 'image/png', 'image/jpeg', 'image/jpg', 'image/gif', ); foreach ($image_formats as $image_format) { if (!isset($data[$image_format])) { continue; } - $raw_data = $data[$image_format]; - if (!is_array($raw_data)) { - $raw_data = array($raw_data); - } - $raw_data = implode('', $raw_data); + $raw_data = $this->readString($data, $image_format); $content = phutil_tag( 'img', array( 'src' => 'data:'.$image_format.';base64,'.$raw_data, )); break 2; } if (isset($data['text/html'])) { $content = $data['text/html']; $classes[] = 'jupyter-output-html'; break; } if (isset($data['application/javascript'])) { $content = $data['application/javascript']; $classes[] = 'jupyter-output-html'; break; } if (isset($data['text/plain'])) { $content = $data['text/plain']; break; } break; case 'stream': default: - $content = idx($output, 'text'); - if (!is_array($content)) { - $content = array(); - } - $content = implode('', $content); + $content = $this->readString($output, 'text'); break; } return phutil_tag( 'div', array( 'class' => implode(' ', $classes), ), $content); } private function newCellLabel(array $cell) { $execution_count = idx($cell, 'execution_count'); if ($execution_count) { $label = 'In ['.$execution_count.']:'; } else { $label = null; } return $label; } private function highlightLines(array $lines, $force_language = null) { if ($force_language === null) { $head = head($lines); $matches = null; if (preg_match('/^%%(.*)$/', $head, $matches)) { $restore = array_shift($lines); $lang = $matches[1]; } else { $restore = null; $lang = 'py'; } } else { $restore = null; $lang = $force_language; } $content = PhabricatorSyntaxHighlighter::highlightWithLanguage( $lang, implode('', $lines)); $content = phutil_split_lines($content); if ($restore !== null) { $language_tag = phutil_tag( 'span', array( 'class' => 'language-tag', ), $restore); array_unshift($content, $language_tag); } return $content; } public function shouldSuggestEngine(PhabricatorDocumentRef $ref) { return true; } + private function readString(array $src, $key) { + $list = $this->readStringList($src, $key); + return implode('', $list); + } + + private function readStringList(array $src, $key) { + $list = idx($src, $key); + + if (is_array($list)) { + $list = $list; + } else if (is_string($list)) { + $list = array($list); + } else { + $list = array(); + } + + return $list; + } + } diff --git a/src/infrastructure/diff/engine/PhabricatorInlineCommentAdjustmentEngine.php b/src/infrastructure/diff/engine/PhabricatorInlineCommentAdjustmentEngine.php index 095a44c221..f1888cc363 100644 --- a/src/infrastructure/diff/engine/PhabricatorInlineCommentAdjustmentEngine.php +++ b/src/infrastructure/diff/engine/PhabricatorInlineCommentAdjustmentEngine.php @@ -1,372 +1,372 @@ viewer = $viewer; return $this; } public function getViewer() { return $this->viewer; } public function setInlines(array $inlines) { assert_instances_of($inlines, 'DifferentialInlineComment'); $this->inlines = $inlines; return $this; } public function getInlines() { return $this->inlines; } public function setOldChangesets(array $old_changesets) { assert_instances_of($old_changesets, 'DifferentialChangeset'); $this->oldChangesets = $old_changesets; return $this; } public function getOldChangesets() { return $this->oldChangesets; } public function setNewChangesets(array $new_changesets) { assert_instances_of($new_changesets, 'DifferentialChangeset'); $this->newChangesets = $new_changesets; return $this; } public function getNewChangesets() { return $this->newChangesets; } public function setRevision(DifferentialRevision $revision) { $this->revision = $revision; return $this; } public function getRevision() { return $this->revision; } public function execute() { $viewer = $this->getViewer(); $inlines = $this->getInlines(); $revision = $this->getRevision(); $old = $this->getOldChangesets(); $new = $this->getNewChangesets(); $no_ghosts = $viewer->compareUserSetting( PhabricatorOlderInlinesSetting::SETTINGKEY, PhabricatorOlderInlinesSetting::VALUE_GHOST_INLINES_DISABLED); if ($no_ghosts) { return $inlines; } $all = array_merge($old, $new); $changeset_ids = mpull($inlines, 'getChangesetID'); $changeset_ids = array_unique($changeset_ids); $all_map = mpull($all, null, 'getID'); // We already have at least some changesets, and we might not need to do // any more data fetching. Remove everything we already have so we can // tell if we need new stuff. foreach ($changeset_ids as $key => $id) { if (isset($all_map[$id])) { unset($changeset_ids[$key]); } } if ($changeset_ids) { $changesets = id(new DifferentialChangesetQuery()) ->setViewer($viewer) ->withIDs($changeset_ids) ->execute(); $changesets = mpull($changesets, null, 'getID'); } else { $changesets = array(); } $changesets += $all_map; $id_map = array(); foreach ($all as $changeset) { $id_map[$changeset->getID()] = $changeset->getID(); } // Generate filename maps for older and newer comments. If we're bringing // an older comment forward in a diff-of-diffs, we want to put it on the // left side of the screen, not the right side. Both sides are "new" files // with the same name, so they're both appropriate targets, but the left // is a better target conceptually for users because it's more consistent // with the rest of the UI, which shows old information on the left and // new information on the right. $move_here = DifferentialChangeType::TYPE_MOVE_HERE; $name_map_old = array(); $name_map_new = array(); $move_map = array(); foreach ($all as $changeset) { $changeset_id = $changeset->getID(); $filenames = array(); $filenames[] = $changeset->getFilename(); // If this is the target of a move, also map comments on the old filename // to this changeset. if ($changeset->getChangeType() == $move_here) { $old_file = $changeset->getOldFile(); $filenames[] = $old_file; $move_map[$changeset_id][$old_file] = true; } foreach ($filenames as $filename) { // We update the old map only if we don't already have an entry (oldest // changeset persists). if (empty($name_map_old[$filename])) { $name_map_old[$filename] = $changeset_id; } // We always update the new map (newest changeset overwrites). $name_map_new[$changeset->getFilename()] = $changeset_id; } } // Find the smallest "new" changeset ID. We'll consider everything // larger than this to be "newer", and everything smaller to be "older". $first_new_id = min(mpull($new, 'getID')); $results = array(); foreach ($inlines as $inline) { $changeset_id = $inline->getChangesetID(); if (isset($id_map[$changeset_id])) { // This inline is legitimately on one of the current changesets, so // we can include it in the result set unmodified. $results[] = $inline; continue; } $changeset = idx($changesets, $changeset_id); if (!$changeset) { // Just discard this inline, as it has bogus data. continue; } $target_id = null; if ($changeset_id >= $first_new_id) { $name_map = $name_map_new; $is_new = true; } else { $name_map = $name_map_old; $is_new = false; } $filename = $changeset->getFilename(); if (isset($name_map[$filename])) { // This changeset is on a file with the same name as the current // changeset, so we're going to port it forward or backward. $target_id = $name_map[$filename]; $is_move = isset($move_map[$target_id][$filename]); if ($is_new) { if ($is_move) { $reason = pht( 'This comment was made on a file with the same name as the '. 'file this file was moved from, but in a newer diff.'); } else { $reason = pht( 'This comment was made on a file with the same name, but '. 'in a newer diff.'); } } else { if ($is_move) { $reason = pht( 'This comment was made on a file with the same name as the '. 'file this file was moved from, but in an older diff.'); } else { $reason = pht( 'This comment was made on a file with the same name, but '. 'in an older diff.'); } } } // If we didn't find a target and this change is the target of a move, // look for a match against the old filename. if (!$target_id) { if ($changeset->getChangeType() == $move_here) { $filename = $changeset->getOldFile(); if (isset($name_map[$filename])) { $target_id = $name_map[$filename]; if ($is_new) { $reason = pht( 'This comment was made on a file which this file was moved '. 'to, but in a newer diff.'); } else { $reason = pht( 'This comment was made on a file which this file was moved '. 'to, but in an older diff.'); } } } } // If we found a changeset to port this comment to, bring it forward // or backward and mark it. if ($target_id) { $diff_id = $changeset->getDiffID(); $inline_id = $inline->getID(); $revision_id = $revision->getID(); $href = "/D{$revision_id}?id={$diff_id}#inline-{$inline_id}"; $inline ->makeEphemeral(true) ->setChangesetID($target_id) ->setIsGhost( array( 'new' => $is_new, 'reason' => $reason, 'href' => $href, 'originalID' => $changeset->getID(), )); $results[] = $inline; } } // Filter out the inlines we ported forward which won't be visible because // they appear on the wrong side of a file. $keep_map = array(); foreach ($old as $changeset) { $keep_map[$changeset->getID()][0] = true; } foreach ($new as $changeset) { $keep_map[$changeset->getID()][1] = true; } foreach ($results as $key => $inline) { $is_new = (int)$inline->getIsNewFile(); $changeset_id = $inline->getChangesetID(); if (!isset($keep_map[$changeset_id][$is_new])) { unset($results[$key]); continue; } } // Adjust inline line numbers to account for content changes across // updates and rebases. $plan = array(); $need = array(); foreach ($results as $inline) { $ghost = $inline->getIsGhost(); if (!$ghost) { // If this isn't a "ghost" inline, ignore it. continue; } $src_id = $ghost['originalID']; $dst_id = $inline->getChangesetID(); $xforms = array(); // If the comment is on the right, transform it through the inverse map // back to the left. if ($inline->getIsNewFile()) { $xforms[] = array($src_id, $src_id, true); } // Transform it across rebases. $xforms[] = array($src_id, $dst_id, false); // If the comment is on the right, transform it back onto the right. if ($inline->getIsNewFile()) { $xforms[] = array($dst_id, $dst_id, false); } $key = array(); foreach ($xforms as $xform) { list($u, $v, $inverse) = $xform; $short = $u.'/'.$v; $need[$short] = array($u, $v); $part = $u.($inverse ? '<' : '>').$v; $key[] = $part; } $key = implode(',', $key); if (empty($plan[$key])) { $plan[$key] = array( 'xforms' => $xforms, 'inlines' => array(), ); } $plan[$key]['inlines'][] = $inline; } if ($need) { $maps = DifferentialLineAdjustmentMap::loadMaps($need); } else { $maps = array(); } foreach ($plan as $step) { $xforms = $step['xforms']; $chain = null; foreach ($xforms as $xform) { list($u, $v, $inverse) = $xform; $map = idx(idx($maps, $u, array()), $v); if (!$map) { continue 2; } if ($inverse) { $map = DifferentialLineAdjustmentMap::newInverseMap($map); } else { $map = clone $map; } if ($chain) { $chain->addMapToChain($map); } else { $chain = $map; } } foreach ($step['inlines'] as $inline) { $head_line = $inline->getLineNumber(); $tail_line = ($head_line + $inline->getLineLength()); $head_info = $chain->mapLine($head_line, false); $tail_info = $chain->mapLine($tail_line, true); list($head_deleted, $head_offset, $head_line) = $head_info; list($tail_deleted, $tail_offset, $tail_line) = $tail_info; if ($head_offset !== false) { - $inline->setLineNumber($head_line + 1 + $head_offset); + $inline->setLineNumber($head_line + $head_offset); } else { $inline->setLineNumber($head_line); $inline->setLineLength($tail_line - $head_line); } } } return $results; } }