diff --git a/.arclint b/.arclint --- a/.arclint +++ b/.arclint @@ -64,13 +64,13 @@ "text": { "type": "text", "exclude": [ - "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))" + "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))" ] }, "text-without-length": { "type": "text", "include": [ - "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))" + "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))" ], "severity": { "3": "disabled" diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -5591,6 +5591,7 @@ 'PhrictionTransactionComment' => 'applications/phriction/storage/PhrictionTransactionComment.php', 'PhrictionTransactionEditor' => 'applications/phriction/editor/PhrictionTransactionEditor.php', 'PhrictionTransactionQuery' => 'applications/phriction/query/PhrictionTransactionQuery.php', + 'PhutilAPCKeyValueCache' => 'infrastructure/cache/PhutilAPCKeyValueCache.php', 'PhutilAmazonAuthAdapter' => 'applications/auth/adapter/PhutilAmazonAuthAdapter.php', 'PhutilAsanaAuthAdapter' => 'applications/auth/adapter/PhutilAsanaAuthAdapter.php', 'PhutilAuthAdapter' => 'applications/auth/adapter/PhutilAuthAdapter.php', @@ -5620,7 +5621,15 @@ 'PhutilCalendarRootNode' => 'applications/calendar/parser/data/PhutilCalendarRootNode.php', 'PhutilCalendarUserNode' => 'applications/calendar/parser/data/PhutilCalendarUserNode.php', 'PhutilCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php', + 'PhutilConsoleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php', + 'PhutilContextFreeGrammar' => 'infrastructure/lipsum/PhutilContextFreeGrammar.php', + 'PhutilDefaultSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php', + 'PhutilDefaultSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php', + 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php', + 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php', + 'PhutilDirectoryKeyValueCache' => 'infrastructure/cache/PhutilDirectoryKeyValueCache.php', 'PhutilDisqusAuthAdapter' => 'applications/auth/adapter/PhutilDisqusAuthAdapter.php', + 'PhutilDivinerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php', 'PhutilEmptyAuthAdapter' => 'applications/auth/adapter/PhutilEmptyAuthAdapter.php', 'PhutilFacebookAuthAdapter' => 'applications/auth/adapter/PhutilFacebookAuthAdapter.php', 'PhutilGitHubAuthAdapter' => 'applications/auth/adapter/PhutilGitHubAuthAdapter.php', @@ -5630,19 +5639,36 @@ 'PhutilICSParserTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php', 'PhutilICSWriter' => 'applications/calendar/parser/ics/PhutilICSWriter.php', 'PhutilICSWriterTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php', + 'PhutilInRequestKeyValueCache' => 'infrastructure/cache/PhutilInRequestKeyValueCache.php', + 'PhutilInvisibleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php', 'PhutilJIRAAuthAdapter' => 'applications/auth/adapter/PhutilJIRAAuthAdapter.php', + 'PhutilJSONFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php', 'PhutilJavaCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php', + 'PhutilKeyValueCache' => 'infrastructure/cache/PhutilKeyValueCache.php', + 'PhutilKeyValueCacheNamespace' => 'infrastructure/cache/PhutilKeyValueCacheNamespace.php', + 'PhutilKeyValueCacheProfiler' => 'infrastructure/cache/PhutilKeyValueCacheProfiler.php', + 'PhutilKeyValueCacheProxy' => 'infrastructure/cache/PhutilKeyValueCacheProxy.php', + 'PhutilKeyValueCacheStack' => 'infrastructure/cache/PhutilKeyValueCacheStack.php', + 'PhutilKeyValueCacheTestCase' => 'infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php', 'PhutilLDAPAuthAdapter' => 'applications/auth/adapter/PhutilLDAPAuthAdapter.php', + 'PhutilLexerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php', 'PhutilLipsumContextFreeGrammar' => 'infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php', + 'PhutilMarkupEngine' => 'infrastructure/markup/PhutilMarkupEngine.php', + 'PhutilMarkupTestCase' => 'infrastructure/markup/__tests__/PhutilMarkupTestCase.php', + 'PhutilMemcacheKeyValueCache' => 'infrastructure/cache/PhutilMemcacheKeyValueCache.php', 'PhutilOAuth1AuthAdapter' => 'applications/auth/adapter/PhutilOAuth1AuthAdapter.php', 'PhutilOAuthAuthAdapter' => 'applications/auth/adapter/PhutilOAuthAuthAdapter.php', + 'PhutilOnDiskKeyValueCache' => 'infrastructure/cache/PhutilOnDiskKeyValueCache.php', 'PhutilPHPCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php', + 'PhutilPHPFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php', 'PhutilPhabricatorAuthAdapter' => 'applications/auth/adapter/PhutilPhabricatorAuthAdapter.php', 'PhutilProseDiff' => 'infrastructure/diff/prose/PhutilProseDiff.php', 'PhutilProseDiffTestCase' => 'infrastructure/diff/prose/__tests__/PhutilProseDiffTestCase.php', 'PhutilProseDifferenceEngine' => 'infrastructure/diff/prose/PhutilProseDifferenceEngine.php', + 'PhutilPygmentsSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php', 'PhutilQsprintfInterface' => 'infrastructure/storage/xsprintf/PhutilQsprintfInterface.php', 'PhutilQueryString' => 'infrastructure/storage/xsprintf/PhutilQueryString.php', + 'PhutilRainbowSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php', 'PhutilRealNameContextFreeGrammar' => 'infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php', 'PhutilRemarkupAnchorRule' => 'infrastructure/markup/markuprule/PhutilRemarkupAnchorRule.php', 'PhutilRemarkupBlockInterpreter' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php', @@ -5678,6 +5704,9 @@ 'PhutilRemarkupTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php', 'PhutilRemarkupTestInterpreterRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php', 'PhutilRemarkupUnderlineRule' => 'infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php', + 'PhutilSafeHTML' => 'infrastructure/markup/PhutilSafeHTML.php', + 'PhutilSafeHTMLProducerInterface' => 'infrastructure/markup/PhutilSafeHTMLProducerInterface.php', + 'PhutilSafeHTMLTestCase' => 'infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php', 'PhutilSearchQueryCompiler' => 'applications/search/compiler/PhutilSearchQueryCompiler.php', 'PhutilSearchQueryCompilerSyntaxException' => 'applications/search/compiler/PhutilSearchQueryCompilerSyntaxException.php', 'PhutilSearchQueryCompilerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchQueryCompilerTestCase.php', @@ -5685,9 +5714,17 @@ 'PhutilSearchStemmer' => 'applications/search/compiler/PhutilSearchStemmer.php', 'PhutilSearchStemmerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchStemmerTestCase.php', 'PhutilSlackAuthAdapter' => 'applications/auth/adapter/PhutilSlackAuthAdapter.php', + 'PhutilSprite' => 'aphront/sprite/PhutilSprite.php', + 'PhutilSpriteSheet' => 'aphront/sprite/PhutilSpriteSheet.php', + 'PhutilSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php', + 'PhutilSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php', + 'PhutilSyntaxHighlighterException' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php', 'PhutilTwitchAuthAdapter' => 'applications/auth/adapter/PhutilTwitchAuthAdapter.php', 'PhutilTwitterAuthAdapter' => 'applications/auth/adapter/PhutilTwitterAuthAdapter.php', 'PhutilWordPressAuthAdapter' => 'applications/auth/adapter/PhutilWordPressAuthAdapter.php', + 'PhutilXHPASTSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php', + 'PhutilXHPASTSyntaxHighlighterFuture' => 'infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php', + 'PhutilXHPASTSyntaxHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php', 'PolicyLockOptionType' => 'applications/policy/config/PolicyLockOptionType.php', 'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php', 'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php', @@ -5879,6 +5916,7 @@ 'function' => array( 'celerity_generate_unique_node_id' => 'applications/celerity/api.php', 'celerity_get_resource_uri' => 'applications/celerity/api.php', + 'hsprintf' => 'infrastructure/markup/render.php', 'javelin_tag' => 'infrastructure/javelin/markup.php', 'phabricator_date' => 'view/viewutils.php', 'phabricator_datetime' => 'view/viewutils.php', @@ -5890,6 +5928,15 @@ 'phid_get_subtype' => 'applications/phid/utils.php', 'phid_get_type' => 'applications/phid/utils.php', 'phid_group_by_type' => 'applications/phid/utils.php', + 'phutil_escape_html' => 'infrastructure/markup/render.php', + 'phutil_escape_html_newlines' => 'infrastructure/markup/render.php', + 'phutil_escape_uri' => 'infrastructure/markup/render.php', + 'phutil_escape_uri_path_component' => 'infrastructure/markup/render.php', + 'phutil_implode_html' => 'infrastructure/markup/render.php', + 'phutil_safe_html' => 'infrastructure/markup/render.php', + 'phutil_tag' => 'infrastructure/markup/render.php', + 'phutil_tag_div' => 'infrastructure/markup/render.php', + 'phutil_unescape_uri_path_component' => 'infrastructure/markup/render.php', 'qsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php', 'qsprintf_check_scalar_type' => 'infrastructure/storage/xsprintf/qsprintf.php', 'qsprintf_check_type' => 'infrastructure/storage/xsprintf/qsprintf.php', @@ -12443,6 +12490,7 @@ 'PhrictionTransactionComment' => 'PhabricatorApplicationTransactionComment', 'PhrictionTransactionEditor' => 'PhabricatorApplicationTransactionEditor', 'PhrictionTransactionQuery' => 'PhabricatorApplicationTransactionQuery', + 'PhutilAPCKeyValueCache' => 'PhutilKeyValueCache', 'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilAuthAdapter' => 'Phobject', @@ -12472,7 +12520,15 @@ 'PhutilCalendarRootNode' => 'PhutilCalendarContainerNode', 'PhutilCalendarUserNode' => 'PhutilCalendarNode', 'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilConsoleSyntaxHighlighter' => 'Phobject', + 'PhutilContextFreeGrammar' => 'Phobject', + 'PhutilDefaultSyntaxHighlighter' => 'Phobject', + 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine', + 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy', + 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'PhutilTestCase', + 'PhutilDirectoryKeyValueCache' => 'PhutilKeyValueCache', 'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilDivinerSyntaxHighlighter' => 'Phobject', 'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter', 'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter', @@ -12482,18 +12538,35 @@ 'PhutilICSParserTestCase' => 'PhutilTestCase', 'PhutilICSWriter' => 'Phobject', 'PhutilICSWriterTestCase' => 'PhutilTestCase', + 'PhutilInRequestKeyValueCache' => 'PhutilKeyValueCache', + 'PhutilInvisibleSyntaxHighlighter' => 'Phobject', 'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter', + 'PhutilJSONFragmentLexerHighlighterTestCase' => 'PhutilTestCase', 'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilKeyValueCache' => 'Phobject', + 'PhutilKeyValueCacheNamespace' => 'PhutilKeyValueCacheProxy', + 'PhutilKeyValueCacheProfiler' => 'PhutilKeyValueCacheProxy', + 'PhutilKeyValueCacheProxy' => 'PhutilKeyValueCache', + 'PhutilKeyValueCacheStack' => 'PhutilKeyValueCache', + 'PhutilKeyValueCacheTestCase' => 'PhutilTestCase', 'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilLexerSyntaxHighlighter' => 'PhutilSyntaxHighlighter', 'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar', + 'PhutilMarkupEngine' => 'Phobject', + 'PhutilMarkupTestCase' => 'PhutilTestCase', + 'PhutilMemcacheKeyValueCache' => 'PhutilKeyValueCache', 'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter', 'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter', + 'PhutilOnDiskKeyValueCache' => 'PhutilKeyValueCache', 'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', + 'PhutilPHPFragmentLexerHighlighterTestCase' => 'PhutilTestCase', 'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilProseDiff' => 'Phobject', 'PhutilProseDiffTestCase' => 'PhabricatorTestCase', 'PhutilProseDifferenceEngine' => 'Phobject', + 'PhutilPygmentsSyntaxHighlighter' => 'Phobject', 'PhutilQueryString' => 'Phobject', + 'PhutilRainbowSyntaxHighlighter' => 'Phobject', 'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar', 'PhutilRemarkupAnchorRule' => 'PhutilRemarkupRule', 'PhutilRemarkupBlockInterpreter' => 'Phobject', @@ -12529,6 +12602,8 @@ 'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter', 'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule', + 'PhutilSafeHTML' => 'Phobject', + 'PhutilSafeHTMLTestCase' => 'PhutilTestCase', 'PhutilSearchQueryCompiler' => 'Phobject', 'PhutilSearchQueryCompilerSyntaxException' => 'Exception', 'PhutilSearchQueryCompilerTestCase' => 'PhutilTestCase', @@ -12536,9 +12611,17 @@ 'PhutilSearchStemmer' => 'Phobject', 'PhutilSearchStemmerTestCase' => 'PhutilTestCase', 'PhutilSlackAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilSprite' => 'Phobject', + 'PhutilSpriteSheet' => 'Phobject', + 'PhutilSyntaxHighlighter' => 'Phobject', + 'PhutilSyntaxHighlighterEngine' => 'Phobject', + 'PhutilSyntaxHighlighterException' => 'Exception', 'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter', 'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilXHPASTSyntaxHighlighter' => 'Phobject', + 'PhutilXHPASTSyntaxHighlighterFuture' => 'FutureProxy', + 'PhutilXHPASTSyntaxHighlighterTestCase' => 'PhutilTestCase', 'PolicyLockOptionType' => 'PhabricatorConfigJSONOptionType', 'PonderAddAnswerView' => 'AphrontView', 'PonderAnswer' => array( diff --git a/src/aphront/sprite/PhutilSprite.php b/src/aphront/sprite/PhutilSprite.php new file mode 100644 --- /dev/null +++ b/src/aphront/sprite/PhutilSprite.php @@ -0,0 +1,76 @@ +name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + public function setTargetCSS($target_css) { + $this->targetCSS = $target_css; + return $this; + } + + public function getTargetCSS() { + return $this->targetCSS; + } + + public function setSourcePosition($x, $y) { + $this->sourceX = $x; + $this->sourceY = $y; + return $this; + } + + public function setSourceSize($w, $h) { + $this->sourceW = $w; + $this->sourceH = $h; + return $this; + } + + public function getSourceH() { + return $this->sourceH; + } + + public function getSourceW() { + return $this->sourceW; + } + + public function getSourceY() { + return $this->sourceY; + } + + public function getSourceX() { + return $this->sourceX; + } + + public function setSourceFile($source_file, $scale = 1) { + $this->sourceFiles[$scale] = $source_file; + return $this; + } + + public function getSourceFile($scale) { + if (empty($this->sourceFiles[$scale])) { + throw new Exception(pht("No source file for scale '%s'!", $scale)); + } + + return $this->sourceFiles[$scale]; + } + +} diff --git a/src/aphront/sprite/PhutilSpriteSheet.php b/src/aphront/sprite/PhutilSpriteSheet.php new file mode 100644 --- /dev/null +++ b/src/aphront/sprite/PhutilSpriteSheet.php @@ -0,0 +1,385 @@ +generated = false; + $this->sprites[] = $sprite; + return $this; + } + + public function setCSSHeader($header) { + $this->generated = false; + $this->cssHeader = $header; + return $this; + } + + public function setScales(array $scales) { + $this->scales = array_values($scales); + return $this; + } + + public function getScales() { + return $this->scales; + } + + public function setSheetType($type) { + $this->type = $type; + return $this; + } + + public function setBasePath($base_path) { + $this->basePath = $base_path; + return $this; + } + + private function generate() { + if ($this->generated) { + return; + } + + $multi_row = true; + $multi_col = true; + $margin_w = 1; + $margin_h = 1; + + $type = $this->type; + switch ($type) { + case self::TYPE_STANDARD: + break; + case self::TYPE_REPEAT_X: + $multi_col = false; + $margin_w = 0; + + $width = null; + foreach ($this->sprites as $sprite) { + if ($width === null) { + $width = $sprite->getSourceW(); + } else if ($width !== $sprite->getSourceW()) { + throw new Exception( + pht( + "All sprites in a '%s' sheet must have the same width.", + 'repeat-x')); + } + } + break; + case self::TYPE_REPEAT_Y: + $multi_row = false; + $margin_h = 0; + + $height = null; + foreach ($this->sprites as $sprite) { + if ($height === null) { + $height = $sprite->getSourceH(); + } else if ($height !== $sprite->getSourceH()) { + throw new Exception( + pht( + "All sprites in a '%s' sheet must have the same height.", + 'repeat-y')); + } + } + break; + default: + throw new Exception(pht("Unknown sprite sheet type '%s'!", $type)); + } + + + $css = array(); + if ($this->cssHeader) { + $css[] = $this->cssHeader; + } + + $out_w = 0; + $out_h = 0; + + // Lay out the sprite sheet. We attempt to build a roughly square sheet + // so it's easier to manage, since 2000x20 is more cumbersome for humans + // to deal with than 200x200. + // + // To do this, we use a simple greedy algorithm, adding sprites one at a + // time. For each sprite, if the sheet is at least as wide as it is tall + // we create a new row. Otherwise, we try to add it to an existing row. + // + // This isn't optimal, but does a reasonable job in most cases and isn't + // too messy. + + // Group the sprites by their sizes. We lay them out in the sheet as + // boxes, but then put them into the boxes in the order they were added + // so similar sprites end up nearby on the final sheet. + $boxes = array(); + foreach (array_reverse($this->sprites) as $sprite) { + $s_w = $sprite->getSourceW() + $margin_w; + $s_h = $sprite->getSourceH() + $margin_h; + $boxes[$s_w][$s_h][] = $sprite; + } + + $rows = array(); + foreach ($this->sprites as $sprite) { + $s_w = $sprite->getSourceW() + $margin_w; + $s_h = $sprite->getSourceH() + $margin_h; + + // Choose a row for this sprite. + $maybe = array(); + foreach ($rows as $key => $row) { + if ($row['h'] < $s_h) { + // We can only add it to a row if the row is at least as tall as the + // sprite. + continue; + } + // We prefer rows which have the same height as the sprite, and then + // rows which aren't yet very wide. + $wasted_v = ($row['h'] - $s_h); + $wasted_h = ($row['w'] / $out_w); + $maybe[$key] = $wasted_v + $wasted_h; + } + + $row_key = null; + if ($maybe && $multi_col) { + // If there were any candidate rows, pick the best one. + asort($maybe); + $row_key = head_key($maybe); + } + + if ($row_key !== null && $multi_row) { + // If there's a candidate row, but adding the sprite to it would make + // the sprite wider than it is tall, create a new row instead. This + // generally keeps the sprite square-ish. + if ($rows[$row_key]['w'] + $s_w > $out_h) { + $row_key = null; + } + } + + if ($row_key === null) { + // Add a new row. + $rows[] = array( + 'w' => 0, + 'h' => $s_h, + 'boxes' => array(), + ); + $row_key = last_key($rows); + $out_h += $s_h; + } + + // Add the sprite box to the row. + $row = $rows[$row_key]; + $row['w'] += $s_w; + $row['boxes'][] = array($s_w, $s_h); + $rows[$row_key] = $row; + + $out_w = max($row['w'], $out_w); + } + + $images = array(); + foreach ($this->scales as $scale) { + $img = imagecreatetruecolor($out_w * $scale, $out_h * $scale); + imagesavealpha($img, true); + imagefill($img, 0, 0, imagecolorallocatealpha($img, 0, 0, 0, 127)); + + $images[$scale] = $img; + } + + + // Put the shorter rows first. At the same height, put the wider rows first. + // This makes the resulting sheet more human-readable. + foreach ($rows as $key => $row) { + $rows[$key]['sort'] = $row['h'] + (1 - ($row['w'] / $out_w)); + } + $rows = isort($rows, 'sort'); + + $pos_x = 0; + $pos_y = 0; + $rules = array(); + foreach ($rows as $row) { + $max_h = 0; + foreach ($row['boxes'] as $box) { + $sprite = array_pop($boxes[$box[0]][$box[1]]); + + foreach ($images as $scale => $img) { + $src = $this->loadSource($sprite, $scale); + imagecopy( + $img, + $src, + $scale * $pos_x, $scale * $pos_y, + $scale * $sprite->getSourceX(), $scale * $sprite->getSourceY(), + $scale * $sprite->getSourceW(), $scale * $sprite->getSourceH()); + } + + $rule = $sprite->getTargetCSS(); + $cssx = (-$pos_x).'px'; + $cssy = (-$pos_y).'px'; + + $rules[$sprite->getName()] = "{$rule} {\n". + " background-position: {$cssx} {$cssy};\n}"; + + $pos_x += $sprite->getSourceW() + $margin_w; + $max_h = max($max_h, $sprite->getSourceH()); + } + $pos_x = 0; + $pos_y += $max_h + $margin_h; + } + + // Generate CSS rules in input order. + foreach ($this->sprites as $sprite) { + $css[] = $rules[$sprite->getName()]; + } + + $this->images = $images; + $this->css = implode("\n\n", $css)."\n"; + $this->generated = true; + } + + public function generateImage($path, $scale = 1) { + $this->generate(); + $this->log(pht("Writing sprite '%s'...", $path)); + imagepng($this->images[$scale], $path); + return $this; + } + + public function generateCSS($path) { + $this->generate(); + $this->log(pht("Writing CSS '%s'...", $path)); + + $out = $this->css; + $out = str_replace('{X}', imagesx($this->images[1]), $out); + $out = str_replace('{Y}', imagesy($this->images[1]), $out); + + Filesystem::writeFile($path, $out); + return $this; + } + + public function needsRegeneration(array $manifest) { + return ($this->buildManifest() !== $manifest); + } + + private function buildManifest() { + $output = array(); + foreach ($this->sprites as $sprite) { + $output[$sprite->getName()] = array( + 'name' => $sprite->getName(), + 'rule' => $sprite->getTargetCSS(), + 'hash' => $this->loadSourceHash($sprite), + ); + } + + ksort($output); + + $data = array( + 'version' => self::MANIFEST_VERSION, + 'sprites' => $output, + 'scales' => $this->scales, + 'header' => $this->cssHeader, + 'type' => $this->type, + ); + + return $data; + } + + public function generateManifest($path) { + $data = $this->buildManifest(); + + $json = new PhutilJSON(); + $data = $json->encodeFormatted($data); + Filesystem::writeFile($path, $data); + return $this; + } + + private function log($message) { + echo $message."\n"; + } + + private function loadSourceHash(PhutilSprite $sprite) { + $inputs = array(); + + foreach ($this->scales as $scale) { + $file = $sprite->getSourceFile($scale); + + // If two users have a project in different places, like: + // + // /home/alincoln/project + // /home/htaft/project + // + // ...we want to ignore the `/home/alincoln` part when hashing the sheet, + // since the sprites don't change when the project directory moves. If + // the base path is set, build the hashes using paths relative to the + // base path. + + $file_key = $file; + if ($this->basePath) { + $file_key = Filesystem::readablePath($file, $this->basePath); + } + + if (empty($this->hashes[$file_key])) { + $this->hashes[$file_key] = md5(Filesystem::readFile($file)); + } + + $inputs[] = $file_key; + $inputs[] = $this->hashes[$file_key]; + } + + $inputs[] = $sprite->getSourceX(); + $inputs[] = $sprite->getSourceY(); + $inputs[] = $sprite->getSourceW(); + $inputs[] = $sprite->getSourceH(); + + return md5(implode(':', $inputs)); + } + + private function loadSource(PhutilSprite $sprite, $scale) { + $file = $sprite->getSourceFile($scale); + if (empty($this->sources[$file])) { + $data = Filesystem::readFile($file); + $image = imagecreatefromstring($data); + $this->sources[$file] = array( + 'image' => $image, + 'x' => imagesx($image), + 'y' => imagesy($image), + ); + } + + $s_w = $sprite->getSourceW() * $scale; + $i_w = $this->sources[$file]['x']; + if ($s_w > $i_w) { + throw new Exception( + pht( + "Sprite source for '%s' is too small (expected width %d, found %d).", + $file, + $s_w, + $i_w)); + } + + $s_h = $sprite->getSourceH() * $scale; + $i_h = $this->sources[$file]['y']; + if ($s_h > $i_h) { + throw new Exception( + pht( + "Sprite source for '%s' is too small (expected height %d, found %d).", + $file, + $s_h, + $i_h)); + } + + return $this->sources[$file]['image']; + } + +} diff --git a/src/infrastructure/cache/PhutilAPCKeyValueCache.php b/src/infrastructure/cache/PhutilAPCKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilAPCKeyValueCache.php @@ -0,0 +1,97 @@ + $value) { + if ($is_apcu) { + apcu_store($key, $value, $ttl); + } else { + apc_store($key, $value, $ttl); + } + } + + return $this; + } + + public function deleteKeys(array $keys) { + static $is_apcu; + if ($is_apcu === null) { + $is_apcu = self::isAPCu(); + } + + foreach ($keys as $key) { + if ($is_apcu) { + apcu_delete($key); + } else { + apc_delete($key); + } + } + + return $this; + } + + public function destroyCache() { + static $is_apcu; + if ($is_apcu === null) { + $is_apcu = self::isAPCu(); + } + + if ($is_apcu) { + apcu_clear_cache(); + } else { + apc_clear_cache('user'); + } + + return $this; + } + + private static function isAPCu() { + return function_exists('apcu_fetch'); + } + +} diff --git a/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php @@ -0,0 +1,244 @@ +validateKeys($keys); + + try { + $this->lockCache(); + } catch (PhutilLockException $ex) { + return array(); + } + + $now = time(); + + $results = array(); + foreach ($keys as $key) { + $key_file = $this->getKeyFile($key); + try { + $data = Filesystem::readFile($key_file); + } catch (FilesystemException $ex) { + continue; + } + + $data = unserialize($data); + if (!$data) { + continue; + } + + if (isset($data['ttl']) && $data['ttl'] < $now) { + continue; + } + + $results[$key] = $data['value']; + } + + $this->unlockCache(); + + return $results; + } + + + public function setKeys(array $keys, $ttl = null) { + $this->validateKeys(array_keys($keys)); + + $this->lockCache(15); + + if ($ttl) { + $ttl_epoch = time() + $ttl; + } else { + $ttl_epoch = null; + } + + foreach ($keys as $key => $value) { + $dict = array( + 'value' => $value, + ); + if ($ttl_epoch) { + $dict['ttl'] = $ttl_epoch; + } + + try { + $key_file = $this->getKeyFile($key); + $key_dir = dirname($key_file); + if (!Filesystem::pathExists($key_dir)) { + Filesystem::createDirectory( + $key_dir, + $mask = 0755, + $recursive = true); + } + + $new_file = $key_file.'.new'; + Filesystem::writeFile($new_file, serialize($dict)); + Filesystem::rename($new_file, $key_file); + } catch (FilesystemException $ex) { + phlog($ex); + } + } + + $this->unlockCache(); + + return $this; + } + + + public function deleteKeys(array $keys) { + $this->validateKeys($keys); + + $this->lockCache(15); + + foreach ($keys as $key) { + $path = $this->getKeyFile($key); + Filesystem::remove($path); + + // If removing this key leaves the directory empty, clean it up. Then + // clean up any empty parent directories. + $path = dirname($path); + do { + if (!Filesystem::isDescendant($path, $this->getCacheDirectory())) { + break; + } + if (Filesystem::listDirectory($path, true)) { + break; + } + Filesystem::remove($path); + $path = dirname($path); + } while (true); + } + + $this->unlockCache(); + + return $this; + } + + + public function destroyCache() { + Filesystem::remove($this->getCacheDirectory()); + return $this; + } + + +/* -( Cache Storage )------------------------------------------------------ */ + + + /** + * @task storage + */ + public function setCacheDirectory($directory) { + $this->cacheDirectory = rtrim($directory, '/').'/'; + return $this; + } + + + /** + * @task storage + */ + private function getCacheDirectory() { + if (!$this->cacheDirectory) { + throw new PhutilInvalidStateException('setCacheDirectory'); + } + return $this->cacheDirectory; + } + + + /** + * @task storage + */ + private function getKeyFile($key) { + // Colon is a drive separator on Windows. + $key = str_replace(':', '_', $key); + + // NOTE: We add ".cache" to each file so we don't get a collision if you + // set the keys "a" and "a/b". Without ".cache", the file "a" would need + // to be both a file and a directory. + return $this->getCacheDirectory().$key.'.cache'; + } + + + /** + * @task storage + */ + private function validateKeys(array $keys) { + foreach ($keys as $key) { + // NOTE: Use of "." is reserved for ".lock", "key.new" and "key.cache". + // Use of "_" is reserved for converting ":". + if (!preg_match('@^[a-zA-Z0-9/:-]+$@', $key)) { + throw new Exception( + pht( + "Invalid key '%s': directory caches may only contain letters, ". + "numbers, hyphen, colon and slash.", + $key)); + } + } + } + + + /** + * @task storage + */ + private function lockCache($wait = 0) { + if ($this->lock) { + throw new Exception( + pht( + 'Trying to %s with a lock!', + __FUNCTION__.'()')); + } + + if (!Filesystem::pathExists($this->getCacheDirectory())) { + Filesystem::createDirectory($this->getCacheDirectory(), 0755, true); + } + + $lock = PhutilFileLock::newForPath($this->getCacheDirectory().'.lock'); + $lock->lock($wait); + + $this->lock = $lock; + } + + + /** + * @task storage + */ + private function unlockCache() { + if (!$this->lock) { + throw new PhutilInvalidStateException('lockCache'); + } + + $this->lock->unlock(); + $this->lock = null; + } + +} diff --git a/src/infrastructure/cache/PhutilInRequestKeyValueCache.php b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php @@ -0,0 +1,118 @@ +limit = $limit; + return $this; + } + + +/* -( Key-Value Cache Implementation )------------------------------------- */ + + + public function isAvailable() { + return true; + } + + public function getKeys(array $keys) { + $results = array(); + $now = time(); + foreach ($keys as $key) { + if (!isset($this->cache[$key]) && !array_key_exists($key, $this->cache)) { + continue; + } + if (isset($this->ttl[$key]) && ($this->ttl[$key] < $now)) { + continue; + } + $results[$key] = $this->cache[$key]; + } + + return $results; + } + + public function setKeys(array $keys, $ttl = null) { + + foreach ($keys as $key => $value) { + $this->cache[$key] = $value; + } + + if ($ttl) { + $end = time() + $ttl; + foreach ($keys as $key => $value) { + $this->ttl[$key] = $end; + } + } else { + foreach ($keys as $key => $value) { + unset($this->ttl[$key]); + } + } + + if ($this->limit) { + $count = count($this->cache); + if ($count > $this->limit) { + $remove = array(); + foreach ($this->cache as $key => $value) { + $remove[] = $key; + + $count--; + if ($count <= $this->limit) { + break; + } + } + + $this->deleteKeys($remove); + } + } + + return $this; + } + + public function deleteKeys(array $keys) { + foreach ($keys as $key) { + unset($this->cache[$key]); + unset($this->ttl[$key]); + } + + return $this; + } + + public function getAllKeys() { + return $this->cache; + } + + public function destroyCache() { + $this->cache = array(); + $this->ttl = array(); + + return $this; + } + +} diff --git a/src/infrastructure/cache/PhutilKeyValueCache.php b/src/infrastructure/cache/PhutilKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilKeyValueCache.php @@ -0,0 +1,121 @@ +getKeys(array($key)); + return idx($map, $key, $default); + } + + + /** + * Set a single key in cache. See @{method:setKeys} to set multiple keys at + * once. + * + * See @{method:setKeys} for a description of TTLs. + * + * @param string Key to set. + * @param wild Value to set. + * @param int|null Optional TTL. + * @return this + * @task kvimpl + */ + final public function setKey($key, $value, $ttl = null) { + return $this->setKeys(array($key => $value), $ttl); + } + + + /** + * Delete a key from the cache. See @{method:deleteKeys} to delete multiple + * keys at once. + * + * @param string Key to delete. + * @return this + * @task kvimpl + */ + final public function deleteKey($key) { + return $this->deleteKeys(array($key)); + } + + + /** + * Get data from the cache. + * + * @param list List of cache keys to retrieve. + * @return dict Dictionary of keys that were found in the + * cache. Keys not present in the cache are + * omitted, so you can detect a cache miss. + * @task kvimpl + */ + abstract public function getKeys(array $keys); + + + /** + * Put data into the key-value cache. + * + * With a TTL ("time to live"), the cache will automatically delete the key + * after a specified number of seconds. By default, there is no expiration + * policy and data will persist in cache indefinitely. + * + * @param dict Map of cache keys to values. + * @param int|null TTL for cache keys, in seconds. + * @return this + * @task kvimpl + */ + abstract public function setKeys(array $keys, $ttl = null); + + + /** + * Delete a list of keys from the cache. + * + * @param list List of keys to delete. + * @return this + * @task kvimpl + */ + abstract public function deleteKeys(array $keys); + + + /** + * Completely destroy all data in the cache. + * + * @return this + * @task kvimpl + */ + abstract public function destroyCache(); + +} diff --git a/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php @@ -0,0 +1,65 @@ +namespace = $namespace.':'; + + return $this; + } + + public function setKeys(array $keys, $ttl = null) { + return parent::setKeys(array_combine( + $this->prefixKeys(array_keys($keys)), + $keys), $ttl); + } + + public function getKeys(array $keys) { + $results = parent::getKeys($this->prefixKeys($keys)); + + if (!$results) { + return array(); + } + + return array_combine( + $this->unprefixKeys(array_keys($results)), + $results); + } + + public function deleteKeys(array $keys) { + return parent::deleteKeys($this->prefixKeys($keys)); + } + + private function prefixKeys(array $keys) { + if ($this->namespace == null) { + throw new Exception(pht('Namespace not set.')); + } + + $prefixed_keys = array(); + foreach ($keys as $key) { + $prefixed_keys[] = $this->namespace.$key; + } + + return $prefixed_keys; + } + + private function unprefixKeys(array $keys) { + if ($this->namespace == null) { + throw new Exception(pht('Namespace not set.')); + } + + $unprefixed_keys = array(); + foreach ($keys as $key) { + $unprefixed_keys[] = substr($key, strlen($this->namespace)); + } + + return $unprefixed_keys; + } + +} diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php @@ -0,0 +1,108 @@ +name = $name; + return $this; + } + + public function getName() { + return $this->name; + } + + /** + * Set a profiler for cache operations. + * + * @param PhutilServiceProfiler Service profiler. + * @return this + * @task kvimpl + */ + public function setProfiler(PhutilServiceProfiler $profiler) { + $this->profiler = $profiler; + return $this; + } + + + /** + * Get the current profiler. + * + * @return PhutilServiceProfiler|null Profiler, or null if none is set. + * @task kvimpl + */ + public function getProfiler() { + return $this->profiler; + } + + + public function getKeys(array $keys) { + $call_id = null; + if ($this->getProfiler()) { + $call_id = $this->getProfiler()->beginServiceCall( + array( + 'type' => 'kvcache-get', + 'name' => $this->getName(), + 'keys' => $keys, + )); + } + + $results = parent::getKeys($keys); + + if ($call_id !== null) { + $this->getProfiler()->endServiceCall( + $call_id, + array( + 'hits' => array_keys($results), + )); + } + + return $results; + } + + + public function setKeys(array $keys, $ttl = null) { + $call_id = null; + if ($this->getProfiler()) { + $call_id = $this->getProfiler()->beginServiceCall( + array( + 'type' => 'kvcache-set', + 'name' => $this->getName(), + 'keys' => array_keys($keys), + 'ttl' => $ttl, + )); + } + + $result = parent::setKeys($keys, $ttl); + + if ($call_id !== null) { + $this->getProfiler()->endServiceCall($call_id, array()); + } + + return $result; + } + + + public function deleteKeys(array $keys) { + $call_id = null; + if ($this->getProfiler()) { + $call_id = $this->getProfiler()->beginServiceCall( + array( + 'type' => 'kvcache-del', + 'name' => $this->getName(), + 'keys' => $keys, + )); + } + + $result = parent::deleteKeys($keys); + + if ($call_id !== null) { + $this->getProfiler()->endServiceCall($call_id, array()); + } + + return $result; + } + +} diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProxy.php b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php @@ -0,0 +1,45 @@ +proxy = $proxy; + } + + final protected function getProxy() { + return $this->proxy; + } + + public function isAvailable() { + return $this->getProxy()->isAvailable(); + } + + + public function getKeys(array $keys) { + return $this->getProxy()->getKeys($keys); + } + + + public function setKeys(array $keys, $ttl = null) { + return $this->getProxy()->setKeys($keys, $ttl); + } + + + public function deleteKeys(array $keys) { + return $this->getProxy()->deleteKeys($keys); + } + + + public function destroyCache() { + return $this->getProxy()->destroyCache(); + } + + public function __call($method, array $arguments) { + return call_user_func_array( + array($this->getProxy(), $method), + $arguments); + } + +} diff --git a/src/infrastructure/cache/PhutilKeyValueCacheStack.php b/src/infrastructure/cache/PhutilKeyValueCacheStack.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilKeyValueCacheStack.php @@ -0,0 +1,131 @@ + Ordered list of key-value caches. + * @return this + * @task config + */ + public function setCaches(array $caches) { + assert_instances_of($caches, 'PhutilKeyValueCache'); + $this->cachesForward = $caches; + $this->cachesBackward = array_reverse($caches); + + return $this; + } + + + /** + * Set the readthrough TTL for the next cache operation. The TTL applies to + * any keys set by the next call to @{method:getKey} or @{method:getKeys}, + * and is reset after the call finishes. + * + * // If this causes any caches to fill, they'll fill with a 15-second TTL. + * $stack->setNextTTL(15)->getKey('porcupine'); + * + * // TTL does not persist; this will use no TTL. + * $stack->getKey('hedgehog'); + * + * @param int TTL in seconds. + * @return this + * + * @task config + */ + public function setNextTTL($ttl) { + $this->nextTTL = $ttl; + return $this; + } + + +/* -( Key-Value Cache Implementation )------------------------------------- */ + + + public function getKeys(array $keys) { + + $remaining = array_fuse($keys); + $results = array(); + $missed = array(); + + try { + foreach ($this->cachesForward as $cache) { + $result = $cache->getKeys($remaining); + $remaining = array_diff_key($remaining, $result); + $results += $result; + if (!$remaining) { + while ($cache = array_pop($missed)) { + // TODO: This sets too many results in the closer caches, although + // it probably isn't a big deal in most cases; normally we're just + // filling the request cache. + $cache->setKeys($result, $this->nextTTL); + } + break; + } + $missed[] = $cache; + } + $this->nextTTL = null; + } catch (Exception $ex) { + $this->nextTTL = null; + throw $ex; + } + + return $results; + } + + + public function setKeys(array $keys, $ttl = null) { + foreach ($this->cachesBackward as $cache) { + $cache->setKeys($keys, $ttl); + } + } + + + public function deleteKeys(array $keys) { + foreach ($this->cachesBackward as $cache) { + $cache->deleteKeys($keys); + } + } + + + public function destroyCache() { + foreach ($this->cachesBackward as $cache) { + $cache->destroyCache(); + } + } + +} diff --git a/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php @@ -0,0 +1,153 @@ +bucketKeys($keys); + $results = array(); + + foreach ($buckets as $bucket => $bucket_keys) { + $conn = $this->getConnection($bucket); + $result = $conn->get($bucket_keys); + if (!$result) { + // If the call fails, treat it as a miss on all keys. + $result = array(); + } + + $results += $result; + } + + return $results; + } + + public function setKeys(array $keys, $ttl = null) { + $buckets = $this->bucketKeys(array_keys($keys)); + + // Memcache interprets TTLs as: + // + // - Seconds from now, for values from 1 to 2592000 (30 days). + // - Epoch timestamp, for values larger than 2592000. + // + // We support only relative TTLs, so convert excessively large relative + // TTLs into epoch TTLs. + if ($ttl > 2592000) { + $effective_ttl = time() + $ttl; + } else { + $effective_ttl = $ttl; + } + + foreach ($buckets as $bucket => $bucket_keys) { + $conn = $this->getConnection($bucket); + + foreach ($bucket_keys as $key) { + $conn->set($key, $keys[$key], 0, $effective_ttl); + } + } + + return $this; + } + + public function deleteKeys(array $keys) { + $buckets = $this->bucketKeys($keys); + + foreach ($buckets as $bucket => $bucket_keys) { + $conn = $this->getConnection($bucket); + foreach ($bucket_keys as $key) { + $conn->delete($key); + } + } + + return $this; + } + + public function destroyCache() { + foreach ($this->servers as $key => $spec) { + $this->getConnection($key)->flush(); + } + return $this; + } + + +/* -( Managing Memcache )-------------------------------------------------- */ + + + /** + * Set available memcache servers. For example: + * + * $cache->setServers( + * array( + * array( + * 'host' => '10.0.0.20', + * 'port' => 11211, + * ), + * array( + * 'host' => '10.0.0.21', + * 'port' => 11211, + * ), + * )); + * + * @param list List of server specifications. + * @return this + * @task memcache + */ + public function setServers(array $servers) { + $this->servers = array_values($servers); + return $this; + } + + private function bucketKeys(array $keys) { + $buckets = array(); + $n = count($this->servers); + + if (!$n) { + throw new PhutilInvalidStateException('setServers'); + } + + foreach ($keys as $key) { + $bucket = (int)((crc32($key) & 0x7FFFFFFF) % $n); + $buckets[$bucket][] = $key; + } + + return $buckets; + } + + + /** + * @phutil-external-symbol function memcache_pconnect + */ + private function getConnection($server) { + if (empty($this->connections[$server])) { + $spec = $this->servers[$server]; + $host = $spec['host']; + $port = $spec['port']; + + $conn = memcache_pconnect($host, $spec['port']); + + if (!$conn) { + throw new Exception( + pht( + 'Unable to connect to memcache server (%s:%d)!', + $host, + $port)); + } + + $this->connections[$server] = $conn; + } + return $this->connections[$server]; + } + +} diff --git a/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php @@ -0,0 +1,205 @@ +wait = $wait; + return $this; + } + + public function getKeys(array $keys) { + $now = time(); + + $results = array(); + $reloaded = false; + foreach ($keys as $key) { + + // Try to read the value from cache. If we miss, load (or reload) the + // cache. + + while (true) { + if (isset($this->cache[$key])) { + $val = $this->cache[$key]; + if (empty($val['ttl']) || $val['ttl'] >= $now) { + $results[$key] = $val['val']; + break; + } + } + + if ($reloaded) { + break; + } + + $this->loadCache($hold_lock = false); + $reloaded = true; + } + } + + return $results; + } + + + public function setKeys(array $keys, $ttl = null) { + if ($ttl) { + $ttl_epoch = time() + $ttl; + } else { + $ttl_epoch = null; + } + + $dicts = array(); + foreach ($keys as $key => $value) { + $dict = array( + 'val' => $value, + ); + if ($ttl_epoch) { + $dict['ttl'] = $ttl_epoch; + } + $dicts[$key] = $dict; + } + + $this->loadCache($hold_lock = true); + foreach ($dicts as $key => $dict) { + $this->cache[$key] = $dict; + } + $this->saveCache(); + + return $this; + } + + + public function deleteKeys(array $keys) { + $this->loadCache($hold_lock = true); + foreach ($keys as $key) { + unset($this->cache[$key]); + } + $this->saveCache(); + + return $this; + } + + + public function destroyCache() { + Filesystem::remove($this->getCacheFile()); + return $this; + } + + +/* -( Cache Storage )------------------------------------------------------ */ + + + /** + * @task storage + */ + public function setCacheFile($file) { + $this->cacheFile = $file; + return $this; + } + + + /** + * @task storage + */ + private function loadCache($hold_lock) { + if ($this->lock) { + throw new Exception( + pht( + 'Trying to %s with a lock!', + __FUNCTION__.'()')); + } + + $lock = PhutilFileLock::newForPath($this->getCacheFile().'.lock'); + try { + $lock->lock($this->wait); + } catch (PhutilLockException $ex) { + if ($hold_lock) { + throw $ex; + } else { + $this->cache = array(); + return; + } + } + + try { + $this->cache = array(); + if (Filesystem::pathExists($this->getCacheFile())) { + $cache = unserialize(Filesystem::readFile($this->getCacheFile())); + if ($cache) { + $this->cache = $cache; + } + } + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } + + if ($hold_lock) { + $this->lock = $lock; + } else { + $lock->unlock(); + } + } + + + /** + * @task storage + */ + private function saveCache() { + if (!$this->lock) { + throw new PhutilInvalidStateException('loadCache'); + } + + // We're holding a lock so we're safe to do a write to a well-known file. + // Write to the same directory as the cache so the rename won't imply a + // copy across volumes. + $new = $this->getCacheFile().'.new'; + Filesystem::writeFile($new, serialize($this->cache)); + Filesystem::rename($new, $this->getCacheFile()); + + $this->lock->unlock(); + $this->lock = null; + } + + + /** + * @task storage + */ + private function getCacheFile() { + if (!$this->cacheFile) { + throw new PhutilInvalidStateException('setCacheFile'); + } + return $this->cacheFile; + } + +} diff --git a/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php @@ -0,0 +1,267 @@ +doCacheTest($cache); + $cache->destroyCache(); + } + + public function testInRequestCacheLimit() { + $cache = new PhutilInRequestKeyValueCache(); + $cache->setLimit(4); + + $cache->setKey(1, 1); + $cache->setKey(2, 2); + $cache->setKey(3, 3); + $cache->setKey(4, 4); + + $this->assertEqual( + array( + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + ), + $cache->getAllKeys()); + + + $cache->setKey(5, 5); + + $this->assertEqual( + array( + 2 => 2, + 3 => 3, + 4 => 4, + 5 => 5, + ), + $cache->getAllKeys()); + } + + public function testOnDiskCache() { + $cache = new PhutilOnDiskKeyValueCache(); + $cache->setCacheFile(new TempFile()); + $this->doCacheTest($cache); + $cache->destroyCache(); + } + + public function testAPCCache() { + $cache = new PhutilAPCKeyValueCache(); + if (!$cache->isAvailable()) { + $this->assertSkipped(pht('Cache not available.')); + } + $this->doCacheTest($cache); + } + + public function testDirectoryCache() { + $cache = new PhutilDirectoryKeyValueCache(); + + $dir = Filesystem::createTemporaryDirectory(); + $cache->setCacheDirectory($dir); + $this->doCacheTest($cache); + $cache->destroyCache(); + } + + public function testDirectoryCacheSpecialDirectoryRules() { + $cache = new PhutilDirectoryKeyValueCache(); + + $dir = Filesystem::createTemporaryDirectory(); + $dir = $dir.'/dircache/'; + $cache->setCacheDirectory($dir); + + $cache->setKey('a', 1); + $this->assertEqual(true, Filesystem::pathExists($dir.'/a.cache')); + + $cache->setKey('a/b', 1); + $this->assertEqual(true, Filesystem::pathExists($dir.'/a/')); + $this->assertEqual(true, Filesystem::pathExists($dir.'/a/b.cache')); + + $cache->deleteKey('a/b'); + $this->assertEqual(false, Filesystem::pathExists($dir.'/a/')); + $this->assertEqual(false, Filesystem::pathExists($dir.'/a/b.cache')); + + $cache->destroyCache(); + $this->assertEqual(false, Filesystem::pathExists($dir)); + } + + public function testNamespaceCache() { + $namespace = 'namespace'.mt_rand(); + $in_request_cache = new PhutilInRequestKeyValueCache(); + $cache = new PhutilKeyValueCacheNamespace($in_request_cache); + $cache->setNamespace($namespace); + + $test_info = get_class($cache); + $keys = array( + 'key1' => mt_rand(), + 'key2' => '', + 'key3' => 'Phabricator', + ); + $cache->setKeys($keys); + $cached_keys = $in_request_cache->getAllKeys(); + + foreach ($keys as $key => $value) { + $cached_key = $namespace.':'.$key; + + $this->assertTrue( + isset($cached_keys[$cached_key]), + $test_info); + + $this->assertEqual( + $value, + $cached_keys[$cached_key], + $test_info); + } + + $cache->destroyCache(); + + $this->doCacheTest($cache); + $cache->destroyCache(); + } + + public function testCacheStack() { + $req_cache = new PhutilInRequestKeyValueCache(); + $disk_cache = new PhutilOnDiskKeyValueCache(); + $disk_cache->setCacheFile(new TempFile()); + $apc_cache = new PhutilAPCKeyValueCache(); + + $stack = array( + $req_cache, + $disk_cache, + ); + + if ($apc_cache->isAvailable()) { + $stack[] = $apc_cache; + } + + $cache = new PhutilKeyValueCacheStack(); + $cache->setCaches($stack); + + $this->doCacheTest($cache); + + $disk_cache->destroyCache(); + $req_cache->destroyCache(); + } + + private function doCacheTest(PhutilKeyValueCache $cache) { + $key1 = 'test:'.mt_rand(); + $key2 = 'test:'.mt_rand(); + + $default = 'cache-miss'; + $value1 = 'cache-hit1'; + $value2 = 'cache-hit2'; + + $test_info = get_class($cache); + + // Test that we miss correctly on missing values. + + $this->assertEqual( + $default, + $cache->getKey($key1, $default), + $test_info); + $this->assertEqual( + array( + ), + $cache->getKeys(array($key1, $key2)), + $test_info); + + + // Test that we can set individual keys. + + $cache->setKey($key1, $value1); + $this->assertEqual( + $value1, + $cache->getKey($key1, $default), + $test_info); + $this->assertEqual( + array( + $key1 => $value1, + ), + $cache->getKeys(array($key1, $key2)), + $test_info); + + + // Test that we can delete individual keys. + + $cache->deleteKey($key1); + + $this->assertEqual( + $default, + $cache->getKey($key1, $default), + $test_info); + $this->assertEqual( + array( + ), + $cache->getKeys(array($key1, $key2)), + $test_info); + + + + // Test that we can set multiple keys. + + $cache->setKeys( + array( + $key1 => $value1, + $key2 => $value2, + )); + + $this->assertEqual( + $value1, + $cache->getKey($key1, $default), + $test_info); + $this->assertEqual( + array( + $key1 => $value1, + $key2 => $value2, + ), + $cache->getKeys(array($key1, $key2)), + $test_info); + + + // Test that we can delete multiple keys. + + $cache->deleteKeys(array($key1, $key2)); + + $this->assertEqual( + $default, + $cache->getKey($key1, $default), + $test_info); + $this->assertEqual( + array( + ), + $cache->getKeys(array($key1, $key2)), + $test_info); + + + // NOTE: The TTL tests are necessarily slow (we must sleep() through the + // TTLs) and do not work with APC (it does not TTL until the next request) + // so they're disabled by default. If you're developing the cache stack, + // it may be useful to run them. + + return; + + // Test that keys expire when they TTL. + + $cache->setKey($key1, $value1, 1); + $cache->setKey($key2, $value2, 5); + + $this->assertEqual($value1, $cache->getKey($key1, $default)); + $this->assertEqual($value2, $cache->getKey($key2, $default)); + + sleep(2); + + $this->assertEqual($default, $cache->getKey($key1, $default)); + $this->assertEqual($value2, $cache->getKey($key2, $default)); + + + // Test that setting a 0 TTL overwrites a nonzero TTL. + + $cache->setKey($key1, $value1, 1); + $this->assertEqual($value1, $cache->getKey($key1, $default)); + $cache->setKey($key1, $value1, 0); + $this->assertEqual($value1, $cache->getKey($key1, $default)); + sleep(2); + $this->assertEqual($value1, $cache->getKey($key1, $default)); + } + +} diff --git a/src/infrastructure/lipsum/PhutilContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php @@ -0,0 +1,93 @@ +generate(); + } + return implode($implode, $paragraph); + } + + public function generate() { + $count = 0; + $rules = $this->getRules(); + return $this->applyRules('[start]', $count, $rules); + } + + final protected function applyRules($input, &$count, array $rules) { + if (++$count > $this->limit) { + throw new Exception(pht('Token replacement count exceeded limit!')); + } + + $matches = null; + preg_match_all('/(\\[[^\\]]+\\])/', $input, $matches, PREG_OFFSET_CAPTURE); + + foreach (array_reverse($matches[1]) as $token_spec) { + list($token, $offset) = $token_spec; + $token_name = substr($token, 1, -1); + $options = array(); + + if (($name_end = strpos($token_name, ','))) { + $options_parser = new PhutilSimpleOptions(); + $options = $options_parser->parse($token_name); + $token_name = substr($token_name, 0, $name_end); + } + + if (empty($rules[$token_name])) { + throw new Exception(pht("Invalid token '%s' in grammar.", $token_name)); + } + + $key = array_rand($rules[$token_name]); + $replacement = $this->applyRules($rules[$token_name][$key], + $count, $rules); + + if (isset($options['indent'])) { + if (is_numeric($options['indent'])) { + $replacement = self::strPadLines($replacement, $options['indent']); + } else { + $replacement = self::strPadLines($replacement); + } + } + if (isset($options['trim'])) { + switch ($options['trim']) { + case 'left': + $replacement = ltrim($replacement); + break; + case 'right': + $replacement = rtrim($replacement); + break; + default: + case 'both': + $replacement = trim($replacement); + break; + } + } + if (isset($options['block'])) { + $replacement = "\n".$replacement."\n"; + } + + $input = substr_replace($input, $replacement, $offset, strlen($token)); + } + + return $input; + } + + private static function strPadLines($text, $num_spaces = 2) { + $text_lines = phutil_split_lines($text); + foreach ($text_lines as $linenr => $line) { + $text_lines[$linenr] = str_repeat(' ', $num_spaces).$line; + } + + return implode('', $text_lines); + } + +} diff --git a/src/infrastructure/markup/PhutilMarkupEngine.php b/src/infrastructure/markup/PhutilMarkupEngine.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/PhutilMarkupEngine.php @@ -0,0 +1,32 @@ +content = (string)$content; + } + + public function __toString() { + return $this->content; + } + + public function getHTMLContent() { + return $this->content; + } + + public function appendHTML($html /* , ... */) { + foreach (func_get_args() as $html) { + $this->content .= phutil_escape_html($html); + } + return $this; + } + + public static function applyFunction($function, $string /* , ... */) { + $args = func_get_args(); + array_shift($args); + $args = array_map('phutil_escape_html', $args); + return new PhutilSafeHTML(call_user_func_array($function, $args)); + } + +// Requires http://pecl.php.net/operator. + + public function __concat($html) { + $clone = clone $this; + return $clone->appendHTML($html); + } + + public function __assign_concat($html) { + return $this->appendHTML($html); + } + +} diff --git a/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php @@ -0,0 +1,12 @@ +assertEqual( + (string)phutil_tag('br'), + (string)phutil_tag('br', array())); + + $this->assertEqual( + (string)phutil_tag('br', array()), + (string)phutil_tag('br', array(), null)); + } + + public function testTagEmpty() { + $this->assertEqual( + '
', + (string)phutil_tag('br', array(), null)); + + $this->assertEqual( + '
', + (string)phutil_tag('div', array(), null)); + + $this->assertEqual( + '
', + (string)phutil_tag('div', array(), '')); + } + + public function testTagBasics() { + $this->assertEqual( + '
', + (string)phutil_tag('br')); + + $this->assertEqual( + '
y
', + (string)phutil_tag('div', array(), 'y')); + } + + public function testTagAttributes() { + $this->assertEqual( + '
y
', + (string)phutil_tag('div', array('u' => 'v'), 'y')); + + $this->assertEqual( + '
', + (string)phutil_tag('br', array('u' => 'v'))); + } + + public function testTagEscapes() { + $this->assertEqual( + '
', + (string)phutil_tag('br', array('u' => '<'))); + + $this->assertEqual( + '

', + (string)phutil_tag('div', array(), phutil_tag('br'))); + } + + public function testTagNullAttribute() { + $this->assertEqual( + '
', + (string)phutil_tag('br', array('y' => null))); + } + + public function testTagJavascriptProtocolRejection() { + $hrefs = array( + 'javascript:alert(1)' => true, + 'JAVASCRIPT:alert(2)' => true, + + // NOTE: When interpreted as a URI, this is dropped because of leading + // whitespace. + ' javascript:alert(3)' => array(true, false), + '/' => false, + '/path/to/stuff/' => false, + '' => false, + 'http://example.com/' => false, + '#' => false, + 'javascript://anything' => true, + + // Chrome 33 and IE11, at a minimum, treat this as Javascript. + "javascript\n:alert(4)" => true, + + // Opera currently accepts a variety of unicode spaces. This test case + // has a smattering of them. + "\xE2\x80\x89javascript:" => true, + "javascript\xE2\x80\x89:" => true, + "\xE2\x80\x84javascript:" => true, + "javascript\xE2\x80\x84:" => true, + + // Because we're aggressive, all of unicode should trigger detection + // by default. + "\xE2\x98\x83javascript:" => true, + "javascript\xE2\x98\x83:" => true, + "\xE2\x98\x83javascript\xE2\x98\x83:" => true, + + // We're aggressive about this, so we'll intentionally raise false + // positives in these cases. + 'javascript~:alert(5)' => true, + '!!!javascript!!!!:alert(6)' => true, + + // However, we should raise true negatives in these slightly more + // reasonable cases. + 'javascript/:docs.html' => false, + 'javascripts:x.png' => false, + 'COOLjavascript:page' => false, + '/javascript:alert(1)' => false, + ); + + foreach (array(true, false) as $use_uri) { + foreach ($hrefs as $href => $expect) { + if (is_array($expect)) { + $expect = ($use_uri ? $expect[1] : $expect[0]); + } + + if ($use_uri) { + $href_value = new PhutilURI($href); + } else { + $href_value = $href; + } + + $caught = null; + try { + phutil_tag('a', array('href' => $href_value), 'click for candy'); + } catch (Exception $ex) { + $caught = $ex; + } + + $desc = pht( + 'Unexpected result for "%s". ', + $href, + $use_uri ? pht('Yes') : pht('No'), + $expect ? pht('Yes') : pht('No')); + + $this->assertEqual( + $expect, + $caught instanceof Exception, + $desc); + } + } + } + + public function testURIEscape() { + $this->assertEqual( + '%2B/%20%3F%23%26%3A%21xyz%25', + phutil_escape_uri('+/ ?#&:!xyz%')); + } + + public function testURIPathComponentEscape() { + $this->assertEqual( + 'a%252Fb', + phutil_escape_uri_path_component('a/b')); + + $str = ''; + for ($ii = 0; $ii <= 255; $ii++) { + $str .= chr($ii); + } + + $this->assertEqual( + $str, + phutil_unescape_uri_path_component( + rawurldecode( // Simulates webserver. + phutil_escape_uri_path_component($str)))); + } + + public function testHsprintf() { + $this->assertEqual( + '
<3
', + (string)hsprintf('
%s
', '<3')); + } + + public function testAppendHTML() { + $html = phutil_tag('hr'); + $html->appendHTML(phutil_tag('br'), ''); + $this->assertEqual('

<evil>', $html->getHTMLContent()); + } + + public function testArrayEscaping() { + $this->assertEqual( + '
<div>
', + phutil_escape_html( + array( + hsprintf('
'), + array( + array( + '<', + array( + 'd', + array( + array( + hsprintf('i'), + ), + 'v', + ), + ), + array( + array( + '>', + ), + ), + ), + ), + hsprintf('
'), + ))); + + $this->assertEqual( + '

', + phutil_tag( + 'div', + array(), + array( + array( + array( + phutil_tag('br'), + array( + phutil_tag('hr'), + ), + phutil_tag('wbr'), + ), + ), + ))->getHTMLContent()); + } + +} diff --git a/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php @@ -0,0 +1,19 @@ +assertSkipped(pht('Operator extension not available.')); + } + + $a = phutil_tag('a'); + $ab = $a.phutil_tag('b'); + $this->assertEqual('', $ab->getHTMLContent()); + $this->assertEqual('', $a->getHTMLContent()); + + $a .= phutil_tag('a'); + $this->assertEqual('', $a->getHTMLContent()); + } + +} diff --git a/src/infrastructure/markup/render.php b/src/infrastructure/markup/render.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/render.php @@ -0,0 +1,251 @@ +` tags, if the `rel` attribute is not specified, it + * is interpreted as `rel="noreferrer"`. + * - When rendering `` tags, the `href` attribute may not begin with + * `javascript:`. + * + * These special cases can not be disabled. + * + * IMPORTANT: The `$tag` attribute and the keys of the `$attributes` array are + * trusted blindly, and not escaped. You should not pass user data in these + * parameters. + * + * @param string The name of the tag, like `a` or `div`. + * @param map A map of tag attributes. + * @param wild Content to put in the tag. + * @return PhutilSafeHTML Tag object. + */ +function phutil_tag($tag, array $attributes = array(), $content = null) { + // If the `href` attribute is present, make sure it is not a "javascript:" + // URI. We never permit these. + if (!empty($attributes['href'])) { + // This might be a URI object, so cast it to a string. + $href = (string)$attributes['href']; + + if (isset($href[0])) { + // Block 'javascript:' hrefs at the tag level: no well-designed + // application should ever use them, and they are a potent attack vector. + + // This function is deep in the core and performance sensitive, so we're + // doing a cheap version of this test first to avoid calling preg_match() + // on URIs which begin with '/' or `#`. These cover essentially all URIs + // in Phabricator. + if (($href[0] !== '/') && ($href[0] !== '#')) { + // Chrome 33 and IE 11 both interpret "javascript\n:" as a Javascript + // URI, and all browsers interpret " javascript:" as a Javascript URI, + // so be aggressive about looking for "javascript:" in the initial + // section of the string. + + $normalized_href = preg_replace('([^a-z0-9/:]+)i', '', $href); + if (preg_match('/^javascript:/i', $normalized_href)) { + throw new Exception( + pht( + "Attempting to render a tag with an '%s' attribute that begins ". + "with '%s'. This is either a serious security concern or a ". + "serious architecture concern. Seek urgent remedy.", + 'href', + 'javascript:')); + } + } + } + } + + // For tags which can't self-close, treat null as the empty string -- for + // example, always render `
`, never `
`. + static $self_closing_tags = array( + 'area' => true, + 'base' => true, + 'br' => true, + 'col' => true, + 'command' => true, + 'embed' => true, + 'frame' => true, + 'hr' => true, + 'img' => true, + 'input' => true, + 'keygen' => true, + 'link' => true, + 'meta' => true, + 'param' => true, + 'source' => true, + 'track' => true, + 'wbr' => true, + ); + + $attr_string = ''; + foreach ($attributes as $k => $v) { + if ($v === null) { + continue; + } + $v = phutil_escape_html($v); + $attr_string .= ' '.$k.'="'.$v.'"'; + } + + if ($content === null) { + if (isset($self_closing_tags[$tag])) { + return new PhutilSafeHTML('<'.$tag.$attr_string.' />'); + } else { + $content = ''; + } + } else { + $content = phutil_escape_html($content); + } + + return new PhutilSafeHTML('<'.$tag.$attr_string.'>'.$content.''); +} + +function phutil_tag_div($class, $content = null) { + return phutil_tag('div', array('class' => $class), $content); +} + +function phutil_escape_html($string) { + if ($string instanceof PhutilSafeHTML) { + return $string; + } else if ($string instanceof PhutilSafeHTMLProducerInterface) { + $result = $string->producePhutilSafeHTML(); + if ($result instanceof PhutilSafeHTML) { + return phutil_escape_html($result); + } else if (is_array($result)) { + return phutil_escape_html($result); + } else if ($result instanceof PhutilSafeHTMLProducerInterface) { + return phutil_escape_html($result); + } else { + try { + assert_stringlike($result); + return phutil_escape_html((string)$result); + } catch (Exception $ex) { + throw new Exception( + pht( + "Object (of class '%s') implements %s but did not return anything ". + "renderable from %s.", + get_class($string), + 'PhutilSafeHTMLProducerInterface', + 'producePhutilSafeHTML()')); + } + } + } else if (is_array($string)) { + $result = ''; + foreach ($string as $item) { + $result .= phutil_escape_html($item); + } + return $result; + } + + return htmlspecialchars($string, ENT_QUOTES, 'UTF-8'); +} + +function phutil_escape_html_newlines($string) { + return PhutilSafeHTML::applyFunction('nl2br', $string); +} + +/** + * Mark string as safe for use in HTML. + */ +function phutil_safe_html($string) { + if ($string == '') { + return $string; + } else if ($string instanceof PhutilSafeHTML) { + return $string; + } else { + return new PhutilSafeHTML($string); + } +} + +/** + * HTML safe version of `implode()`. + */ +function phutil_implode_html($glue, array $pieces) { + $glue = phutil_escape_html($glue); + + foreach ($pieces as $k => $piece) { + $pieces[$k] = phutil_escape_html($piece); + } + + return phutil_safe_html(implode($glue, $pieces)); +} + +/** + * Format a HTML code. This function behaves like `sprintf()`, except that all + * the normal conversions (like %s) will be properly escaped. + */ +function hsprintf($html /* , ... */) { + $args = func_get_args(); + array_shift($args); + return new PhutilSafeHTML( + vsprintf($html, array_map('phutil_escape_html', $args))); +} + + +/** + * Escape text for inclusion in a URI or a query parameter. Note that this + * method does NOT escape '/', because "%2F" is invalid in paths and Apache + * will automatically 404 the page if it's present. This will produce correct + * (the URIs will work) and desirable (the URIs will be readable) behavior in + * these cases: + * + * '/path/?param='.phutil_escape_uri($string); # OK: Query Parameter + * '/path/to/'.phutil_escape_uri($string); # OK: URI Suffix + * + * It will potentially produce the WRONG behavior in this special case: + * + * COUNTEREXAMPLE + * '/path/to/'.phutil_escape_uri($string).'/thing/'; # BAD: URI Infix + * + * In this case, any '/' characters in the string will not be escaped, so you + * will not be able to distinguish between the string and the suffix (unless + * you have more information, like you know the format of the suffix). For infix + * URI components, use @{function:phutil_escape_uri_path_component} instead. + * + * @param string Some string. + * @return string URI encoded string, except for '/'. + */ +function phutil_escape_uri($string) { + return str_replace('%2F', '/', rawurlencode($string)); +} + + +/** + * Escape text for inclusion as an infix URI substring. See discussion at + * @{function:phutil_escape_uri}. This function covers an unusual special case; + * @{function:phutil_escape_uri} is usually the correct function to use. + * + * This function will escape a string into a format which is safe to put into + * a URI path and which does not contain '/' so it can be correctly parsed when + * embedded as a URI infix component. + * + * However, you MUST decode the string with + * @{function:phutil_unescape_uri_path_component} before it can be used in the + * application. + * + * @param string Some string. + * @return string URI encoded string that is safe for infix composition. + */ +function phutil_escape_uri_path_component($string) { + return rawurlencode(rawurlencode($string)); +} + + +/** + * Unescape text that was escaped by + * @{function:phutil_escape_uri_path_component}. See + * @{function:phutil_escape_uri} for discussion. + * + * Note that this function is NOT the inverse of + * @{function:phutil_escape_uri_path_component}! It undoes additional escaping + * which is added to survive the implied unescaping performed by the webserver + * when interpreting the request. + * + * @param string Some string emitted + * from @{function:phutil_escape_uri_path_component} and + * then accessed via a web server. + * @return string Original string. + */ +function phutil_unescape_uri_path_component($string) { + return rawurldecode($string); +} diff --git a/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php @@ -0,0 +1,115 @@ +config[$key] = $value; + return $this; + } + + public function getLanguageFromFilename($filename) { + static $default_map = array( + // All files which have file extensions that we haven't already matched + // map to their extensions. + '@\\.([^./]+)$@' => 1, + ); + + $maps = array(); + if (!empty($this->config['filename.map'])) { + $maps[] = $this->config['filename.map']; + } + $maps[] = $default_map; + + foreach ($maps as $map) { + foreach ($map as $regexp => $lang) { + $matches = null; + if (preg_match($regexp, $filename, $matches)) { + if (is_numeric($lang)) { + return idx($matches, $lang); + } else { + return $lang; + } + } + } + } + + return null; + } + + public function getHighlightFuture($language, $source) { + if ($language === null) { + $language = PhutilLanguageGuesser::guessLanguage($source); + } + + $have_pygments = !empty($this->config['pygments.enabled']); + + if ($language == 'php' && PhutilXHPASTBinary::isAvailable()) { + return id(new PhutilXHPASTSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + if ($language == 'console') { + return id(new PhutilConsoleSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + if ($language == 'diviner' || $language == 'remarkup') { + return id(new PhutilDivinerSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + if ($language == 'rainbow') { + return id(new PhutilRainbowSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + if ($language == 'php') { + return id(new PhutilLexerSyntaxHighlighter()) + ->setConfig('lexer', new PhutilPHPFragmentLexer()) + ->setConfig('language', 'php') + ->getHighlightFuture($source); + } + + if ($language == 'py' || $language == 'python') { + return id(new PhutilLexerSyntaxHighlighter()) + ->setConfig('lexer', new PhutilPythonFragmentLexer()) + ->setConfig('language', 'py') + ->getHighlightFuture($source); + } + + if ($language == 'java') { + return id(new PhutilLexerSyntaxHighlighter()) + ->setConfig('lexer', new PhutilJavaFragmentLexer()) + ->setConfig('language', 'java') + ->getHighlightFuture($source); + } + + if ($language == 'json') { + return id(new PhutilLexerSyntaxHighlighter()) + ->setConfig('lexer', new PhutilJSONFragmentLexer()) + ->getHighlightFuture($source); + } + + if ($language == 'invisible') { + return id(new PhutilInvisibleSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + // Don't invoke Pygments for plain text, since it's expensive and has + // no effect. + if ($language !== 'text' && $language !== 'txt') { + if ($have_pygments) { + return id(new PhutilPygmentsSyntaxHighlighter()) + ->setConfig('language', $language) + ->getHighlightFuture($source); + } + } + + return id(new PhutilDefaultSyntaxHighlighter()) + ->getHighlightFuture($source); + } + +} diff --git a/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php @@ -0,0 +1,19 @@ +getHighlightFuture($language, $source)->resolve(); + } catch (PhutilSyntaxHighlighterException $ex) { + return id(new PhutilDefaultSyntaxHighlighter()) + ->getHighlightFuture($source) + ->resolve(); + } + } + +} diff --git a/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php @@ -0,0 +1,28 @@ + 'php', + '/x.php' => 'php', + 'x.y.php' => 'php', + '/x.y/z.php' => 'php', + '/x.php/' => null, + ); + + $engine = new PhutilDefaultSyntaxHighlighterEngine(); + foreach ($names as $path => $language) { + $detect = $engine->getLanguageFromFilename($path); + $this->assertEqual( + $language, + $detect, + pht('Language detect for %s', $path)); + } + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php @@ -0,0 +1,51 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + $in_command = false; + $lines = explode("\n", $source); + foreach ($lines as $key => $line) { + $matches = null; + + // Parse commands like this: + // + // some/path/ $ ./bin/example # Do things + // + // ...into path, command, and comment components. + + $pattern = + '@'. + ($in_command ? '()(.*?)' : '^(\S+[\\\\/] )?([$] .*?)'). + '(#.*|\\\\)?$@'; + + if (preg_match($pattern, $line, $matches)) { + $lines[$key] = hsprintf( + '%s%s%s', + $matches[1], + $matches[2], + (!empty($matches[3]) + ? hsprintf('%s', $matches[3]) + : '')); + $in_command = (idx($matches, 3) == '\\'); + } else { + $lines[$key] = hsprintf('%s', $line); + } + } + $lines = phutil_implode_html("\n", $lines); + + return new ImmediateFuture($lines); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php @@ -0,0 +1,14 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + $source = phutil_escape_html($source); + + // This highlighter isn't perfect but tries to do an okay job at getting + // some of the basics at least. There's lots of room for improvement. + + $blocks = explode("\n\n", $source); + foreach ($blocks as $key => $block) { + if (preg_match('/^[^ ](?! )/m', $block)) { + $blocks[$key] = $this->highlightBlock($block); + } + } + $source = implode("\n\n", $blocks); + + $source = phutil_safe_html($source); + return new ImmediateFuture($source); + } + + private function highlightBlock($source) { + // Highlight "@{class:...}" links to other documentation pages. + $source = $this->highlightPattern('/@{([\w@]+?):([^}]+?)}/', $source, 'nc'); + + // Highlight "@title", "@group", etc. + $source = $this->highlightPattern('/^@(\w+)/m', $source, 'k'); + + // Highlight bold, italic and monospace. + $source = $this->highlightPattern('@\\*\\*(.+?)\\*\\*@s', $source, 's'); + $source = $this->highlightPattern('@(?highlightPattern( + '@##([\s\S]+?)##|\B`(.+?)`\B@', + $source, + 's'); + + // Highlight stuff that looks like headers. + $source = $this->highlightPattern('/^=(.*)$/m', $source, 'nv'); + + return $source; + } + + private function highlightPattern($regexp, $source, $class) { + $this->replaceClass = $class; + $source = preg_replace_callback( + $regexp, + array($this, 'replacePattern'), + $source); + + return $source; + } + + public function replacePattern($matches) { + + // NOTE: The goal here is to make sure a never crosses a newline. + + $content = $matches[0]; + $content = explode("\n", $content); + foreach ($content as $key => $line) { + $content[$key] = + ''. + $line. + ''; + } + return implode("\n", $content); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php @@ -0,0 +1,43 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + $keys = array_map('chr', range(0x0, 0x1F)); + $vals = array_map( + array($this, 'decimalToHtmlEntityDecoded'), range(0x2400, 0x241F)); + + $invisible = array_combine($keys, $vals); + + $result = array(); + foreach (str_split($source) as $character) { + if (isset($invisible[$character])) { + $result[] = phutil_tag( + 'span', + array('class' => 'invisible'), + $invisible[$character]); + + if ($character === "\n") { + $result[] = $character; + } + } else { + $result[] = $character; + } + } + + $result = phutil_implode_html('', $result); + return new ImmediateFuture($result); + } + + private function decimalToHtmlEntityDecoded($dec) { + return html_entity_decode("&#{$dec};"); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php @@ -0,0 +1,72 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + $strip = false; + $state = 'start'; + $lang = idx($this->config, 'language'); + + if ($lang == 'php') { + if (strpos($source, 'config, 'lexer'); + $tokens = $lexer->getTokens($source, $state); + $tokens = $lexer->mergeTokens($tokens); + + $result = array(); + foreach ($tokens as $token) { + list($type, $value, $context) = $token; + + $data_name = null; + switch ($type) { + case 'nc': + case 'nf': + case 'na': + $data_name = $value; + break; + } + + if (strpos($value, "\n") !== false) { + $value = explode("\n", $value); + } else { + $value = array($value); + } + foreach ($value as $part) { + if (strlen($part)) { + if ($type) { + $result[] = phutil_tag( + 'span', + array( + 'class' => $type, + 'data-symbol-context' => $context, + 'data-symbol-name' => $data_name, + ), + $part); + } else { + $result[] = $part; + } + } + $result[] = "\n"; + } + + // Throw away the last "\n". + array_pop($result); + } + + $result = phutil_implode_html('', $result); + + return new ImmediateFuture($result); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php @@ -0,0 +1,229 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + $language = idx($this->config, 'language'); + + if (preg_match('/\r(?!\n)/', $source)) { + // TODO: Pygments converts "\r" newlines into "\n" newlines, so we can't + // use it on files with "\r" newlines. If we have "\r" not followed by + // "\n" in the file, skip highlighting. + $language = null; + } + + if ($language) { + $language = $this->getPygmentsLexerNameFromLanguageName($language); + + // See T13224. Under Ubuntu, avoid leaving an intermedite "dash" shell + // process so we hit "pygmentize" directly if we have to SIGKILL this + // because it explodes. + + $future = new ExecFuture( + 'exec pygmentize -O encoding=utf-8 -O stripnl=False -f html -l %s', + $language); + + $scrub = false; + if ($language == 'php' && strpos($source, 'setTimeout(15); + + $future->write($source); + + return new PhutilDefaultSyntaxHighlighterEnginePygmentsFuture( + $future, + $source, + $scrub); + } + + return id(new PhutilDefaultSyntaxHighlighter()) + ->getHighlightFuture($source); + } + + private function getPygmentsLexerNameFromLanguageName($language) { + static $map = array( + 'adb' => 'ada', + 'ads' => 'ada', + 'ahkl' => 'ahk', + 'as' => 'as3', + 'asax' => 'aspx-vb', + 'ascx' => 'aspx-vb', + 'ashx' => 'aspx-vb', + 'ASM' => 'nasm', + 'asm' => 'nasm', + 'asmx' => 'aspx-vb', + 'aspx' => 'aspx-vb', + 'autodelegate' => 'myghty', + 'autohandler' => 'mason', + 'aux' => 'tex', + 'axd' => 'aspx-vb', + 'b' => 'brainfuck', + 'bas' => 'vb.net', + 'bf' => 'brainfuck', + 'bmx' => 'blitzmax', + 'c++' => 'cpp', + 'c++-objdump' => 'cpp-objdump', + 'cc' => 'cpp', + 'cfc' => 'cfm', + 'cfg' => 'ini', + 'cfml' => 'cfm', + 'cl' => 'common-lisp', + 'clj' => 'clojure', + 'cmd' => 'bat', + 'coffee' => 'coffee-script', + 'cs' => 'csharp', + 'csh' => 'tcsh', + 'cw' => 'redcode', + 'cxx' => 'cpp', + 'cxx-objdump' => 'cpp-objdump', + 'darcspatch' => 'dpatch', + 'def' => 'modula2', + 'dhandler' => 'mason', + 'di' => 'd', + 'duby' => 'rb', + 'dyl' => 'dylan', + 'ebuild' => 'bash', + 'eclass' => 'bash', + 'el' => 'common-lisp', + 'eps' => 'postscript', + 'erl' => 'erlang', + 'erl-sh' => 'erl', + 'f' => 'fortran', + 'f90' => 'fortran', + 'feature' => 'Cucumber', + 'fhtml' => 'velocity', + 'flx' => 'felix', + 'flxh' => 'felix', + 'frag' => 'glsl', + 'g' => 'antlr-ruby', + 'G' => 'antlr-ruby', + 'gdc' => 'gooddata-cl', + 'gemspec' => 'rb', + 'geo' => 'glsl', + 'GNUmakefile' => 'make', + 'h' => 'c', + 'h++' => 'cpp', + 'hh' => 'cpp', + 'hpp' => 'cpp', + 'hql' => 'sql', + 'hrl' => 'erlang', + 'hs' => 'haskell', + 'htaccess' => 'apacheconf', + 'htm' => 'html', + 'html' => 'html+evoque', + 'hxx' => 'cpp', + 'hy' => 'hybris', + 'hyb' => 'hybris', + 'ik' => 'ioke', + 'inc' => 'pov', + 'j' => 'objective-j', + 'jbst' => 'duel', + 'kid' => 'genshi', + 'ksh' => 'bash', + 'less' => 'css', + 'lgt' => 'logtalk', + 'lisp' => 'common-lisp', + 'll' => 'llvm', + 'm' => 'objective-c', + 'mak' => 'make', + 'Makefile' => 'make', + 'makefile' => 'make', + 'man' => 'groff', + 'mao' => 'mako', + 'mc' => 'mason', + 'md' => 'minid', + 'mhtml' => 'mason', + 'mi' => 'mason', + 'ml' => 'ocaml', + 'mli' => 'ocaml', + 'mll' => 'ocaml', + 'mly' => 'ocaml', + 'mm' => 'objective-c', + 'mo' => 'modelica', + 'mod' => 'modula2', + 'moo' => 'moocode', + 'mu' => 'mupad', + 'myt' => 'myghty', + 'ns2' => 'newspeak', + 'pas' => 'delphi', + 'patch' => 'diff', + 'phtml' => 'html+php', + 'pl' => 'prolog', + 'plot' => 'gnuplot', + 'plt' => 'gnuplot', + 'pm' => 'perl', + 'po' => 'pot', + 'pp' => 'puppet', + 'pro' => 'prolog', + 'proto' => 'protobuf', + 'ps' => 'postscript', + 'pxd' => 'cython', + 'pxi' => 'cython', + 'py' => 'python', + 'pyw' => 'python', + 'pyx' => 'cython', + 'R' => 'splus', + 'r' => 'rebol', + 'r3' => 'rebol', + 'rake' => 'rb', + 'Rakefile' => 'rb', + 'rbw' => 'rb', + 'rbx' => 'rb', + 'rest' => 'rst', + 'rl' => 'ragel-em', + 'robot' => 'robotframework', + 'Rout' => 'rconsole', + 'rss' => 'xml', + 's' => 'gas', + 'S' => 'splus', + 'sc' => 'python', + 'scm' => 'scheme', + 'SConscript' => 'python', + 'SConstruct' => 'python', + 'scss' => 'css', + 'sh' => 'bash', + 'sh-session' => 'console', + 'spt' => 'cheetah', + 'sqlite3-console' => 'sqlite3', + 'st' => 'smalltalk', + 'sv' => 'v', + 'tac' => 'python', + 'tmpl' => 'cheetah', + 'toc' => 'tex', + 'tpl' => 'smarty', + 'txt' => 'text', + 'vapi' => 'vala', + 'vb' => 'vb.net', + 'vert' => 'glsl', + 'vhd' => 'vhdl', + 'vimrc' => 'vim', + 'vm' => 'velocity', + 'weechatlog' => 'irc', + 'wlua' => 'lua', + 'wsdl' => 'xml', + 'xhtml' => 'html', + 'xml' => 'xml+evoque', + 'xqy' => 'xquery', + 'xsd' => 'xml', + 'xsl' => 'xslt', + 'xslt' => 'xml', + 'yml' => 'yaml', + ); + + return idx($map, $language, $language); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php @@ -0,0 +1,46 @@ +config[$key] = $value; + return $this; + } + + public function getHighlightFuture($source) { + + $color = 0; + $colors = array( + 'rbw_r', + 'rbw_o', + 'rbw_y', + 'rbw_g', + 'rbw_b', + 'rbw_i', + 'rbw_v', + ); + + $result = array(); + foreach (phutil_utf8v($source) as $character) { + if ($character == ' ' || $character == "\n") { + $result[] = $character; + continue; + } + $result[] = phutil_tag( + 'span', + array('class' => $colors[$color]), + $character); + $color = ($color + 1) % count($colors); + } + + $result = phutil_implode_html('', $result); + return new ImmediateFuture($result); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php @@ -0,0 +1,6 @@ +setConfig('language', 'json') + ->setConfig('lexer', new PhutilJSONFragmentLexer()); + + $path = dirname(__FILE__).'/data/jsonfragment/'; + foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) { + if (preg_match('/.test$/', $f)) { + $expect = preg_replace('/.test$/', '.expect', $f); + $source = Filesystem::readFile($path.'/'.$f); + + $this->assertEqual( + Filesystem::readFile($path.'/'.$expect), + (string)$highlighter->getHighlightFuture($source)->resolve(), + $f); + } + } + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php @@ -0,0 +1,25 @@ +setConfig('language', 'php'); + $highlighter->setConfig('lexer', new PhutilPHPFragmentLexer()); + + + $path = dirname(__FILE__).'/phpfragment/'; + foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) { + if (preg_match('/.test$/', $f)) { + $expect = preg_replace('/.test$/', '.expect', $f); + $source = Filesystem::readFile($path.'/'.$f); + + $this->assertEqual( + Filesystem::readFile($path.'/'.$expect), + (string)$highlighter->getHighlightFuture($source)->resolve(), + $f); + } + } + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php @@ -0,0 +1,39 @@ +getHighlightFuture($source); + return $future->resolve(); + } + + private function read($file) { + $path = dirname(__FILE__).'/xhpast/'.$file; + return Filesystem::readFile($path); + } + + public function testBuiltinClassnames() { + $this->assertEqual( + $this->read('builtin-classname.expect'), + (string)$this->highlight($this->read('builtin-classname.source')), + pht('Builtin classnames should not be marked as linkable symbols.')); + $this->assertEqual( + rtrim($this->read('trailing-comment.expect')), + (string)$this->highlight($this->read('trailing-comment.source')), + pht('Trailing comments should not be dropped.')); + $this->assertEqual( + $this->read('multiline-token.expect'), + (string)$this->highlight($this->read('multiline-token.source')), + pht('Multi-line tokens should be split across lines.')); + $this->assertEqual( + $this->read('leading-whitespace.expect'), + (string)$this->highlight($this->read('leading-whitespace.source')), + pht('Snippets with leading whitespace should be preserved.')); + $this->assertEqual( + $this->read('no-leading-whitespace.expect'), + (string)$this->highlight($this->read('no-leading-whitespace.source')), + pht('Snippets with no leading whitespace should be preserved.')); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect @@ -0,0 +1,12 @@ +{ + "key": 3.5, + "true": true, + "false": false, + "null": null, + "list": [1, 2, 3], + "object": { + "k1": "v1" + }, + "numbers": [0e1, 1e-1, -1e-1, -1e+1], + "\"\u1234'abc[]{}..." +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test @@ -0,0 +1,12 @@ +{ + "key": 3.5, + "true": true, + "false": false, + "null": null, + "list": [1, 2, 3], + "object": { + "k1": "v1" + }, + "numbers": [0e1, 1e-1, -1e-1, -1e+1], + "\"\u1234'abc[]{}..." +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect @@ -0,0 +1,16 @@ +<? + +// comment? comment! ?> + +data + +<?php + +__halt_compiler /* ! */ ( // ) +) /* ;;;; */ + +; + +data data +<?php +data diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test @@ -0,0 +1,16 @@ + + +data + +public function f() { + ExampleClass::EXAMPLE_CONSTANT; + ExampleClass::exampleMethod(); + example_function(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test @@ -0,0 +1,5 @@ +public function f() { + ExampleClass::EXAMPLE_CONSTANT; + ExampleClass::exampleMethod(); + example_function(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect @@ -0,0 +1,3 @@ + foreach ($x as $y) { + z(); + } diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test @@ -0,0 +1,3 @@ + foreach ($x as $y) { + z(); + } diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect @@ -0,0 +1,3 @@ +foreach ($x as $y) { + z(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test @@ -0,0 +1,3 @@ +foreach ($x as $y) { + z(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect @@ -0,0 +1,10 @@ +<?php + +class C { + public function f() { + D::X; + self::X; + parent::X; + static::X; + } +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source @@ -0,0 +1,10 @@ +foreach ($x as $y) { + z(); + } diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source @@ -0,0 +1,3 @@ + foreach ($x as $y) { + z(); + } diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect @@ -0,0 +1,5 @@ +<?php + +/* this comment +extends across +multiple lines */ diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source @@ -0,0 +1,5 @@ +foreach ($x as $y) { + z(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source @@ -0,0 +1,3 @@ +foreach ($x as $y) { + z(); +} diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect @@ -0,0 +1,3 @@ +<?php +// xyz + diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source @@ -0,0 +1,2 @@ +source = $source; + $this->scrub = $scrub; + } + + protected function didReceiveResult($result) { + list($err, $stdout, $stderr) = $result; + + if (!$err && strlen($stdout)) { + // Strip off fluff Pygments adds. + $stdout = preg_replace( + '@^
(.*)
\s*$@s', + '\1', + $stdout); + if ($this->scrub) { + $stdout = preg_replace('/^.*\n/', '', $stdout); + } + return phutil_safe_html($stdout); + } + + throw new PhutilSyntaxHighlighterException($stderr, $err); + } + +} diff --git a/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php @@ -0,0 +1,262 @@ +source = $source; + $this->scrub = $scrub; + } + + protected function didReceiveResult($result) { + try { + return $this->applyXHPHighlight($result); + } catch (Exception $ex) { + // XHP can't highlight source that isn't syntactically valid. Fall back + // to the fragment lexer. + $source = ($this->scrub + ? preg_replace('/^.*\n/', '', $this->source) + : $this->source); + return id(new PhutilLexerSyntaxHighlighter()) + ->setConfig('lexer', new PhutilPHPFragmentLexer()) + ->setConfig('language', 'php') + ->getHighlightFuture($source) + ->resolve(); + } + } + + private function applyXHPHighlight($result) { + + // We perform two passes here: one using the AST to find symbols we care + // about -- particularly, class names and function names. These are used + // in the crossreference stuff to link into Diffusion. After we've done our + // AST pass, we do a followup pass on the token stream to catch all the + // simple stuff like strings and comments. + + $tree = XHPASTTree::newFromDataAndResolvedExecFuture( + $this->source, + $result); + + $root = $tree->getRootNode(); + + $tokens = $root->getTokens(); + $interesting_symbols = $this->findInterestingSymbols($root); + + + if ($this->scrub) { + // If we're scrubbing, we prepended "= 2) { + if ($tokens[0]->getTypeName() === 'T_OPEN_TAG') { + if ($tokens[1]->getTypeName() === 'T_WHITESPACE') { + $ok = true; + } + } + } + + if (!$ok) { + throw new Exception( + pht( + 'Expected T_OPEN_TAG, T_WHITESPACE tokens at head of results '. + 'for highlighting parse of PHP snippet.')); + } + + // Remove the "getValue(); + if ((strlen($value) < 1) || ($value[0] != "\n")) { + throw new Exception( + pht( + 'Expected "\\n" at beginning of T_WHITESPACE token at head of '. + 'tokens for highlighting parse of PHP snippet.')); + } + + $value = substr($value, 1); + $tokens[1]->overwriteValue($value); + } + + $out = array(); + foreach ($tokens as $key => $token) { + $value = $token->getValue(); + $class = null; + $multi = false; + $attrs = array(); + if (isset($interesting_symbols[$key])) { + $sym = $interesting_symbols[$key]; + $class = $sym[0]; + $attrs['data-symbol-context'] = idx($sym, 'context'); + $attrs['data-symbol-name'] = idx($sym, 'symbol'); + } else { + switch ($token->getTypeName()) { + case 'T_WHITESPACE': + break; + case 'T_DOC_COMMENT': + $class = 'dc'; + $multi = true; + break; + case 'T_COMMENT': + $class = 'c'; + $multi = true; + break; + case 'T_CONSTANT_ENCAPSED_STRING': + case 'T_ENCAPSED_AND_WHITESPACE': + case 'T_INLINE_HTML': + $class = 's'; + $multi = true; + break; + case 'T_VARIABLE': + $class = 'nv'; + break; + case 'T_OPEN_TAG': + case 'T_OPEN_TAG_WITH_ECHO': + case 'T_CLOSE_TAG': + $class = 'o'; + break; + case 'T_LNUMBER': + case 'T_DNUMBER': + $class = 'm'; + break; + case 'T_STRING': + static $magic = array( + 'true' => true, + 'false' => true, + 'null' => true, + ); + if (isset($magic[strtolower($value)])) { + $class = 'k'; + break; + } + $class = 'nx'; + break; + default: + $class = 'k'; + break; + } + } + + if ($class) { + $attrs['class'] = $class; + if ($multi) { + // If the token may have multiple lines in it, make sure each + // crosses no more than one line so the lines can be put + // in a table, etc., later. + $value = phutil_split_lines($value, $retain_endings = true); + } else { + $value = array($value); + } + foreach ($value as $val) { + $out[] = phutil_tag('span', $attrs, $val); + } + } else { + $out[] = $value; + } + } + + return phutil_implode_html('', $out); + } + + private function findInterestingSymbols(XHPASTNode $root) { + // Class name symbols appear in: + // class X extends X implements X, X { ... } + // new X(); + // $x instanceof X + // catch (X $x) + // function f(X $x) + // X::f(); + // X::$m; + // X::CONST; + + // These are PHP builtin tokens which can appear in a classname context. + // Don't link them since they don't go anywhere useful. + static $builtin_class_tokens = array( + 'self' => true, + 'parent' => true, + 'static' => true, + ); + + // Fortunately XHPAST puts all of these in a special node type so it's + // easy to find them. + $result_map = array(); + $class_names = $root->selectDescendantsOfType('n_CLASS_NAME'); + foreach ($class_names as $class_name) { + foreach ($class_name->getTokens() as $key => $token) { + if (isset($builtin_class_tokens[$token->getValue()])) { + // This is something like "self::method()". + continue; + } + $result_map[$key] = array( + 'nc', // "Name, Class" + 'symbol' => $class_name->getConcreteString(), + ); + } + } + + // Function name symbols appear in: + // f() + + $function_calls = $root->selectDescendantsOfType('n_FUNCTION_CALL'); + foreach ($function_calls as $call) { + $call = $call->getChildByIndex(0); + if ($call->getTypeName() == 'n_SYMBOL_NAME') { + // This is a normal function call, not some $f() shenanigans. + foreach ($call->getTokens() as $key => $token) { + $result_map[$key] = array( + 'nf', // "Name, Function" + 'symbol' => $call->getConcreteString(), + ); + } + } + } + + // Upon encountering $x->y, link y without context, since $x is unknown. + + $prop_access = $root->selectDescendantsOfType('n_OBJECT_PROPERTY_ACCESS'); + foreach ($prop_access as $access) { + $right = $access->getChildByIndex(1); + if ($right->getTypeName() == 'n_INDEX_ACCESS') { + // otherwise $x->y[0] doesn't get highlighted + $right = $right->getChildByIndex(0); + } + if ($right->getTypeName() == 'n_STRING') { + foreach ($right->getTokens() as $key => $token) { + $result_map[$key] = array( + 'na', // "Name, Attribute" + 'symbol' => $right->getConcreteString(), + ); + } + } + } + + // Upon encountering x::y, try to link y with context x. + + $static_access = $root->selectDescendantsOfType('n_CLASS_STATIC_ACCESS'); + foreach ($static_access as $access) { + $class = $access->getChildByIndex(0); + $right = $access->getChildByIndex(1); + if ($class->getTypeName() == 'n_CLASS_NAME' && + ($right->getTypeName() == 'n_STRING' || + $right->getTypeName() == 'n_VARIABLE')) { + $classname = head($class->getTokens())->getValue(); + $result = array( + 'na', + 'symbol' => ltrim($right->getConcreteString(), '$'), + ); + if (!isset($builtin_class_tokens[$classname])) { + $result['context'] = $classname; + } + foreach ($right->getTokens() as $key => $token) { + $result_map[$key] = $result; + } + } + } + + return $result_map; + } + +}