Page MenuHomePhabricator
Files F18363520

D20977.id49983.diff
No OneTemporary
Actions

D20977.id49983.diff
View Options

diff --git a/.arclint b/.arclint
--- a/.arclint
+++ b/.arclint
@@ -64,13 +64,13 @@
"text": {
"type": "text",
"exclude": [
- "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))"
+ "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))"
]
},
"text-without-length": {
"type": "text",
"include": [
- "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))"
+ "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))"
],
"severity": {
"3": "disabled"
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -5591,6 +5591,7 @@
'PhrictionTransactionComment' => 'applications/phriction/storage/PhrictionTransactionComment.php',
'PhrictionTransactionEditor' => 'applications/phriction/editor/PhrictionTransactionEditor.php',
'PhrictionTransactionQuery' => 'applications/phriction/query/PhrictionTransactionQuery.php',
+ 'PhutilAPCKeyValueCache' => 'infrastructure/cache/PhutilAPCKeyValueCache.php',
'PhutilAmazonAuthAdapter' => 'applications/auth/adapter/PhutilAmazonAuthAdapter.php',
'PhutilAsanaAuthAdapter' => 'applications/auth/adapter/PhutilAsanaAuthAdapter.php',
'PhutilAuthAdapter' => 'applications/auth/adapter/PhutilAuthAdapter.php',
@@ -5620,7 +5621,15 @@
'PhutilCalendarRootNode' => 'applications/calendar/parser/data/PhutilCalendarRootNode.php',
'PhutilCalendarUserNode' => 'applications/calendar/parser/data/PhutilCalendarUserNode.php',
'PhutilCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php',
+ 'PhutilConsoleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php',
+ 'PhutilContextFreeGrammar' => 'infrastructure/lipsum/PhutilContextFreeGrammar.php',
+ 'PhutilDefaultSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php',
+ 'PhutilDefaultSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php',
+ 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php',
+ 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php',
+ 'PhutilDirectoryKeyValueCache' => 'infrastructure/cache/PhutilDirectoryKeyValueCache.php',
'PhutilDisqusAuthAdapter' => 'applications/auth/adapter/PhutilDisqusAuthAdapter.php',
+ 'PhutilDivinerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php',
'PhutilEmptyAuthAdapter' => 'applications/auth/adapter/PhutilEmptyAuthAdapter.php',
'PhutilFacebookAuthAdapter' => 'applications/auth/adapter/PhutilFacebookAuthAdapter.php',
'PhutilGitHubAuthAdapter' => 'applications/auth/adapter/PhutilGitHubAuthAdapter.php',
@@ -5630,19 +5639,38 @@
'PhutilICSParserTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php',
'PhutilICSWriter' => 'applications/calendar/parser/ics/PhutilICSWriter.php',
'PhutilICSWriterTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php',
+ 'PhutilInRequestKeyValueCache' => 'infrastructure/cache/PhutilInRequestKeyValueCache.php',
+ 'PhutilInvisibleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php',
'PhutilJIRAAuthAdapter' => 'applications/auth/adapter/PhutilJIRAAuthAdapter.php',
+ 'PhutilJSONFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php',
'PhutilJavaCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php',
+ 'PhutilKeyValueCache' => 'infrastructure/cache/PhutilKeyValueCache.php',
+ 'PhutilKeyValueCacheNamespace' => 'infrastructure/cache/PhutilKeyValueCacheNamespace.php',
+ 'PhutilKeyValueCacheProfiler' => 'infrastructure/cache/PhutilKeyValueCacheProfiler.php',
+ 'PhutilKeyValueCacheProxy' => 'infrastructure/cache/PhutilKeyValueCacheProxy.php',
+ 'PhutilKeyValueCacheStack' => 'infrastructure/cache/PhutilKeyValueCacheStack.php',
+ 'PhutilKeyValueCacheTestCase' => 'infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php',
'PhutilLDAPAuthAdapter' => 'applications/auth/adapter/PhutilLDAPAuthAdapter.php',
+ 'PhutilLexerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php',
'PhutilLipsumContextFreeGrammar' => 'infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php',
+ 'PhutilMarkupEngine' => 'infrastructure/markup/PhutilMarkupEngine.php',
+ 'PhutilMarkupTestCase' => 'infrastructure/markup/__tests__/PhutilMarkupTestCase.php',
+ 'PhutilMemcacheKeyValueCache' => 'infrastructure/cache/PhutilMemcacheKeyValueCache.php',
'PhutilOAuth1AuthAdapter' => 'applications/auth/adapter/PhutilOAuth1AuthAdapter.php',
'PhutilOAuthAuthAdapter' => 'applications/auth/adapter/PhutilOAuthAuthAdapter.php',
+ 'PhutilOnDiskKeyValueCache' => 'infrastructure/cache/PhutilOnDiskKeyValueCache.php',
'PhutilPHPCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php',
+ 'PhutilPHPFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php',
'PhutilPhabricatorAuthAdapter' => 'applications/auth/adapter/PhutilPhabricatorAuthAdapter.php',
'PhutilProseDiff' => 'infrastructure/diff/prose/PhutilProseDiff.php',
'PhutilProseDiffTestCase' => 'infrastructure/diff/prose/__tests__/PhutilProseDiffTestCase.php',
'PhutilProseDifferenceEngine' => 'infrastructure/diff/prose/PhutilProseDifferenceEngine.php',
+ 'PhutilPygmentizeParser' => 'infrastructure/parser/PhutilPygmentizeParser.php',
+ 'PhutilPygmentizeParserTestCase' => 'infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php',
+ 'PhutilPygmentsSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php',
'PhutilQsprintfInterface' => 'infrastructure/storage/xsprintf/PhutilQsprintfInterface.php',
'PhutilQueryString' => 'infrastructure/storage/xsprintf/PhutilQueryString.php',
+ 'PhutilRainbowSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php',
'PhutilRealNameContextFreeGrammar' => 'infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php',
'PhutilRemarkupAnchorRule' => 'infrastructure/markup/markuprule/PhutilRemarkupAnchorRule.php',
'PhutilRemarkupBlockInterpreter' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php',
@@ -5678,6 +5706,9 @@
'PhutilRemarkupTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php',
'PhutilRemarkupTestInterpreterRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php',
'PhutilRemarkupUnderlineRule' => 'infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php',
+ 'PhutilSafeHTML' => 'infrastructure/markup/PhutilSafeHTML.php',
+ 'PhutilSafeHTMLProducerInterface' => 'infrastructure/markup/PhutilSafeHTMLProducerInterface.php',
+ 'PhutilSafeHTMLTestCase' => 'infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php',
'PhutilSearchQueryCompiler' => 'applications/search/compiler/PhutilSearchQueryCompiler.php',
'PhutilSearchQueryCompilerSyntaxException' => 'applications/search/compiler/PhutilSearchQueryCompilerSyntaxException.php',
'PhutilSearchQueryCompilerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchQueryCompilerTestCase.php',
@@ -5685,9 +5716,17 @@
'PhutilSearchStemmer' => 'applications/search/compiler/PhutilSearchStemmer.php',
'PhutilSearchStemmerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchStemmerTestCase.php',
'PhutilSlackAuthAdapter' => 'applications/auth/adapter/PhutilSlackAuthAdapter.php',
+ 'PhutilSprite' => 'aphront/sprite/PhutilSprite.php',
+ 'PhutilSpriteSheet' => 'aphront/sprite/PhutilSpriteSheet.php',
+ 'PhutilSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php',
+ 'PhutilSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php',
+ 'PhutilSyntaxHighlighterException' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php',
'PhutilTwitchAuthAdapter' => 'applications/auth/adapter/PhutilTwitchAuthAdapter.php',
'PhutilTwitterAuthAdapter' => 'applications/auth/adapter/PhutilTwitterAuthAdapter.php',
'PhutilWordPressAuthAdapter' => 'applications/auth/adapter/PhutilWordPressAuthAdapter.php',
+ 'PhutilXHPASTSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php',
+ 'PhutilXHPASTSyntaxHighlighterFuture' => 'infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php',
+ 'PhutilXHPASTSyntaxHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php',
'PolicyLockOptionType' => 'applications/policy/config/PolicyLockOptionType.php',
'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php',
'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php',
@@ -5879,6 +5918,7 @@
'function' => array(
'celerity_generate_unique_node_id' => 'applications/celerity/api.php',
'celerity_get_resource_uri' => 'applications/celerity/api.php',
+ 'hsprintf' => 'infrastructure/markup/render.php',
'javelin_tag' => 'infrastructure/javelin/markup.php',
'phabricator_date' => 'view/viewutils.php',
'phabricator_datetime' => 'view/viewutils.php',
@@ -5890,6 +5930,15 @@
'phid_get_subtype' => 'applications/phid/utils.php',
'phid_get_type' => 'applications/phid/utils.php',
'phid_group_by_type' => 'applications/phid/utils.php',
+ 'phutil_escape_html' => 'infrastructure/markup/render.php',
+ 'phutil_escape_html_newlines' => 'infrastructure/markup/render.php',
+ 'phutil_escape_uri' => 'infrastructure/markup/render.php',
+ 'phutil_escape_uri_path_component' => 'infrastructure/markup/render.php',
+ 'phutil_implode_html' => 'infrastructure/markup/render.php',
+ 'phutil_safe_html' => 'infrastructure/markup/render.php',
+ 'phutil_tag' => 'infrastructure/markup/render.php',
+ 'phutil_tag_div' => 'infrastructure/markup/render.php',
+ 'phutil_unescape_uri_path_component' => 'infrastructure/markup/render.php',
'qsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php',
'qsprintf_check_scalar_type' => 'infrastructure/storage/xsprintf/qsprintf.php',
'qsprintf_check_type' => 'infrastructure/storage/xsprintf/qsprintf.php',
@@ -12443,6 +12492,7 @@
'PhrictionTransactionComment' => 'PhabricatorApplicationTransactionComment',
'PhrictionTransactionEditor' => 'PhabricatorApplicationTransactionEditor',
'PhrictionTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
+ 'PhutilAPCKeyValueCache' => 'PhutilKeyValueCache',
'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilAuthAdapter' => 'Phobject',
@@ -12472,7 +12522,15 @@
'PhutilCalendarRootNode' => 'PhutilCalendarContainerNode',
'PhutilCalendarUserNode' => 'PhutilCalendarNode',
'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar',
+ 'PhutilConsoleSyntaxHighlighter' => 'Phobject',
+ 'PhutilContextFreeGrammar' => 'Phobject',
+ 'PhutilDefaultSyntaxHighlighter' => 'Phobject',
+ 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine',
+ 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy',
+ 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'PhutilTestCase',
+ 'PhutilDirectoryKeyValueCache' => 'PhutilKeyValueCache',
'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilDivinerSyntaxHighlighter' => 'Phobject',
'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter',
'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter',
@@ -12482,18 +12540,37 @@
'PhutilICSParserTestCase' => 'PhutilTestCase',
'PhutilICSWriter' => 'Phobject',
'PhutilICSWriterTestCase' => 'PhutilTestCase',
+ 'PhutilInRequestKeyValueCache' => 'PhutilKeyValueCache',
+ 'PhutilInvisibleSyntaxHighlighter' => 'Phobject',
'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter',
+ 'PhutilJSONFragmentLexerHighlighterTestCase' => 'PhutilTestCase',
'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar',
+ 'PhutilKeyValueCache' => 'Phobject',
+ 'PhutilKeyValueCacheNamespace' => 'PhutilKeyValueCacheProxy',
+ 'PhutilKeyValueCacheProfiler' => 'PhutilKeyValueCacheProxy',
+ 'PhutilKeyValueCacheProxy' => 'PhutilKeyValueCache',
+ 'PhutilKeyValueCacheStack' => 'PhutilKeyValueCache',
+ 'PhutilKeyValueCacheTestCase' => 'PhutilTestCase',
'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter',
+ 'PhutilLexerSyntaxHighlighter' => 'PhutilSyntaxHighlighter',
'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar',
+ 'PhutilMarkupEngine' => 'Phobject',
+ 'PhutilMarkupTestCase' => 'PhutilTestCase',
+ 'PhutilMemcacheKeyValueCache' => 'PhutilKeyValueCache',
'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter',
'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter',
+ 'PhutilOnDiskKeyValueCache' => 'PhutilKeyValueCache',
'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar',
+ 'PhutilPHPFragmentLexerHighlighterTestCase' => 'PhutilTestCase',
'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilProseDiff' => 'Phobject',
'PhutilProseDiffTestCase' => 'PhabricatorTestCase',
'PhutilProseDifferenceEngine' => 'Phobject',
+ 'PhutilPygmentizeParser' => 'Phobject',
+ 'PhutilPygmentizeParserTestCase' => 'PhutilTestCase',
+ 'PhutilPygmentsSyntaxHighlighter' => 'Phobject',
'PhutilQueryString' => 'Phobject',
+ 'PhutilRainbowSyntaxHighlighter' => 'Phobject',
'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar',
'PhutilRemarkupAnchorRule' => 'PhutilRemarkupRule',
'PhutilRemarkupBlockInterpreter' => 'Phobject',
@@ -12529,6 +12606,8 @@
'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule',
'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter',
'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule',
+ 'PhutilSafeHTML' => 'Phobject',
+ 'PhutilSafeHTMLTestCase' => 'PhutilTestCase',
'PhutilSearchQueryCompiler' => 'Phobject',
'PhutilSearchQueryCompilerSyntaxException' => 'Exception',
'PhutilSearchQueryCompilerTestCase' => 'PhutilTestCase',
@@ -12536,9 +12615,17 @@
'PhutilSearchStemmer' => 'Phobject',
'PhutilSearchStemmerTestCase' => 'PhutilTestCase',
'PhutilSlackAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilSprite' => 'Phobject',
+ 'PhutilSpriteSheet' => 'Phobject',
+ 'PhutilSyntaxHighlighter' => 'Phobject',
+ 'PhutilSyntaxHighlighterEngine' => 'Phobject',
+ 'PhutilSyntaxHighlighterException' => 'Exception',
'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter',
'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilXHPASTSyntaxHighlighter' => 'Phobject',
+ 'PhutilXHPASTSyntaxHighlighterFuture' => 'FutureProxy',
+ 'PhutilXHPASTSyntaxHighlighterTestCase' => 'PhutilTestCase',
'PolicyLockOptionType' => 'PhabricatorConfigJSONOptionType',
'PonderAddAnswerView' => 'AphrontView',
'PonderAnswer' => array(
diff --git a/src/aphront/sprite/PhutilSprite.php b/src/aphront/sprite/PhutilSprite.php
new file mode 100644
--- /dev/null
+++ b/src/aphront/sprite/PhutilSprite.php
@@ -0,0 +1,76 @@
+<?php
+
+/**
+ * NOTE: This is very new and unstable.
+ */
+final class PhutilSprite extends Phobject {
+
+ private $sourceFiles = array();
+ private $sourceX;
+ private $sourceY;
+ private $sourceW;
+ private $sourceH;
+ private $targetCSS;
+ private $spriteSheet;
+ private $name;
+
+ public function setName($name) {
+ $this->name = $name;
+ return $this;
+ }
+
+ public function getName() {
+ return $this->name;
+ }
+
+ public function setTargetCSS($target_css) {
+ $this->targetCSS = $target_css;
+ return $this;
+ }
+
+ public function getTargetCSS() {
+ return $this->targetCSS;
+ }
+
+ public function setSourcePosition($x, $y) {
+ $this->sourceX = $x;
+ $this->sourceY = $y;
+ return $this;
+ }
+
+ public function setSourceSize($w, $h) {
+ $this->sourceW = $w;
+ $this->sourceH = $h;
+ return $this;
+ }
+
+ public function getSourceH() {
+ return $this->sourceH;
+ }
+
+ public function getSourceW() {
+ return $this->sourceW;
+ }
+
+ public function getSourceY() {
+ return $this->sourceY;
+ }
+
+ public function getSourceX() {
+ return $this->sourceX;
+ }
+
+ public function setSourceFile($source_file, $scale = 1) {
+ $this->sourceFiles[$scale] = $source_file;
+ return $this;
+ }
+
+ public function getSourceFile($scale) {
+ if (empty($this->sourceFiles[$scale])) {
+ throw new Exception(pht("No source file for scale '%s'!", $scale));
+ }
+
+ return $this->sourceFiles[$scale];
+ }
+
+}
diff --git a/src/aphront/sprite/PhutilSpriteSheet.php b/src/aphront/sprite/PhutilSpriteSheet.php
new file mode 100644
--- /dev/null
+++ b/src/aphront/sprite/PhutilSpriteSheet.php
@@ -0,0 +1,385 @@
+<?php
+
+/**
+ * NOTE: This is very new and unstable.
+ */
+final class PhutilSpriteSheet extends Phobject {
+
+ const MANIFEST_VERSION = 1;
+
+ const TYPE_STANDARD = 'standard';
+ const TYPE_REPEAT_X = 'repeat-x';
+ const TYPE_REPEAT_Y = 'repeat-y';
+
+ private $sprites = array();
+ private $sources = array();
+ private $hashes = array();
+ private $cssHeader;
+ private $generated;
+ private $scales = array(1);
+ private $type = self::TYPE_STANDARD;
+ private $basePath;
+
+ private $css;
+ private $images;
+
+ public function addSprite(PhutilSprite $sprite) {
+ $this->generated = false;
+ $this->sprites[] = $sprite;
+ return $this;
+ }
+
+ public function setCSSHeader($header) {
+ $this->generated = false;
+ $this->cssHeader = $header;
+ return $this;
+ }
+
+ public function setScales(array $scales) {
+ $this->scales = array_values($scales);
+ return $this;
+ }
+
+ public function getScales() {
+ return $this->scales;
+ }
+
+ public function setSheetType($type) {
+ $this->type = $type;
+ return $this;
+ }
+
+ public function setBasePath($base_path) {
+ $this->basePath = $base_path;
+ return $this;
+ }
+
+ private function generate() {
+ if ($this->generated) {
+ return;
+ }
+
+ $multi_row = true;
+ $multi_col = true;
+ $margin_w = 1;
+ $margin_h = 1;
+
+ $type = $this->type;
+ switch ($type) {
+ case self::TYPE_STANDARD:
+ break;
+ case self::TYPE_REPEAT_X:
+ $multi_col = false;
+ $margin_w = 0;
+
+ $width = null;
+ foreach ($this->sprites as $sprite) {
+ if ($width === null) {
+ $width = $sprite->getSourceW();
+ } else if ($width !== $sprite->getSourceW()) {
+ throw new Exception(
+ pht(
+ "All sprites in a '%s' sheet must have the same width.",
+ 'repeat-x'));
+ }
+ }
+ break;
+ case self::TYPE_REPEAT_Y:
+ $multi_row = false;
+ $margin_h = 0;
+
+ $height = null;
+ foreach ($this->sprites as $sprite) {
+ if ($height === null) {
+ $height = $sprite->getSourceH();
+ } else if ($height !== $sprite->getSourceH()) {
+ throw new Exception(
+ pht(
+ "All sprites in a '%s' sheet must have the same height.",
+ 'repeat-y'));
+ }
+ }
+ break;
+ default:
+ throw new Exception(pht("Unknown sprite sheet type '%s'!", $type));
+ }
+
+
+ $css = array();
+ if ($this->cssHeader) {
+ $css[] = $this->cssHeader;
+ }
+
+ $out_w = 0;
+ $out_h = 0;
+
+ // Lay out the sprite sheet. We attempt to build a roughly square sheet
+ // so it's easier to manage, since 2000x20 is more cumbersome for humans
+ // to deal with than 200x200.
+ //
+ // To do this, we use a simple greedy algorithm, adding sprites one at a
+ // time. For each sprite, if the sheet is at least as wide as it is tall
+ // we create a new row. Otherwise, we try to add it to an existing row.
+ //
+ // This isn't optimal, but does a reasonable job in most cases and isn't
+ // too messy.
+
+ // Group the sprites by their sizes. We lay them out in the sheet as
+ // boxes, but then put them into the boxes in the order they were added
+ // so similar sprites end up nearby on the final sheet.
+ $boxes = array();
+ foreach (array_reverse($this->sprites) as $sprite) {
+ $s_w = $sprite->getSourceW() + $margin_w;
+ $s_h = $sprite->getSourceH() + $margin_h;
+ $boxes[$s_w][$s_h][] = $sprite;
+ }
+
+ $rows = array();
+ foreach ($this->sprites as $sprite) {
+ $s_w = $sprite->getSourceW() + $margin_w;
+ $s_h = $sprite->getSourceH() + $margin_h;
+
+ // Choose a row for this sprite.
+ $maybe = array();
+ foreach ($rows as $key => $row) {
+ if ($row['h'] < $s_h) {
+ // We can only add it to a row if the row is at least as tall as the
+ // sprite.
+ continue;
+ }
+ // We prefer rows which have the same height as the sprite, and then
+ // rows which aren't yet very wide.
+ $wasted_v = ($row['h'] - $s_h);
+ $wasted_h = ($row['w'] / $out_w);
+ $maybe[$key] = $wasted_v + $wasted_h;
+ }
+
+ $row_key = null;
+ if ($maybe && $multi_col) {
+ // If there were any candidate rows, pick the best one.
+ asort($maybe);
+ $row_key = head_key($maybe);
+ }
+
+ if ($row_key !== null && $multi_row) {
+ // If there's a candidate row, but adding the sprite to it would make
+ // the sprite wider than it is tall, create a new row instead. This
+ // generally keeps the sprite square-ish.
+ if ($rows[$row_key]['w'] + $s_w > $out_h) {
+ $row_key = null;
+ }
+ }
+
+ if ($row_key === null) {
+ // Add a new row.
+ $rows[] = array(
+ 'w' => 0,
+ 'h' => $s_h,
+ 'boxes' => array(),
+ );
+ $row_key = last_key($rows);
+ $out_h += $s_h;
+ }
+
+ // Add the sprite box to the row.
+ $row = $rows[$row_key];
+ $row['w'] += $s_w;
+ $row['boxes'][] = array($s_w, $s_h);
+ $rows[$row_key] = $row;
+
+ $out_w = max($row['w'], $out_w);
+ }
+
+ $images = array();
+ foreach ($this->scales as $scale) {
+ $img = imagecreatetruecolor($out_w * $scale, $out_h * $scale);
+ imagesavealpha($img, true);
+ imagefill($img, 0, 0, imagecolorallocatealpha($img, 0, 0, 0, 127));
+
+ $images[$scale] = $img;
+ }
+
+
+ // Put the shorter rows first. At the same height, put the wider rows first.
+ // This makes the resulting sheet more human-readable.
+ foreach ($rows as $key => $row) {
+ $rows[$key]['sort'] = $row['h'] + (1 - ($row['w'] / $out_w));
+ }
+ $rows = isort($rows, 'sort');
+
+ $pos_x = 0;
+ $pos_y = 0;
+ $rules = array();
+ foreach ($rows as $row) {
+ $max_h = 0;
+ foreach ($row['boxes'] as $box) {
+ $sprite = array_pop($boxes[$box[0]][$box[1]]);
+
+ foreach ($images as $scale => $img) {
+ $src = $this->loadSource($sprite, $scale);
+ imagecopy(
+ $img,
+ $src,
+ $scale * $pos_x, $scale * $pos_y,
+ $scale * $sprite->getSourceX(), $scale * $sprite->getSourceY(),
+ $scale * $sprite->getSourceW(), $scale * $sprite->getSourceH());
+ }
+
+ $rule = $sprite->getTargetCSS();
+ $cssx = (-$pos_x).'px';
+ $cssy = (-$pos_y).'px';
+
+ $rules[$sprite->getName()] = "{$rule} {\n".
+ " background-position: {$cssx} {$cssy};\n}";
+
+ $pos_x += $sprite->getSourceW() + $margin_w;
+ $max_h = max($max_h, $sprite->getSourceH());
+ }
+ $pos_x = 0;
+ $pos_y += $max_h + $margin_h;
+ }
+
+ // Generate CSS rules in input order.
+ foreach ($this->sprites as $sprite) {
+ $css[] = $rules[$sprite->getName()];
+ }
+
+ $this->images = $images;
+ $this->css = implode("\n\n", $css)."\n";
+ $this->generated = true;
+ }
+
+ public function generateImage($path, $scale = 1) {
+ $this->generate();
+ $this->log(pht("Writing sprite '%s'...", $path));
+ imagepng($this->images[$scale], $path);
+ return $this;
+ }
+
+ public function generateCSS($path) {
+ $this->generate();
+ $this->log(pht("Writing CSS '%s'...", $path));
+
+ $out = $this->css;
+ $out = str_replace('{X}', imagesx($this->images[1]), $out);
+ $out = str_replace('{Y}', imagesy($this->images[1]), $out);
+
+ Filesystem::writeFile($path, $out);
+ return $this;
+ }
+
+ public function needsRegeneration(array $manifest) {
+ return ($this->buildManifest() !== $manifest);
+ }
+
+ private function buildManifest() {
+ $output = array();
+ foreach ($this->sprites as $sprite) {
+ $output[$sprite->getName()] = array(
+ 'name' => $sprite->getName(),
+ 'rule' => $sprite->getTargetCSS(),
+ 'hash' => $this->loadSourceHash($sprite),
+ );
+ }
+
+ ksort($output);
+
+ $data = array(
+ 'version' => self::MANIFEST_VERSION,
+ 'sprites' => $output,
+ 'scales' => $this->scales,
+ 'header' => $this->cssHeader,
+ 'type' => $this->type,
+ );
+
+ return $data;
+ }
+
+ public function generateManifest($path) {
+ $data = $this->buildManifest();
+
+ $json = new PhutilJSON();
+ $data = $json->encodeFormatted($data);
+ Filesystem::writeFile($path, $data);
+ return $this;
+ }
+
+ private function log($message) {
+ echo $message."\n";
+ }
+
+ private function loadSourceHash(PhutilSprite $sprite) {
+ $inputs = array();
+
+ foreach ($this->scales as $scale) {
+ $file = $sprite->getSourceFile($scale);
+
+ // If two users have a project in different places, like:
+ //
+ // /home/alincoln/project
+ // /home/htaft/project
+ //
+ // ...we want to ignore the `/home/alincoln` part when hashing the sheet,
+ // since the sprites don't change when the project directory moves. If
+ // the base path is set, build the hashes using paths relative to the
+ // base path.
+
+ $file_key = $file;
+ if ($this->basePath) {
+ $file_key = Filesystem::readablePath($file, $this->basePath);
+ }
+
+ if (empty($this->hashes[$file_key])) {
+ $this->hashes[$file_key] = md5(Filesystem::readFile($file));
+ }
+
+ $inputs[] = $file_key;
+ $inputs[] = $this->hashes[$file_key];
+ }
+
+ $inputs[] = $sprite->getSourceX();
+ $inputs[] = $sprite->getSourceY();
+ $inputs[] = $sprite->getSourceW();
+ $inputs[] = $sprite->getSourceH();
+
+ return md5(implode(':', $inputs));
+ }
+
+ private function loadSource(PhutilSprite $sprite, $scale) {
+ $file = $sprite->getSourceFile($scale);
+ if (empty($this->sources[$file])) {
+ $data = Filesystem::readFile($file);
+ $image = imagecreatefromstring($data);
+ $this->sources[$file] = array(
+ 'image' => $image,
+ 'x' => imagesx($image),
+ 'y' => imagesy($image),
+ );
+ }
+
+ $s_w = $sprite->getSourceW() * $scale;
+ $i_w = $this->sources[$file]['x'];
+ if ($s_w > $i_w) {
+ throw new Exception(
+ pht(
+ "Sprite source for '%s' is too small (expected width %d, found %d).",
+ $file,
+ $s_w,
+ $i_w));
+ }
+
+ $s_h = $sprite->getSourceH() * $scale;
+ $i_h = $this->sources[$file]['y'];
+ if ($s_h > $i_h) {
+ throw new Exception(
+ pht(
+ "Sprite source for '%s' is too small (expected height %d, found %d).",
+ $file,
+ $s_h,
+ $i_h));
+ }
+
+ return $this->sources[$file]['image'];
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilAPCKeyValueCache.php b/src/infrastructure/cache/PhutilAPCKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilAPCKeyValueCache.php
@@ -0,0 +1,97 @@
+<?php
+
+/**
+ * Interface to the APC key-value cache. This is a very high-performance cache
+ * which is local to the current machine.
+ */
+final class PhutilAPCKeyValueCache extends PhutilKeyValueCache {
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return (function_exists('apc_fetch') || function_exists('apcu_fetch')) &&
+ ini_get('apc.enabled') &&
+ (ini_get('apc.enable_cli') || php_sapi_name() != 'cli');
+ }
+
+ public function getKeys(array $keys, $ttl = null) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ $results = array();
+ $fetched = false;
+ foreach ($keys as $key) {
+ if ($is_apcu) {
+ $result = apcu_fetch($key, $fetched);
+ } else {
+ $result = apc_fetch($key, $fetched);
+ }
+
+ if ($fetched) {
+ $results[$key] = $result;
+ }
+ }
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ // NOTE: Although modern APC supports passing an array to `apc_store()`,
+ // it is not supported by older version of APC or by HPHP.
+
+ foreach ($keys as $key => $value) {
+ if ($is_apcu) {
+ apcu_store($key, $value, $ttl);
+ } else {
+ apc_store($key, $value, $ttl);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ foreach ($keys as $key) {
+ if ($is_apcu) {
+ apcu_delete($key);
+ } else {
+ apc_delete($key);
+ }
+ }
+
+ return $this;
+ }
+
+ public function destroyCache() {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ if ($is_apcu) {
+ apcu_clear_cache();
+ } else {
+ apc_clear_cache('user');
+ }
+
+ return $this;
+ }
+
+ private static function isAPCu() {
+ return function_exists('apcu_fetch');
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php
@@ -0,0 +1,244 @@
+<?php
+
+/**
+ * Interface to a directory-based disk cache. Storage persists across requests.
+ *
+ * This cache is very very slow, and most suitable for command line scripts
+ * which need to build large caches derived from sources like working copies
+ * (for example, Diviner). This cache performs better for large amounts of
+ * data than @{class:PhutilOnDiskKeyValueCache} because each key is serialized
+ * individually, but this comes at the cost of having even slower reads and
+ * writes.
+ *
+ * In addition to having slow reads and writes, this entire cache locks for
+ * any read or write activity.
+ *
+ * Keys for this cache treat the character "/" specially, and encode it as
+ * a new directory on disk. This can help keep the cache organized and keep the
+ * number of items in any single directory under control, by using keys like
+ * "ab/cd/efghijklmn".
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ * @task storage Cache Storage
+ */
+final class PhutilDirectoryKeyValueCache extends PhutilKeyValueCache {
+
+ private $lock;
+ private $cacheDirectory;
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+
+ public function getKeys(array $keys) {
+ $this->validateKeys($keys);
+
+ try {
+ $this->lockCache();
+ } catch (PhutilLockException $ex) {
+ return array();
+ }
+
+ $now = time();
+
+ $results = array();
+ foreach ($keys as $key) {
+ $key_file = $this->getKeyFile($key);
+ try {
+ $data = Filesystem::readFile($key_file);
+ } catch (FilesystemException $ex) {
+ continue;
+ }
+
+ $data = unserialize($data);
+ if (!$data) {
+ continue;
+ }
+
+ if (isset($data['ttl']) && $data['ttl'] < $now) {
+ continue;
+ }
+
+ $results[$key] = $data['value'];
+ }
+
+ $this->unlockCache();
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ $this->validateKeys(array_keys($keys));
+
+ $this->lockCache(15);
+
+ if ($ttl) {
+ $ttl_epoch = time() + $ttl;
+ } else {
+ $ttl_epoch = null;
+ }
+
+ foreach ($keys as $key => $value) {
+ $dict = array(
+ 'value' => $value,
+ );
+ if ($ttl_epoch) {
+ $dict['ttl'] = $ttl_epoch;
+ }
+
+ try {
+ $key_file = $this->getKeyFile($key);
+ $key_dir = dirname($key_file);
+ if (!Filesystem::pathExists($key_dir)) {
+ Filesystem::createDirectory(
+ $key_dir,
+ $mask = 0755,
+ $recursive = true);
+ }
+
+ $new_file = $key_file.'.new';
+ Filesystem::writeFile($new_file, serialize($dict));
+ Filesystem::rename($new_file, $key_file);
+ } catch (FilesystemException $ex) {
+ phlog($ex);
+ }
+ }
+
+ $this->unlockCache();
+
+ return $this;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $this->validateKeys($keys);
+
+ $this->lockCache(15);
+
+ foreach ($keys as $key) {
+ $path = $this->getKeyFile($key);
+ Filesystem::remove($path);
+
+ // If removing this key leaves the directory empty, clean it up. Then
+ // clean up any empty parent directories.
+ $path = dirname($path);
+ do {
+ if (!Filesystem::isDescendant($path, $this->getCacheDirectory())) {
+ break;
+ }
+ if (Filesystem::listDirectory($path, true)) {
+ break;
+ }
+ Filesystem::remove($path);
+ $path = dirname($path);
+ } while (true);
+ }
+
+ $this->unlockCache();
+
+ return $this;
+ }
+
+
+ public function destroyCache() {
+ Filesystem::remove($this->getCacheDirectory());
+ return $this;
+ }
+
+
+/* -( Cache Storage )------------------------------------------------------ */
+
+
+ /**
+ * @task storage
+ */
+ public function setCacheDirectory($directory) {
+ $this->cacheDirectory = rtrim($directory, '/').'/';
+ return $this;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getCacheDirectory() {
+ if (!$this->cacheDirectory) {
+ throw new PhutilInvalidStateException('setCacheDirectory');
+ }
+ return $this->cacheDirectory;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getKeyFile($key) {
+ // Colon is a drive separator on Windows.
+ $key = str_replace(':', '_', $key);
+
+ // NOTE: We add ".cache" to each file so we don't get a collision if you
+ // set the keys "a" and "a/b". Without ".cache", the file "a" would need
+ // to be both a file and a directory.
+ return $this->getCacheDirectory().$key.'.cache';
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function validateKeys(array $keys) {
+ foreach ($keys as $key) {
+ // NOTE: Use of "." is reserved for ".lock", "key.new" and "key.cache".
+ // Use of "_" is reserved for converting ":".
+ if (!preg_match('@^[a-zA-Z0-9/:-]+$@', $key)) {
+ throw new Exception(
+ pht(
+ "Invalid key '%s': directory caches may only contain letters, ".
+ "numbers, hyphen, colon and slash.",
+ $key));
+ }
+ }
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function lockCache($wait = 0) {
+ if ($this->lock) {
+ throw new Exception(
+ pht(
+ 'Trying to %s with a lock!',
+ __FUNCTION__.'()'));
+ }
+
+ if (!Filesystem::pathExists($this->getCacheDirectory())) {
+ Filesystem::createDirectory($this->getCacheDirectory(), 0755, true);
+ }
+
+ $lock = PhutilFileLock::newForPath($this->getCacheDirectory().'.lock');
+ $lock->lock($wait);
+
+ $this->lock = $lock;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function unlockCache() {
+ if (!$this->lock) {
+ throw new PhutilInvalidStateException('lockCache');
+ }
+
+ $this->lock->unlock();
+ $this->lock = null;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilInRequestKeyValueCache.php b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php
@@ -0,0 +1,118 @@
+<?php
+
+/**
+ * Key-value cache implemented in the current request. All storage is local
+ * to this request (i.e., the current page) and destroyed after the request
+ * exits. This means the first request to this cache for a given key on a page
+ * will ALWAYS miss.
+ *
+ * This cache exists mostly to support unit tests. In a well-designed
+ * applications, you generally should not be fetching the same data over and
+ * over again in one request, so this cache should be of limited utility.
+ * If using this cache improves application performance, especially if it
+ * improves it significantly, it may indicate an architectural problem in your
+ * application.
+ */
+final class PhutilInRequestKeyValueCache extends PhutilKeyValueCache {
+
+ private $cache = array();
+ private $ttl = array();
+ private $limit = 0;
+
+
+ /**
+ * Set a limit on the number of keys this cache may contain.
+ *
+ * When too many keys are inserted, the oldest keys are removed from the
+ * cache. Setting a limit of `0` disables the cache.
+ *
+ * @param int Maximum number of items to store in the cache.
+ * @return this
+ */
+ public function setLimit($limit) {
+ $this->limit = $limit;
+ return $this;
+ }
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+ public function getKeys(array $keys) {
+ $results = array();
+ $now = time();
+ foreach ($keys as $key) {
+ if (!isset($this->cache[$key]) && !array_key_exists($key, $this->cache)) {
+ continue;
+ }
+ if (isset($this->ttl[$key]) && ($this->ttl[$key] < $now)) {
+ continue;
+ }
+ $results[$key] = $this->cache[$key];
+ }
+
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+
+ foreach ($keys as $key => $value) {
+ $this->cache[$key] = $value;
+ }
+
+ if ($ttl) {
+ $end = time() + $ttl;
+ foreach ($keys as $key => $value) {
+ $this->ttl[$key] = $end;
+ }
+ } else {
+ foreach ($keys as $key => $value) {
+ unset($this->ttl[$key]);
+ }
+ }
+
+ if ($this->limit) {
+ $count = count($this->cache);
+ if ($count > $this->limit) {
+ $remove = array();
+ foreach ($this->cache as $key => $value) {
+ $remove[] = $key;
+
+ $count--;
+ if ($count <= $this->limit) {
+ break;
+ }
+ }
+
+ $this->deleteKeys($remove);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ foreach ($keys as $key) {
+ unset($this->cache[$key]);
+ unset($this->ttl[$key]);
+ }
+
+ return $this;
+ }
+
+ public function getAllKeys() {
+ return $this->cache;
+ }
+
+ public function destroyCache() {
+ $this->cache = array();
+ $this->ttl = array();
+
+ return $this;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCache.php b/src/infrastructure/cache/PhutilKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCache.php
@@ -0,0 +1,121 @@
+<?php
+
+/**
+ * Interface to a key-value cache like Memcache or APC. This class provides a
+ * uniform interface to multiple different key-value caches and integration
+ * with PhutilServiceProfiler.
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ */
+abstract class PhutilKeyValueCache extends Phobject {
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ /**
+ * Determine if the cache is available. For example, the APC cache tests if
+ * APC is installed. If this method returns false, the cache is not
+ * operational and can not be used.
+ *
+ * @return bool True if the cache can be used.
+ * @task kvimpl
+ */
+ public function isAvailable() {
+ return false;
+ }
+
+
+ /**
+ * Get a single key from cache. See @{method:getKeys} to get multiple keys at
+ * once.
+ *
+ * @param string Key to retrieve.
+ * @param wild Optional value to return if the key is not found. By
+ * default, returns null.
+ * @return wild Cache value (on cache hit) or default value (on cache
+ * miss).
+ * @task kvimpl
+ */
+ final public function getKey($key, $default = null) {
+ $map = $this->getKeys(array($key));
+ return idx($map, $key, $default);
+ }
+
+
+ /**
+ * Set a single key in cache. See @{method:setKeys} to set multiple keys at
+ * once.
+ *
+ * See @{method:setKeys} for a description of TTLs.
+ *
+ * @param string Key to set.
+ * @param wild Value to set.
+ * @param int|null Optional TTL.
+ * @return this
+ * @task kvimpl
+ */
+ final public function setKey($key, $value, $ttl = null) {
+ return $this->setKeys(array($key => $value), $ttl);
+ }
+
+
+ /**
+ * Delete a key from the cache. See @{method:deleteKeys} to delete multiple
+ * keys at once.
+ *
+ * @param string Key to delete.
+ * @return this
+ * @task kvimpl
+ */
+ final public function deleteKey($key) {
+ return $this->deleteKeys(array($key));
+ }
+
+
+ /**
+ * Get data from the cache.
+ *
+ * @param list<string> List of cache keys to retrieve.
+ * @return dict<string, wild> Dictionary of keys that were found in the
+ * cache. Keys not present in the cache are
+ * omitted, so you can detect a cache miss.
+ * @task kvimpl
+ */
+ abstract public function getKeys(array $keys);
+
+
+ /**
+ * Put data into the key-value cache.
+ *
+ * With a TTL ("time to live"), the cache will automatically delete the key
+ * after a specified number of seconds. By default, there is no expiration
+ * policy and data will persist in cache indefinitely.
+ *
+ * @param dict<string, wild> Map of cache keys to values.
+ * @param int|null TTL for cache keys, in seconds.
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function setKeys(array $keys, $ttl = null);
+
+
+ /**
+ * Delete a list of keys from the cache.
+ *
+ * @param list<string> List of keys to delete.
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function deleteKeys(array $keys);
+
+
+ /**
+ * Completely destroy all data in the cache.
+ *
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function destroyCache();
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php
@@ -0,0 +1,65 @@
+<?php
+
+final class PhutilKeyValueCacheNamespace extends PhutilKeyValueCacheProxy {
+
+ private $namespace;
+
+ public function setNamespace($namespace) {
+ if (strpos($namespace, ':') !== false) {
+ throw new Exception(pht("Namespace can't contain colons."));
+ }
+
+ $this->namespace = $namespace.':';
+
+ return $this;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ return parent::setKeys(array_combine(
+ $this->prefixKeys(array_keys($keys)),
+ $keys), $ttl);
+ }
+
+ public function getKeys(array $keys) {
+ $results = parent::getKeys($this->prefixKeys($keys));
+
+ if (!$results) {
+ return array();
+ }
+
+ return array_combine(
+ $this->unprefixKeys(array_keys($results)),
+ $results);
+ }
+
+ public function deleteKeys(array $keys) {
+ return parent::deleteKeys($this->prefixKeys($keys));
+ }
+
+ private function prefixKeys(array $keys) {
+ if ($this->namespace == null) {
+ throw new Exception(pht('Namespace not set.'));
+ }
+
+ $prefixed_keys = array();
+ foreach ($keys as $key) {
+ $prefixed_keys[] = $this->namespace.$key;
+ }
+
+ return $prefixed_keys;
+ }
+
+ private function unprefixKeys(array $keys) {
+ if ($this->namespace == null) {
+ throw new Exception(pht('Namespace not set.'));
+ }
+
+ $unprefixed_keys = array();
+ foreach ($keys as $key) {
+ $unprefixed_keys[] = substr($key, strlen($this->namespace));
+ }
+
+ return $unprefixed_keys;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php
@@ -0,0 +1,108 @@
+<?php
+
+final class PhutilKeyValueCacheProfiler extends PhutilKeyValueCacheProxy {
+
+ private $profiler;
+ private $name;
+
+ public function setName($name) {
+ $this->name = $name;
+ return $this;
+ }
+
+ public function getName() {
+ return $this->name;
+ }
+
+ /**
+ * Set a profiler for cache operations.
+ *
+ * @param PhutilServiceProfiler Service profiler.
+ * @return this
+ * @task kvimpl
+ */
+ public function setProfiler(PhutilServiceProfiler $profiler) {
+ $this->profiler = $profiler;
+ return $this;
+ }
+
+
+ /**
+ * Get the current profiler.
+ *
+ * @return PhutilServiceProfiler|null Profiler, or null if none is set.
+ * @task kvimpl
+ */
+ public function getProfiler() {
+ return $this->profiler;
+ }
+
+
+ public function getKeys(array $keys) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-get',
+ 'name' => $this->getName(),
+ 'keys' => $keys,
+ ));
+ }
+
+ $results = parent::getKeys($keys);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall(
+ $call_id,
+ array(
+ 'hits' => array_keys($results),
+ ));
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-set',
+ 'name' => $this->getName(),
+ 'keys' => array_keys($keys),
+ 'ttl' => $ttl,
+ ));
+ }
+
+ $result = parent::setKeys($keys, $ttl);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall($call_id, array());
+ }
+
+ return $result;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-del',
+ 'name' => $this->getName(),
+ 'keys' => $keys,
+ ));
+ }
+
+ $result = parent::deleteKeys($keys);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall($call_id, array());
+ }
+
+ return $result;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProxy.php b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php
@@ -0,0 +1,45 @@
+<?php
+
+abstract class PhutilKeyValueCacheProxy extends PhutilKeyValueCache {
+
+ private $proxy;
+
+ final public function __construct(PhutilKeyValueCache $proxy) {
+ $this->proxy = $proxy;
+ }
+
+ final protected function getProxy() {
+ return $this->proxy;
+ }
+
+ public function isAvailable() {
+ return $this->getProxy()->isAvailable();
+ }
+
+
+ public function getKeys(array $keys) {
+ return $this->getProxy()->getKeys($keys);
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ return $this->getProxy()->setKeys($keys, $ttl);
+ }
+
+
+ public function deleteKeys(array $keys) {
+ return $this->getProxy()->deleteKeys($keys);
+ }
+
+
+ public function destroyCache() {
+ return $this->getProxy()->destroyCache();
+ }
+
+ public function __call($method, array $arguments) {
+ return call_user_func_array(
+ array($this->getProxy(), $method),
+ $arguments);
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheStack.php b/src/infrastructure/cache/PhutilKeyValueCacheStack.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheStack.php
@@ -0,0 +1,131 @@
+<?php
+
+/**
+ * Stacks multiple caches on top of each other, with readthrough semantics:
+ *
+ * - For reads, we try each cache in order until we find all the keys.
+ * - For writes, we set the keys in each cache.
+ *
+ * @task config Configuring the Stack
+ */
+final class PhutilKeyValueCacheStack extends PhutilKeyValueCache {
+
+
+ /**
+ * Forward list of caches in the stack (from the nearest cache to the farthest
+ * cache).
+ */
+ private $cachesForward;
+
+
+ /**
+ * Backward list of caches in the stack (from the farthest cache to the
+ * nearest cache).
+ */
+ private $cachesBackward;
+
+
+ /**
+ * TTL to use for any writes which are side effects of the next read
+ * operation.
+ */
+ private $nextTTL;
+
+
+/* -( Configuring the Stack )---------------------------------------------- */
+
+
+ /**
+ * Set the caches which comprise this stack.
+ *
+ * @param list<PhutilKeyValueCache> Ordered list of key-value caches.
+ * @return this
+ * @task config
+ */
+ public function setCaches(array $caches) {
+ assert_instances_of($caches, 'PhutilKeyValueCache');
+ $this->cachesForward = $caches;
+ $this->cachesBackward = array_reverse($caches);
+
+ return $this;
+ }
+
+
+ /**
+ * Set the readthrough TTL for the next cache operation. The TTL applies to
+ * any keys set by the next call to @{method:getKey} or @{method:getKeys},
+ * and is reset after the call finishes.
+ *
+ * // If this causes any caches to fill, they'll fill with a 15-second TTL.
+ * $stack->setNextTTL(15)->getKey('porcupine');
+ *
+ * // TTL does not persist; this will use no TTL.
+ * $stack->getKey('hedgehog');
+ *
+ * @param int TTL in seconds.
+ * @return this
+ *
+ * @task config
+ */
+ public function setNextTTL($ttl) {
+ $this->nextTTL = $ttl;
+ return $this;
+ }
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function getKeys(array $keys) {
+
+ $remaining = array_fuse($keys);
+ $results = array();
+ $missed = array();
+
+ try {
+ foreach ($this->cachesForward as $cache) {
+ $result = $cache->getKeys($remaining);
+ $remaining = array_diff_key($remaining, $result);
+ $results += $result;
+ if (!$remaining) {
+ while ($cache = array_pop($missed)) {
+ // TODO: This sets too many results in the closer caches, although
+ // it probably isn't a big deal in most cases; normally we're just
+ // filling the request cache.
+ $cache->setKeys($result, $this->nextTTL);
+ }
+ break;
+ }
+ $missed[] = $cache;
+ }
+ $this->nextTTL = null;
+ } catch (Exception $ex) {
+ $this->nextTTL = null;
+ throw $ex;
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->setKeys($keys, $ttl);
+ }
+ }
+
+
+ public function deleteKeys(array $keys) {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->deleteKeys($keys);
+ }
+ }
+
+
+ public function destroyCache() {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->destroyCache();
+ }
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php
@@ -0,0 +1,153 @@
+<?php
+
+/**
+ * @task memcache Managing Memcache
+ */
+final class PhutilMemcacheKeyValueCache extends PhutilKeyValueCache {
+
+ private $servers = array();
+ private $connections = array();
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return function_exists('memcache_pconnect');
+ }
+
+ public function getKeys(array $keys) {
+ $buckets = $this->bucketKeys($keys);
+ $results = array();
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+ $result = $conn->get($bucket_keys);
+ if (!$result) {
+ // If the call fails, treat it as a miss on all keys.
+ $result = array();
+ }
+
+ $results += $result;
+ }
+
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ $buckets = $this->bucketKeys(array_keys($keys));
+
+ // Memcache interprets TTLs as:
+ //
+ // - Seconds from now, for values from 1 to 2592000 (30 days).
+ // - Epoch timestamp, for values larger than 2592000.
+ //
+ // We support only relative TTLs, so convert excessively large relative
+ // TTLs into epoch TTLs.
+ if ($ttl > 2592000) {
+ $effective_ttl = time() + $ttl;
+ } else {
+ $effective_ttl = $ttl;
+ }
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+
+ foreach ($bucket_keys as $key) {
+ $conn->set($key, $keys[$key], 0, $effective_ttl);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ $buckets = $this->bucketKeys($keys);
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+ foreach ($bucket_keys as $key) {
+ $conn->delete($key);
+ }
+ }
+
+ return $this;
+ }
+
+ public function destroyCache() {
+ foreach ($this->servers as $key => $spec) {
+ $this->getConnection($key)->flush();
+ }
+ return $this;
+ }
+
+
+/* -( Managing Memcache )-------------------------------------------------- */
+
+
+ /**
+ * Set available memcache servers. For example:
+ *
+ * $cache->setServers(
+ * array(
+ * array(
+ * 'host' => '10.0.0.20',
+ * 'port' => 11211,
+ * ),
+ * array(
+ * 'host' => '10.0.0.21',
+ * 'port' => 11211,
+ * ),
+ * ));
+ *
+ * @param list<dict> List of server specifications.
+ * @return this
+ * @task memcache
+ */
+ public function setServers(array $servers) {
+ $this->servers = array_values($servers);
+ return $this;
+ }
+
+ private function bucketKeys(array $keys) {
+ $buckets = array();
+ $n = count($this->servers);
+
+ if (!$n) {
+ throw new PhutilInvalidStateException('setServers');
+ }
+
+ foreach ($keys as $key) {
+ $bucket = (int)((crc32($key) & 0x7FFFFFFF) % $n);
+ $buckets[$bucket][] = $key;
+ }
+
+ return $buckets;
+ }
+
+
+ /**
+ * @phutil-external-symbol function memcache_pconnect
+ */
+ private function getConnection($server) {
+ if (empty($this->connections[$server])) {
+ $spec = $this->servers[$server];
+ $host = $spec['host'];
+ $port = $spec['port'];
+
+ $conn = memcache_pconnect($host, $spec['port']);
+
+ if (!$conn) {
+ throw new Exception(
+ pht(
+ 'Unable to connect to memcache server (%s:%d)!',
+ $host,
+ $port));
+ }
+
+ $this->connections[$server] = $conn;
+ }
+ return $this->connections[$server];
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php
@@ -0,0 +1,205 @@
+<?php
+
+/**
+ * Interface to a disk cache. Storage persists across requests.
+ *
+ * This cache is very slow compared to caches like APC. It is intended as a
+ * specialized alternative to APC when APC is not available.
+ *
+ * This is a highly specialized cache and not appropriate for use as a
+ * generalized key-value cache for arbitrary application data.
+ *
+ * Also note that reading and writing keys from the cache currently involves
+ * loading and saving the entire cache, no matter how little data you touch.
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ * @task storage Cache Storage
+ */
+final class PhutilOnDiskKeyValueCache extends PhutilKeyValueCache {
+
+ private $cache = array();
+ private $cacheFile;
+ private $lock;
+ private $wait = 0;
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+
+ /**
+ * Set duration (in seconds) to wait for the file lock.
+ */
+ public function setWait($wait) {
+ $this->wait = $wait;
+ return $this;
+ }
+
+ public function getKeys(array $keys) {
+ $now = time();
+
+ $results = array();
+ $reloaded = false;
+ foreach ($keys as $key) {
+
+ // Try to read the value from cache. If we miss, load (or reload) the
+ // cache.
+
+ while (true) {
+ if (isset($this->cache[$key])) {
+ $val = $this->cache[$key];
+ if (empty($val['ttl']) || $val['ttl'] >= $now) {
+ $results[$key] = $val['val'];
+ break;
+ }
+ }
+
+ if ($reloaded) {
+ break;
+ }
+
+ $this->loadCache($hold_lock = false);
+ $reloaded = true;
+ }
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ if ($ttl) {
+ $ttl_epoch = time() + $ttl;
+ } else {
+ $ttl_epoch = null;
+ }
+
+ $dicts = array();
+ foreach ($keys as $key => $value) {
+ $dict = array(
+ 'val' => $value,
+ );
+ if ($ttl_epoch) {
+ $dict['ttl'] = $ttl_epoch;
+ }
+ $dicts[$key] = $dict;
+ }
+
+ $this->loadCache($hold_lock = true);
+ foreach ($dicts as $key => $dict) {
+ $this->cache[$key] = $dict;
+ }
+ $this->saveCache();
+
+ return $this;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $this->loadCache($hold_lock = true);
+ foreach ($keys as $key) {
+ unset($this->cache[$key]);
+ }
+ $this->saveCache();
+
+ return $this;
+ }
+
+
+ public function destroyCache() {
+ Filesystem::remove($this->getCacheFile());
+ return $this;
+ }
+
+
+/* -( Cache Storage )------------------------------------------------------ */
+
+
+ /**
+ * @task storage
+ */
+ public function setCacheFile($file) {
+ $this->cacheFile = $file;
+ return $this;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function loadCache($hold_lock) {
+ if ($this->lock) {
+ throw new Exception(
+ pht(
+ 'Trying to %s with a lock!',
+ __FUNCTION__.'()'));
+ }
+
+ $lock = PhutilFileLock::newForPath($this->getCacheFile().'.lock');
+ try {
+ $lock->lock($this->wait);
+ } catch (PhutilLockException $ex) {
+ if ($hold_lock) {
+ throw $ex;
+ } else {
+ $this->cache = array();
+ return;
+ }
+ }
+
+ try {
+ $this->cache = array();
+ if (Filesystem::pathExists($this->getCacheFile())) {
+ $cache = unserialize(Filesystem::readFile($this->getCacheFile()));
+ if ($cache) {
+ $this->cache = $cache;
+ }
+ }
+ } catch (Exception $ex) {
+ $lock->unlock();
+ throw $ex;
+ }
+
+ if ($hold_lock) {
+ $this->lock = $lock;
+ } else {
+ $lock->unlock();
+ }
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function saveCache() {
+ if (!$this->lock) {
+ throw new PhutilInvalidStateException('loadCache');
+ }
+
+ // We're holding a lock so we're safe to do a write to a well-known file.
+ // Write to the same directory as the cache so the rename won't imply a
+ // copy across volumes.
+ $new = $this->getCacheFile().'.new';
+ Filesystem::writeFile($new, serialize($this->cache));
+ Filesystem::rename($new, $this->getCacheFile());
+
+ $this->lock->unlock();
+ $this->lock = null;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getCacheFile() {
+ if (!$this->cacheFile) {
+ throw new PhutilInvalidStateException('setCacheFile');
+ }
+ return $this->cacheFile;
+ }
+
+}
diff --git a/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php
@@ -0,0 +1,267 @@
+<?php
+
+final class PhutilKeyValueCacheTestCase extends PhutilTestCase {
+
+ public function testInRequestCache() {
+ $cache = new PhutilInRequestKeyValueCache();
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testInRequestCacheLimit() {
+ $cache = new PhutilInRequestKeyValueCache();
+ $cache->setLimit(4);
+
+ $cache->setKey(1, 1);
+ $cache->setKey(2, 2);
+ $cache->setKey(3, 3);
+ $cache->setKey(4, 4);
+
+ $this->assertEqual(
+ array(
+ 1 => 1,
+ 2 => 2,
+ 3 => 3,
+ 4 => 4,
+ ),
+ $cache->getAllKeys());
+
+
+ $cache->setKey(5, 5);
+
+ $this->assertEqual(
+ array(
+ 2 => 2,
+ 3 => 3,
+ 4 => 4,
+ 5 => 5,
+ ),
+ $cache->getAllKeys());
+ }
+
+ public function testOnDiskCache() {
+ $cache = new PhutilOnDiskKeyValueCache();
+ $cache->setCacheFile(new TempFile());
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testAPCCache() {
+ $cache = new PhutilAPCKeyValueCache();
+ if (!$cache->isAvailable()) {
+ $this->assertSkipped(pht('Cache not available.'));
+ }
+ $this->doCacheTest($cache);
+ }
+
+ public function testDirectoryCache() {
+ $cache = new PhutilDirectoryKeyValueCache();
+
+ $dir = Filesystem::createTemporaryDirectory();
+ $cache->setCacheDirectory($dir);
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testDirectoryCacheSpecialDirectoryRules() {
+ $cache = new PhutilDirectoryKeyValueCache();
+
+ $dir = Filesystem::createTemporaryDirectory();
+ $dir = $dir.'/dircache/';
+ $cache->setCacheDirectory($dir);
+
+ $cache->setKey('a', 1);
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a.cache'));
+
+ $cache->setKey('a/b', 1);
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a/'));
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a/b.cache'));
+
+ $cache->deleteKey('a/b');
+ $this->assertEqual(false, Filesystem::pathExists($dir.'/a/'));
+ $this->assertEqual(false, Filesystem::pathExists($dir.'/a/b.cache'));
+
+ $cache->destroyCache();
+ $this->assertEqual(false, Filesystem::pathExists($dir));
+ }
+
+ public function testNamespaceCache() {
+ $namespace = 'namespace'.mt_rand();
+ $in_request_cache = new PhutilInRequestKeyValueCache();
+ $cache = new PhutilKeyValueCacheNamespace($in_request_cache);
+ $cache->setNamespace($namespace);
+
+ $test_info = get_class($cache);
+ $keys = array(
+ 'key1' => mt_rand(),
+ 'key2' => '',
+ 'key3' => 'Phabricator',
+ );
+ $cache->setKeys($keys);
+ $cached_keys = $in_request_cache->getAllKeys();
+
+ foreach ($keys as $key => $value) {
+ $cached_key = $namespace.':'.$key;
+
+ $this->assertTrue(
+ isset($cached_keys[$cached_key]),
+ $test_info);
+
+ $this->assertEqual(
+ $value,
+ $cached_keys[$cached_key],
+ $test_info);
+ }
+
+ $cache->destroyCache();
+
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testCacheStack() {
+ $req_cache = new PhutilInRequestKeyValueCache();
+ $disk_cache = new PhutilOnDiskKeyValueCache();
+ $disk_cache->setCacheFile(new TempFile());
+ $apc_cache = new PhutilAPCKeyValueCache();
+
+ $stack = array(
+ $req_cache,
+ $disk_cache,
+ );
+
+ if ($apc_cache->isAvailable()) {
+ $stack[] = $apc_cache;
+ }
+
+ $cache = new PhutilKeyValueCacheStack();
+ $cache->setCaches($stack);
+
+ $this->doCacheTest($cache);
+
+ $disk_cache->destroyCache();
+ $req_cache->destroyCache();
+ }
+
+ private function doCacheTest(PhutilKeyValueCache $cache) {
+ $key1 = 'test:'.mt_rand();
+ $key2 = 'test:'.mt_rand();
+
+ $default = 'cache-miss';
+ $value1 = 'cache-hit1';
+ $value2 = 'cache-hit2';
+
+ $test_info = get_class($cache);
+
+ // Test that we miss correctly on missing values.
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can set individual keys.
+
+ $cache->setKey($key1, $value1);
+ $this->assertEqual(
+ $value1,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ $key1 => $value1,
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can delete individual keys.
+
+ $cache->deleteKey($key1);
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+
+ // Test that we can set multiple keys.
+
+ $cache->setKeys(
+ array(
+ $key1 => $value1,
+ $key2 => $value2,
+ ));
+
+ $this->assertEqual(
+ $value1,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ $key1 => $value1,
+ $key2 => $value2,
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can delete multiple keys.
+
+ $cache->deleteKeys(array($key1, $key2));
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // NOTE: The TTL tests are necessarily slow (we must sleep() through the
+ // TTLs) and do not work with APC (it does not TTL until the next request)
+ // so they're disabled by default. If you're developing the cache stack,
+ // it may be useful to run them.
+
+ return;
+
+ // Test that keys expire when they TTL.
+
+ $cache->setKey($key1, $value1, 1);
+ $cache->setKey($key2, $value2, 5);
+
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ $this->assertEqual($value2, $cache->getKey($key2, $default));
+
+ sleep(2);
+
+ $this->assertEqual($default, $cache->getKey($key1, $default));
+ $this->assertEqual($value2, $cache->getKey($key2, $default));
+
+
+ // Test that setting a 0 TTL overwrites a nonzero TTL.
+
+ $cache->setKey($key1, $value1, 1);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ $cache->setKey($key1, $value1, 0);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ sleep(2);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ }
+
+}
diff --git a/src/infrastructure/lipsum/PhutilContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php
@@ -0,0 +1,93 @@
+<?php
+
+/**
+ * Generate nonsense test data according to a context-free grammar definition.
+ */
+abstract class PhutilContextFreeGrammar extends Phobject {
+
+ private $limit = 65535;
+
+ abstract protected function getRules();
+
+ public function generateSeveral($count, $implode = ' ') {
+ $paragraph = array();
+ for ($ii = 0; $ii < $count; $ii++) {
+ $paragraph[$ii] = $this->generate();
+ }
+ return implode($implode, $paragraph);
+ }
+
+ public function generate() {
+ $count = 0;
+ $rules = $this->getRules();
+ return $this->applyRules('[start]', $count, $rules);
+ }
+
+ final protected function applyRules($input, &$count, array $rules) {
+ if (++$count > $this->limit) {
+ throw new Exception(pht('Token replacement count exceeded limit!'));
+ }
+
+ $matches = null;
+ preg_match_all('/(\\[[^\\]]+\\])/', $input, $matches, PREG_OFFSET_CAPTURE);
+
+ foreach (array_reverse($matches[1]) as $token_spec) {
+ list($token, $offset) = $token_spec;
+ $token_name = substr($token, 1, -1);
+ $options = array();
+
+ if (($name_end = strpos($token_name, ','))) {
+ $options_parser = new PhutilSimpleOptions();
+ $options = $options_parser->parse($token_name);
+ $token_name = substr($token_name, 0, $name_end);
+ }
+
+ if (empty($rules[$token_name])) {
+ throw new Exception(pht("Invalid token '%s' in grammar.", $token_name));
+ }
+
+ $key = array_rand($rules[$token_name]);
+ $replacement = $this->applyRules($rules[$token_name][$key],
+ $count, $rules);
+
+ if (isset($options['indent'])) {
+ if (is_numeric($options['indent'])) {
+ $replacement = self::strPadLines($replacement, $options['indent']);
+ } else {
+ $replacement = self::strPadLines($replacement);
+ }
+ }
+ if (isset($options['trim'])) {
+ switch ($options['trim']) {
+ case 'left':
+ $replacement = ltrim($replacement);
+ break;
+ case 'right':
+ $replacement = rtrim($replacement);
+ break;
+ default:
+ case 'both':
+ $replacement = trim($replacement);
+ break;
+ }
+ }
+ if (isset($options['block'])) {
+ $replacement = "\n".$replacement."\n";
+ }
+
+ $input = substr_replace($input, $replacement, $offset, strlen($token));
+ }
+
+ return $input;
+ }
+
+ private static function strPadLines($text, $num_spaces = 2) {
+ $text_lines = phutil_split_lines($text);
+ foreach ($text_lines as $linenr => $line) {
+ $text_lines[$linenr] = str_repeat(' ', $num_spaces).$line;
+ }
+
+ return implode('', $text_lines);
+ }
+
+}
diff --git a/src/infrastructure/markup/PhutilMarkupEngine.php b/src/infrastructure/markup/PhutilMarkupEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilMarkupEngine.php
@@ -0,0 +1,32 @@
+<?php
+
+abstract class PhutilMarkupEngine extends Phobject {
+
+ /**
+ * Set a configuration parameter which the engine can read to customize how
+ * the text is marked up. This is a generic interface; consult the
+ * documentation for specific rules and blocks for what options are available
+ * for configuration.
+ *
+ * @param string Key to set in the configuration dictionary.
+ * @param string Value to set.
+ * @return this
+ */
+ abstract public function setConfig($key, $value);
+
+ /**
+ * After text has been marked up with @{method:markupText}, you can retrieve
+ * any metadata the markup process generated by calling this method. This is
+ * a generic interface that allows rules to export extra information about
+ * text; consult the documentation for specific rules and blocks to see what
+ * metadata may be available in your configuration.
+ *
+ * @param string Key to retrieve from metadata.
+ * @param mixed Default value to return if the key is not available.
+ * @return mixed Metadata property, or default value.
+ */
+ abstract public function getTextMetadata($key, $default = null);
+
+ abstract public function markupText($text);
+
+}
diff --git a/src/infrastructure/markup/PhutilSafeHTML.php b/src/infrastructure/markup/PhutilSafeHTML.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilSafeHTML.php
@@ -0,0 +1,44 @@
+<?php
+
+final class PhutilSafeHTML extends Phobject {
+
+ private $content;
+
+ public function __construct($content) {
+ $this->content = (string)$content;
+ }
+
+ public function __toString() {
+ return $this->content;
+ }
+
+ public function getHTMLContent() {
+ return $this->content;
+ }
+
+ public function appendHTML($html /* , ... */) {
+ foreach (func_get_args() as $html) {
+ $this->content .= phutil_escape_html($html);
+ }
+ return $this;
+ }
+
+ public static function applyFunction($function, $string /* , ... */) {
+ $args = func_get_args();
+ array_shift($args);
+ $args = array_map('phutil_escape_html', $args);
+ return new PhutilSafeHTML(call_user_func_array($function, $args));
+ }
+
+// Requires http://pecl.php.net/operator.
+
+ public function __concat($html) {
+ $clone = clone $this;
+ return $clone->appendHTML($html);
+ }
+
+ public function __assign_concat($html) {
+ return $this->appendHTML($html);
+ }
+
+}
diff --git a/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php
@@ -0,0 +1,12 @@
+<?php
+
+/**
+ * Implement this interface to mark an object as capable of producing a
+ * PhutilSafeHTML representation. This is primarily useful for building
+ * renderable HTML views.
+ */
+interface PhutilSafeHTMLProducerInterface {
+
+ public function producePhutilSafeHTML();
+
+}
diff --git a/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php b/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php
@@ -0,0 +1,223 @@
+<?php
+
+final class PhutilMarkupTestCase extends PhutilTestCase {
+
+ public function testTagDefaults() {
+ $this->assertEqual(
+ (string)phutil_tag('br'),
+ (string)phutil_tag('br', array()));
+
+ $this->assertEqual(
+ (string)phutil_tag('br', array()),
+ (string)phutil_tag('br', array(), null));
+ }
+
+ public function testTagEmpty() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br', array(), null));
+
+ $this->assertEqual(
+ '<div></div>',
+ (string)phutil_tag('div', array(), null));
+
+ $this->assertEqual(
+ '<div></div>',
+ (string)phutil_tag('div', array(), ''));
+ }
+
+ public function testTagBasics() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br'));
+
+ $this->assertEqual(
+ '<div>y</div>',
+ (string)phutil_tag('div', array(), 'y'));
+ }
+
+ public function testTagAttributes() {
+ $this->assertEqual(
+ '<div u="v">y</div>',
+ (string)phutil_tag('div', array('u' => 'v'), 'y'));
+
+ $this->assertEqual(
+ '<br u="v" />',
+ (string)phutil_tag('br', array('u' => 'v')));
+ }
+
+ public function testTagEscapes() {
+ $this->assertEqual(
+ '<br u="&lt;" />',
+ (string)phutil_tag('br', array('u' => '<')));
+
+ $this->assertEqual(
+ '<div><br /></div>',
+ (string)phutil_tag('div', array(), phutil_tag('br')));
+ }
+
+ public function testTagNullAttribute() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br', array('y' => null)));
+ }
+
+ public function testTagJavascriptProtocolRejection() {
+ $hrefs = array(
+ 'javascript:alert(1)' => true,
+ 'JAVASCRIPT:alert(2)' => true,
+
+ // NOTE: When interpreted as a URI, this is dropped because of leading
+ // whitespace.
+ ' javascript:alert(3)' => array(true, false),
+ '/' => false,
+ '/path/to/stuff/' => false,
+ '' => false,
+ 'http://example.com/' => false,
+ '#' => false,
+ 'javascript://anything' => true,
+
+ // Chrome 33 and IE11, at a minimum, treat this as Javascript.
+ "javascript\n:alert(4)" => true,
+
+ // Opera currently accepts a variety of unicode spaces. This test case
+ // has a smattering of them.
+ "\xE2\x80\x89javascript:" => true,
+ "javascript\xE2\x80\x89:" => true,
+ "\xE2\x80\x84javascript:" => true,
+ "javascript\xE2\x80\x84:" => true,
+
+ // Because we're aggressive, all of unicode should trigger detection
+ // by default.
+ "\xE2\x98\x83javascript:" => true,
+ "javascript\xE2\x98\x83:" => true,
+ "\xE2\x98\x83javascript\xE2\x98\x83:" => true,
+
+ // We're aggressive about this, so we'll intentionally raise false
+ // positives in these cases.
+ 'javascript~:alert(5)' => true,
+ '!!!javascript!!!!:alert(6)' => true,
+
+ // However, we should raise true negatives in these slightly more
+ // reasonable cases.
+ 'javascript/:docs.html' => false,
+ 'javascripts:x.png' => false,
+ 'COOLjavascript:page' => false,
+ '/javascript:alert(1)' => false,
+ );
+
+ foreach (array(true, false) as $use_uri) {
+ foreach ($hrefs as $href => $expect) {
+ if (is_array($expect)) {
+ $expect = ($use_uri ? $expect[1] : $expect[0]);
+ }
+
+ if ($use_uri) {
+ $href_value = new PhutilURI($href);
+ } else {
+ $href_value = $href;
+ }
+
+ $caught = null;
+ try {
+ phutil_tag('a', array('href' => $href_value), 'click for candy');
+ } catch (Exception $ex) {
+ $caught = $ex;
+ }
+
+ $desc = pht(
+ 'Unexpected result for "%s". <uri = %s, expect exception = %s>',
+ $href,
+ $use_uri ? pht('Yes') : pht('No'),
+ $expect ? pht('Yes') : pht('No'));
+
+ $this->assertEqual(
+ $expect,
+ $caught instanceof Exception,
+ $desc);
+ }
+ }
+ }
+
+ public function testURIEscape() {
+ $this->assertEqual(
+ '%2B/%20%3F%23%26%3A%21xyz%25',
+ phutil_escape_uri('+/ ?#&:!xyz%'));
+ }
+
+ public function testURIPathComponentEscape() {
+ $this->assertEqual(
+ 'a%252Fb',
+ phutil_escape_uri_path_component('a/b'));
+
+ $str = '';
+ for ($ii = 0; $ii <= 255; $ii++) {
+ $str .= chr($ii);
+ }
+
+ $this->assertEqual(
+ $str,
+ phutil_unescape_uri_path_component(
+ rawurldecode( // Simulates webserver.
+ phutil_escape_uri_path_component($str))));
+ }
+
+ public function testHsprintf() {
+ $this->assertEqual(
+ '<div>&lt;3</div>',
+ (string)hsprintf('<div>%s</div>', '<3'));
+ }
+
+ public function testAppendHTML() {
+ $html = phutil_tag('hr');
+ $html->appendHTML(phutil_tag('br'), '<evil>');
+ $this->assertEqual('<hr /><br />&lt;evil&gt;', $html->getHTMLContent());
+ }
+
+ public function testArrayEscaping() {
+ $this->assertEqual(
+ '<div>&lt;div&gt;</div>',
+ phutil_escape_html(
+ array(
+ hsprintf('<div>'),
+ array(
+ array(
+ '<',
+ array(
+ 'd',
+ array(
+ array(
+ hsprintf('i'),
+ ),
+ 'v',
+ ),
+ ),
+ array(
+ array(
+ '>',
+ ),
+ ),
+ ),
+ ),
+ hsprintf('</div>'),
+ )));
+
+ $this->assertEqual(
+ '<div><br /><hr /><wbr /></div>',
+ phutil_tag(
+ 'div',
+ array(),
+ array(
+ array(
+ array(
+ phutil_tag('br'),
+ array(
+ phutil_tag('hr'),
+ ),
+ phutil_tag('wbr'),
+ ),
+ ),
+ ))->getHTMLContent());
+ }
+
+}
diff --git a/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php
@@ -0,0 +1,19 @@
+<?php
+
+final class PhutilSafeHTMLTestCase extends PhutilTestCase {
+
+ public function testOperator() {
+ if (!extension_loaded('operator')) {
+ $this->assertSkipped(pht('Operator extension not available.'));
+ }
+
+ $a = phutil_tag('a');
+ $ab = $a.phutil_tag('b');
+ $this->assertEqual('<a></a><b></b>', $ab->getHTMLContent());
+ $this->assertEqual('<a></a>', $a->getHTMLContent());
+
+ $a .= phutil_tag('a');
+ $this->assertEqual('<a></a><a></a>', $a->getHTMLContent());
+ }
+
+}
diff --git a/src/infrastructure/markup/render.php b/src/infrastructure/markup/render.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/render.php
@@ -0,0 +1,251 @@
+<?php
+
+/**
+ * Render an HTML tag in a way that treats user content as unsafe by default.
+ *
+ * Tag rendering has some special logic which implements security features:
+ *
+ * - When rendering `<a>` tags, if the `rel` attribute is not specified, it
+ * is interpreted as `rel="noreferrer"`.
+ * - When rendering `<a>` tags, the `href` attribute may not begin with
+ * `javascript:`.
+ *
+ * These special cases can not be disabled.
+ *
+ * IMPORTANT: The `$tag` attribute and the keys of the `$attributes` array are
+ * trusted blindly, and not escaped. You should not pass user data in these
+ * parameters.
+ *
+ * @param string The name of the tag, like `a` or `div`.
+ * @param map<string, string> A map of tag attributes.
+ * @param wild Content to put in the tag.
+ * @return PhutilSafeHTML Tag object.
+ */
+function phutil_tag($tag, array $attributes = array(), $content = null) {
+ // If the `href` attribute is present, make sure it is not a "javascript:"
+ // URI. We never permit these.
+ if (!empty($attributes['href'])) {
+ // This might be a URI object, so cast it to a string.
+ $href = (string)$attributes['href'];
+
+ if (isset($href[0])) {
+ // Block 'javascript:' hrefs at the tag level: no well-designed
+ // application should ever use them, and they are a potent attack vector.
+
+ // This function is deep in the core and performance sensitive, so we're
+ // doing a cheap version of this test first to avoid calling preg_match()
+ // on URIs which begin with '/' or `#`. These cover essentially all URIs
+ // in Phabricator.
+ if (($href[0] !== '/') && ($href[0] !== '#')) {
+ // Chrome 33 and IE 11 both interpret "javascript\n:" as a Javascript
+ // URI, and all browsers interpret " javascript:" as a Javascript URI,
+ // so be aggressive about looking for "javascript:" in the initial
+ // section of the string.
+
+ $normalized_href = preg_replace('([^a-z0-9/:]+)i', '', $href);
+ if (preg_match('/^javascript:/i', $normalized_href)) {
+ throw new Exception(
+ pht(
+ "Attempting to render a tag with an '%s' attribute that begins ".
+ "with '%s'. This is either a serious security concern or a ".
+ "serious architecture concern. Seek urgent remedy.",
+ 'href',
+ 'javascript:'));
+ }
+ }
+ }
+ }
+
+ // For tags which can't self-close, treat null as the empty string -- for
+ // example, always render `<div></div>`, never `<div />`.
+ static $self_closing_tags = array(
+ 'area' => true,
+ 'base' => true,
+ 'br' => true,
+ 'col' => true,
+ 'command' => true,
+ 'embed' => true,
+ 'frame' => true,
+ 'hr' => true,
+ 'img' => true,
+ 'input' => true,
+ 'keygen' => true,
+ 'link' => true,
+ 'meta' => true,
+ 'param' => true,
+ 'source' => true,
+ 'track' => true,
+ 'wbr' => true,
+ );
+
+ $attr_string = '';
+ foreach ($attributes as $k => $v) {
+ if ($v === null) {
+ continue;
+ }
+ $v = phutil_escape_html($v);
+ $attr_string .= ' '.$k.'="'.$v.'"';
+ }
+
+ if ($content === null) {
+ if (isset($self_closing_tags[$tag])) {
+ return new PhutilSafeHTML('<'.$tag.$attr_string.' />');
+ } else {
+ $content = '';
+ }
+ } else {
+ $content = phutil_escape_html($content);
+ }
+
+ return new PhutilSafeHTML('<'.$tag.$attr_string.'>'.$content.'</'.$tag.'>');
+}
+
+function phutil_tag_div($class, $content = null) {
+ return phutil_tag('div', array('class' => $class), $content);
+}
+
+function phutil_escape_html($string) {
+ if ($string instanceof PhutilSafeHTML) {
+ return $string;
+ } else if ($string instanceof PhutilSafeHTMLProducerInterface) {
+ $result = $string->producePhutilSafeHTML();
+ if ($result instanceof PhutilSafeHTML) {
+ return phutil_escape_html($result);
+ } else if (is_array($result)) {
+ return phutil_escape_html($result);
+ } else if ($result instanceof PhutilSafeHTMLProducerInterface) {
+ return phutil_escape_html($result);
+ } else {
+ try {
+ assert_stringlike($result);
+ return phutil_escape_html((string)$result);
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ "Object (of class '%s') implements %s but did not return anything ".
+ "renderable from %s.",
+ get_class($string),
+ 'PhutilSafeHTMLProducerInterface',
+ 'producePhutilSafeHTML()'));
+ }
+ }
+ } else if (is_array($string)) {
+ $result = '';
+ foreach ($string as $item) {
+ $result .= phutil_escape_html($item);
+ }
+ return $result;
+ }
+
+ return htmlspecialchars($string, ENT_QUOTES, 'UTF-8');
+}
+
+function phutil_escape_html_newlines($string) {
+ return PhutilSafeHTML::applyFunction('nl2br', $string);
+}
+
+/**
+ * Mark string as safe for use in HTML.
+ */
+function phutil_safe_html($string) {
+ if ($string == '') {
+ return $string;
+ } else if ($string instanceof PhutilSafeHTML) {
+ return $string;
+ } else {
+ return new PhutilSafeHTML($string);
+ }
+}
+
+/**
+ * HTML safe version of `implode()`.
+ */
+function phutil_implode_html($glue, array $pieces) {
+ $glue = phutil_escape_html($glue);
+
+ foreach ($pieces as $k => $piece) {
+ $pieces[$k] = phutil_escape_html($piece);
+ }
+
+ return phutil_safe_html(implode($glue, $pieces));
+}
+
+/**
+ * Format a HTML code. This function behaves like `sprintf()`, except that all
+ * the normal conversions (like %s) will be properly escaped.
+ */
+function hsprintf($html /* , ... */) {
+ $args = func_get_args();
+ array_shift($args);
+ return new PhutilSafeHTML(
+ vsprintf($html, array_map('phutil_escape_html', $args)));
+}
+
+
+/**
+ * Escape text for inclusion in a URI or a query parameter. Note that this
+ * method does NOT escape '/', because "%2F" is invalid in paths and Apache
+ * will automatically 404 the page if it's present. This will produce correct
+ * (the URIs will work) and desirable (the URIs will be readable) behavior in
+ * these cases:
+ *
+ * '/path/?param='.phutil_escape_uri($string); # OK: Query Parameter
+ * '/path/to/'.phutil_escape_uri($string); # OK: URI Suffix
+ *
+ * It will potentially produce the WRONG behavior in this special case:
+ *
+ * COUNTEREXAMPLE
+ * '/path/to/'.phutil_escape_uri($string).'/thing/'; # BAD: URI Infix
+ *
+ * In this case, any '/' characters in the string will not be escaped, so you
+ * will not be able to distinguish between the string and the suffix (unless
+ * you have more information, like you know the format of the suffix). For infix
+ * URI components, use @{function:phutil_escape_uri_path_component} instead.
+ *
+ * @param string Some string.
+ * @return string URI encoded string, except for '/'.
+ */
+function phutil_escape_uri($string) {
+ return str_replace('%2F', '/', rawurlencode($string));
+}
+
+
+/**
+ * Escape text for inclusion as an infix URI substring. See discussion at
+ * @{function:phutil_escape_uri}. This function covers an unusual special case;
+ * @{function:phutil_escape_uri} is usually the correct function to use.
+ *
+ * This function will escape a string into a format which is safe to put into
+ * a URI path and which does not contain '/' so it can be correctly parsed when
+ * embedded as a URI infix component.
+ *
+ * However, you MUST decode the string with
+ * @{function:phutil_unescape_uri_path_component} before it can be used in the
+ * application.
+ *
+ * @param string Some string.
+ * @return string URI encoded string that is safe for infix composition.
+ */
+function phutil_escape_uri_path_component($string) {
+ return rawurlencode(rawurlencode($string));
+}
+
+
+/**
+ * Unescape text that was escaped by
+ * @{function:phutil_escape_uri_path_component}. See
+ * @{function:phutil_escape_uri} for discussion.
+ *
+ * Note that this function is NOT the inverse of
+ * @{function:phutil_escape_uri_path_component}! It undoes additional escaping
+ * which is added to survive the implied unescaping performed by the webserver
+ * when interpreting the request.
+ *
+ * @param string Some string emitted
+ * from @{function:phutil_escape_uri_path_component} and
+ * then accessed via a web server.
+ * @return string Original string.
+ */
+function phutil_unescape_uri_path_component($string) {
+ return rawurldecode($string);
+}
diff --git a/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php
@@ -0,0 +1,115 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighterEngine
+ extends PhutilSyntaxHighlighterEngine {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getLanguageFromFilename($filename) {
+ static $default_map = array(
+ // All files which have file extensions that we haven't already matched
+ // map to their extensions.
+ '@\\.([^./]+)$@' => 1,
+ );
+
+ $maps = array();
+ if (!empty($this->config['filename.map'])) {
+ $maps[] = $this->config['filename.map'];
+ }
+ $maps[] = $default_map;
+
+ foreach ($maps as $map) {
+ foreach ($map as $regexp => $lang) {
+ $matches = null;
+ if (preg_match($regexp, $filename, $matches)) {
+ if (is_numeric($lang)) {
+ return idx($matches, $lang);
+ } else {
+ return $lang;
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public function getHighlightFuture($language, $source) {
+ if ($language === null) {
+ $language = PhutilLanguageGuesser::guessLanguage($source);
+ }
+
+ $have_pygments = !empty($this->config['pygments.enabled']);
+
+ if ($language == 'php' && PhutilXHPASTBinary::isAvailable()) {
+ return id(new PhutilXHPASTSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'console') {
+ return id(new PhutilConsoleSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'diviner' || $language == 'remarkup') {
+ return id(new PhutilDivinerSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'rainbow') {
+ return id(new PhutilRainbowSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'php') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPHPFragmentLexer())
+ ->setConfig('language', 'php')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'py' || $language == 'python') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPythonFragmentLexer())
+ ->setConfig('language', 'py')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'java') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilJavaFragmentLexer())
+ ->setConfig('language', 'java')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'json') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilJSONFragmentLexer())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'invisible') {
+ return id(new PhutilInvisibleSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ // Don't invoke Pygments for plain text, since it's expensive and has
+ // no effect.
+ if ($language !== 'text' && $language !== 'txt') {
+ if ($have_pygments) {
+ return id(new PhutilPygmentsSyntaxHighlighter())
+ ->setConfig('language', $language)
+ ->getHighlightFuture($source);
+ }
+ }
+
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php
@@ -0,0 +1,19 @@
+<?php
+
+abstract class PhutilSyntaxHighlighterEngine extends Phobject {
+
+ abstract public function setConfig($key, $value);
+ abstract public function getHighlightFuture($language, $source);
+ abstract public function getLanguageFromFilename($filename);
+
+ final public function highlightSource($language, $source) {
+ try {
+ return $this->getHighlightFuture($language, $source)->resolve();
+ } catch (PhutilSyntaxHighlighterException $ex) {
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source)
+ ->resolve();
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php
@@ -0,0 +1,28 @@
+<?php
+
+/**
+ * Test cases for @{class:PhutilDefaultSyntaxHighlighterEngine}.
+ */
+final class PhutilDefaultSyntaxHighlighterEngineTestCase
+ extends PhutilTestCase {
+
+ public function testFilenameGreediness() {
+ $names = array(
+ 'x.php' => 'php',
+ '/x.php' => 'php',
+ 'x.y.php' => 'php',
+ '/x.y/z.php' => 'php',
+ '/x.php/' => null,
+ );
+
+ $engine = new PhutilDefaultSyntaxHighlighterEngine();
+ foreach ($names as $path => $language) {
+ $detect = $engine->getLanguageFromFilename($path);
+ $this->assertEqual(
+ $language,
+ $detect,
+ pht('Language detect for %s', $path));
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php
@@ -0,0 +1,51 @@
+<?php
+
+/**
+ * Simple syntax highlighter for console output. We just try to highlight the
+ * commands so it's easier to follow transcripts.
+ */
+final class PhutilConsoleSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $in_command = false;
+ $lines = explode("\n", $source);
+ foreach ($lines as $key => $line) {
+ $matches = null;
+
+ // Parse commands like this:
+ //
+ // some/path/ $ ./bin/example # Do things
+ //
+ // ...into path, command, and comment components.
+
+ $pattern =
+ '@'.
+ ($in_command ? '()(.*?)' : '^(\S+[\\\\/] )?([$] .*?)').
+ '(#.*|\\\\)?$@';
+
+ if (preg_match($pattern, $line, $matches)) {
+ $lines[$key] = hsprintf(
+ '%s<span class="gp">%s</span>%s',
+ $matches[1],
+ $matches[2],
+ (!empty($matches[3])
+ ? hsprintf('<span class="k">%s</span>', $matches[3])
+ : ''));
+ $in_command = (idx($matches, 3) == '\\');
+ } else {
+ $lines[$key] = hsprintf('<span class="go">%s</span>', $line);
+ }
+ }
+ $lines = phutil_implode_html("\n", $lines);
+
+ return new ImmediateFuture($lines);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php
@@ -0,0 +1,14 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighter extends Phobject {
+
+ public function setConfig($key, $value) {
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $result = hsprintf('%s', $source);
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php
@@ -0,0 +1,81 @@
+<?php
+
+/**
+ * Simple syntax highlighter for the ".diviner" format, which is just Remarkup
+ * with a specific ruleset. This should also work alright for Remarkup.
+ */
+final class PhutilDivinerSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+ private $replaceClass;
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $source = phutil_escape_html($source);
+
+ // This highlighter isn't perfect but tries to do an okay job at getting
+ // some of the basics at least. There's lots of room for improvement.
+
+ $blocks = explode("\n\n", $source);
+ foreach ($blocks as $key => $block) {
+ if (preg_match('/^[^ ](?! )/m', $block)) {
+ $blocks[$key] = $this->highlightBlock($block);
+ }
+ }
+ $source = implode("\n\n", $blocks);
+
+ $source = phutil_safe_html($source);
+ return new ImmediateFuture($source);
+ }
+
+ private function highlightBlock($source) {
+ // Highlight "@{class:...}" links to other documentation pages.
+ $source = $this->highlightPattern('/@{([\w@]+?):([^}]+?)}/', $source, 'nc');
+
+ // Highlight "@title", "@group", etc.
+ $source = $this->highlightPattern('/^@(\w+)/m', $source, 'k');
+
+ // Highlight bold, italic and monospace.
+ $source = $this->highlightPattern('@\\*\\*(.+?)\\*\\*@s', $source, 's');
+ $source = $this->highlightPattern('@(?<!:)//(.+?)//@s', $source, 's');
+ $source = $this->highlightPattern(
+ '@##([\s\S]+?)##|\B`(.+?)`\B@',
+ $source,
+ 's');
+
+ // Highlight stuff that looks like headers.
+ $source = $this->highlightPattern('/^=(.*)$/m', $source, 'nv');
+
+ return $source;
+ }
+
+ private function highlightPattern($regexp, $source, $class) {
+ $this->replaceClass = $class;
+ $source = preg_replace_callback(
+ $regexp,
+ array($this, 'replacePattern'),
+ $source);
+
+ return $source;
+ }
+
+ public function replacePattern($matches) {
+
+ // NOTE: The goal here is to make sure a <span> never crosses a newline.
+
+ $content = $matches[0];
+ $content = explode("\n", $content);
+ foreach ($content as $key => $line) {
+ $content[$key] =
+ '<span class="'.$this->replaceClass.'">'.
+ $line.
+ '</span>';
+ }
+ return implode("\n", $content);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php
@@ -0,0 +1,43 @@
+<?php
+
+final class PhutilInvisibleSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $keys = array_map('chr', range(0x0, 0x1F));
+ $vals = array_map(
+ array($this, 'decimalToHtmlEntityDecoded'), range(0x2400, 0x241F));
+
+ $invisible = array_combine($keys, $vals);
+
+ $result = array();
+ foreach (str_split($source) as $character) {
+ if (isset($invisible[$character])) {
+ $result[] = phutil_tag(
+ 'span',
+ array('class' => 'invisible'),
+ $invisible[$character]);
+
+ if ($character === "\n") {
+ $result[] = $character;
+ }
+ } else {
+ $result[] = $character;
+ }
+ }
+
+ $result = phutil_implode_html('', $result);
+ return new ImmediateFuture($result);
+ }
+
+ private function decimalToHtmlEntityDecoded($dec) {
+ return html_entity_decode("&#{$dec};");
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php
@@ -0,0 +1,72 @@
+<?php
+
+final class PhutilLexerSyntaxHighlighter extends PhutilSyntaxHighlighter {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $strip = false;
+ $state = 'start';
+ $lang = idx($this->config, 'language');
+
+ if ($lang == 'php') {
+ if (strpos($source, '<?') === false) {
+ $state = 'php';
+ }
+ }
+
+ $lexer = idx($this->config, 'lexer');
+ $tokens = $lexer->getTokens($source, $state);
+ $tokens = $lexer->mergeTokens($tokens);
+
+ $result = array();
+ foreach ($tokens as $token) {
+ list($type, $value, $context) = $token;
+
+ $data_name = null;
+ switch ($type) {
+ case 'nc':
+ case 'nf':
+ case 'na':
+ $data_name = $value;
+ break;
+ }
+
+ if (strpos($value, "\n") !== false) {
+ $value = explode("\n", $value);
+ } else {
+ $value = array($value);
+ }
+ foreach ($value as $part) {
+ if (strlen($part)) {
+ if ($type) {
+ $result[] = phutil_tag(
+ 'span',
+ array(
+ 'class' => $type,
+ 'data-symbol-context' => $context,
+ 'data-symbol-name' => $data_name,
+ ),
+ $part);
+ } else {
+ $result[] = $part;
+ }
+ }
+ $result[] = "\n";
+ }
+
+ // Throw away the last "\n".
+ array_pop($result);
+ }
+
+ $result = phutil_implode_html('', $result);
+
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php
@@ -0,0 +1,229 @@
+<?php
+
+final class PhutilPygmentsSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $language = idx($this->config, 'language');
+
+ if (preg_match('/\r(?!\n)/', $source)) {
+ // TODO: Pygments converts "\r" newlines into "\n" newlines, so we can't
+ // use it on files with "\r" newlines. If we have "\r" not followed by
+ // "\n" in the file, skip highlighting.
+ $language = null;
+ }
+
+ if ($language) {
+ $language = $this->getPygmentsLexerNameFromLanguageName($language);
+
+ // See T13224. Under Ubuntu, avoid leaving an intermedite "dash" shell
+ // process so we hit "pygmentize" directly if we have to SIGKILL this
+ // because it explodes.
+
+ $future = new ExecFuture(
+ 'exec pygmentize -O encoding=utf-8 -O stripnl=False -f html -l %s',
+ $language);
+
+ $scrub = false;
+ if ($language == 'php' && strpos($source, '<?') === false) {
+ $source = "<?php\n".$source;
+ $scrub = true;
+ }
+
+ // See T13224. In some cases, "pygmentize" has explosive runtime on small
+ // inputs. Put a hard cap on how long it is allowed to run for to limit
+ // the amount of damage it can do.
+ $future->setTimeout(15);
+
+ $future->write($source);
+
+ return new PhutilDefaultSyntaxHighlighterEnginePygmentsFuture(
+ $future,
+ $source,
+ $scrub);
+ }
+
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ private function getPygmentsLexerNameFromLanguageName($language) {
+ static $map = array(
+ 'adb' => 'ada',
+ 'ads' => 'ada',
+ 'ahkl' => 'ahk',
+ 'as' => 'as3',
+ 'asax' => 'aspx-vb',
+ 'ascx' => 'aspx-vb',
+ 'ashx' => 'aspx-vb',
+ 'ASM' => 'nasm',
+ 'asm' => 'nasm',
+ 'asmx' => 'aspx-vb',
+ 'aspx' => 'aspx-vb',
+ 'autodelegate' => 'myghty',
+ 'autohandler' => 'mason',
+ 'aux' => 'tex',
+ 'axd' => 'aspx-vb',
+ 'b' => 'brainfuck',
+ 'bas' => 'vb.net',
+ 'bf' => 'brainfuck',
+ 'bmx' => 'blitzmax',
+ 'c++' => 'cpp',
+ 'c++-objdump' => 'cpp-objdump',
+ 'cc' => 'cpp',
+ 'cfc' => 'cfm',
+ 'cfg' => 'ini',
+ 'cfml' => 'cfm',
+ 'cl' => 'common-lisp',
+ 'clj' => 'clojure',
+ 'cmd' => 'bat',
+ 'coffee' => 'coffee-script',
+ 'cs' => 'csharp',
+ 'csh' => 'tcsh',
+ 'cw' => 'redcode',
+ 'cxx' => 'cpp',
+ 'cxx-objdump' => 'cpp-objdump',
+ 'darcspatch' => 'dpatch',
+ 'def' => 'modula2',
+ 'dhandler' => 'mason',
+ 'di' => 'd',
+ 'duby' => 'rb',
+ 'dyl' => 'dylan',
+ 'ebuild' => 'bash',
+ 'eclass' => 'bash',
+ 'el' => 'common-lisp',
+ 'eps' => 'postscript',
+ 'erl' => 'erlang',
+ 'erl-sh' => 'erl',
+ 'f' => 'fortran',
+ 'f90' => 'fortran',
+ 'feature' => 'Cucumber',
+ 'fhtml' => 'velocity',
+ 'flx' => 'felix',
+ 'flxh' => 'felix',
+ 'frag' => 'glsl',
+ 'g' => 'antlr-ruby',
+ 'G' => 'antlr-ruby',
+ 'gdc' => 'gooddata-cl',
+ 'gemspec' => 'rb',
+ 'geo' => 'glsl',
+ 'GNUmakefile' => 'make',
+ 'h' => 'c',
+ 'h++' => 'cpp',
+ 'hh' => 'cpp',
+ 'hpp' => 'cpp',
+ 'hql' => 'sql',
+ 'hrl' => 'erlang',
+ 'hs' => 'haskell',
+ 'htaccess' => 'apacheconf',
+ 'htm' => 'html',
+ 'html' => 'html+evoque',
+ 'hxx' => 'cpp',
+ 'hy' => 'hybris',
+ 'hyb' => 'hybris',
+ 'ik' => 'ioke',
+ 'inc' => 'pov',
+ 'j' => 'objective-j',
+ 'jbst' => 'duel',
+ 'kid' => 'genshi',
+ 'ksh' => 'bash',
+ 'less' => 'css',
+ 'lgt' => 'logtalk',
+ 'lisp' => 'common-lisp',
+ 'll' => 'llvm',
+ 'm' => 'objective-c',
+ 'mak' => 'make',
+ 'Makefile' => 'make',
+ 'makefile' => 'make',
+ 'man' => 'groff',
+ 'mao' => 'mako',
+ 'mc' => 'mason',
+ 'md' => 'minid',
+ 'mhtml' => 'mason',
+ 'mi' => 'mason',
+ 'ml' => 'ocaml',
+ 'mli' => 'ocaml',
+ 'mll' => 'ocaml',
+ 'mly' => 'ocaml',
+ 'mm' => 'objective-c',
+ 'mo' => 'modelica',
+ 'mod' => 'modula2',
+ 'moo' => 'moocode',
+ 'mu' => 'mupad',
+ 'myt' => 'myghty',
+ 'ns2' => 'newspeak',
+ 'pas' => 'delphi',
+ 'patch' => 'diff',
+ 'phtml' => 'html+php',
+ 'pl' => 'prolog',
+ 'plot' => 'gnuplot',
+ 'plt' => 'gnuplot',
+ 'pm' => 'perl',
+ 'po' => 'pot',
+ 'pp' => 'puppet',
+ 'pro' => 'prolog',
+ 'proto' => 'protobuf',
+ 'ps' => 'postscript',
+ 'pxd' => 'cython',
+ 'pxi' => 'cython',
+ 'py' => 'python',
+ 'pyw' => 'python',
+ 'pyx' => 'cython',
+ 'R' => 'splus',
+ 'r' => 'rebol',
+ 'r3' => 'rebol',
+ 'rake' => 'rb',
+ 'Rakefile' => 'rb',
+ 'rbw' => 'rb',
+ 'rbx' => 'rb',
+ 'rest' => 'rst',
+ 'rl' => 'ragel-em',
+ 'robot' => 'robotframework',
+ 'Rout' => 'rconsole',
+ 'rss' => 'xml',
+ 's' => 'gas',
+ 'S' => 'splus',
+ 'sc' => 'python',
+ 'scm' => 'scheme',
+ 'SConscript' => 'python',
+ 'SConstruct' => 'python',
+ 'scss' => 'css',
+ 'sh' => 'bash',
+ 'sh-session' => 'console',
+ 'spt' => 'cheetah',
+ 'sqlite3-console' => 'sqlite3',
+ 'st' => 'smalltalk',
+ 'sv' => 'v',
+ 'tac' => 'python',
+ 'tmpl' => 'cheetah',
+ 'toc' => 'tex',
+ 'tpl' => 'smarty',
+ 'txt' => 'text',
+ 'vapi' => 'vala',
+ 'vb' => 'vb.net',
+ 'vert' => 'glsl',
+ 'vhd' => 'vhdl',
+ 'vimrc' => 'vim',
+ 'vm' => 'velocity',
+ 'weechatlog' => 'irc',
+ 'wlua' => 'lua',
+ 'wsdl' => 'xml',
+ 'xhtml' => 'html',
+ 'xml' => 'xml+evoque',
+ 'xqy' => 'xquery',
+ 'xsd' => 'xml',
+ 'xsl' => 'xslt',
+ 'xslt' => 'xml',
+ 'yml' => 'yaml',
+ );
+
+ return idx($map, $language, $language);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php
@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Highlights source code with a rainbow of colors, regardless of the language.
+ * This highlighter is useless, absurd, and extremely slow.
+ */
+final class PhutilRainbowSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+
+ $color = 0;
+ $colors = array(
+ 'rbw_r',
+ 'rbw_o',
+ 'rbw_y',
+ 'rbw_g',
+ 'rbw_b',
+ 'rbw_i',
+ 'rbw_v',
+ );
+
+ $result = array();
+ foreach (phutil_utf8v($source) as $character) {
+ if ($character == ' ' || $character == "\n") {
+ $result[] = $character;
+ continue;
+ }
+ $result[] = phutil_tag(
+ 'span',
+ array('class' => $colors[$color]),
+ $character);
+ $color = ($color + 1) % count($colors);
+ }
+
+ $result = phutil_implode_html('', $result);
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php
@@ -0,0 +1,6 @@
+<?php
+
+abstract class PhutilSyntaxHighlighter extends Phobject {
+ abstract public function setConfig($key, $value);
+ abstract public function getHighlightFuture($source);
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php
@@ -0,0 +1,3 @@
+<?php
+
+final class PhutilSyntaxHighlighterException extends Exception {}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php
@@ -0,0 +1,18 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighter extends Phobject {
+
+ public function getHighlightFuture($source) {
+ $scrub = false;
+ if (strpos($source, '<?') === false) {
+ $source = "<?php\n".$source;
+ $scrub = true;
+ }
+
+ return new PhutilXHPASTSyntaxHighlighterFuture(
+ PhutilXHPASTBinary::getParserFuture($source),
+ $source,
+ $scrub);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php
@@ -0,0 +1,24 @@
+<?php
+
+final class PhutilJSONFragmentLexerHighlighterTestCase extends PhutilTestCase {
+
+ public function testLexer() {
+ $highlighter = id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('language', 'json')
+ ->setConfig('lexer', new PhutilJSONFragmentLexer());
+
+ $path = dirname(__FILE__).'/data/jsonfragment/';
+ foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) {
+ if (preg_match('/.test$/', $f)) {
+ $expect = preg_replace('/.test$/', '.expect', $f);
+ $source = Filesystem::readFile($path.'/'.$f);
+
+ $this->assertEqual(
+ Filesystem::readFile($path.'/'.$expect),
+ (string)$highlighter->getHighlightFuture($source)->resolve(),
+ $f);
+ }
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php
@@ -0,0 +1,25 @@
+<?php
+
+final class PhutilPHPFragmentLexerHighlighterTestCase extends PhutilTestCase {
+
+ public function testLexer() {
+ $highlighter = new PhutilLexerSyntaxHighlighter();
+ $highlighter->setConfig('language', 'php');
+ $highlighter->setConfig('lexer', new PhutilPHPFragmentLexer());
+
+
+ $path = dirname(__FILE__).'/phpfragment/';
+ foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) {
+ if (preg_match('/.test$/', $f)) {
+ $expect = preg_replace('/.test$/', '.expect', $f);
+ $source = Filesystem::readFile($path.'/'.$f);
+
+ $this->assertEqual(
+ Filesystem::readFile($path.'/'.$expect),
+ (string)$highlighter->getHighlightFuture($source)->resolve(),
+ $f);
+ }
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php
@@ -0,0 +1,39 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighterTestCase extends PhutilTestCase {
+
+ private function highlight($source) {
+ $highlighter = new PhutilXHPASTSyntaxHighlighter();
+ $future = $highlighter->getHighlightFuture($source);
+ return $future->resolve();
+ }
+
+ private function read($file) {
+ $path = dirname(__FILE__).'/xhpast/'.$file;
+ return Filesystem::readFile($path);
+ }
+
+ public function testBuiltinClassnames() {
+ $this->assertEqual(
+ $this->read('builtin-classname.expect'),
+ (string)$this->highlight($this->read('builtin-classname.source')),
+ pht('Builtin classnames should not be marked as linkable symbols.'));
+ $this->assertEqual(
+ rtrim($this->read('trailing-comment.expect')),
+ (string)$this->highlight($this->read('trailing-comment.source')),
+ pht('Trailing comments should not be dropped.'));
+ $this->assertEqual(
+ $this->read('multiline-token.expect'),
+ (string)$this->highlight($this->read('multiline-token.source')),
+ pht('Multi-line tokens should be split across lines.'));
+ $this->assertEqual(
+ $this->read('leading-whitespace.expect'),
+ (string)$this->highlight($this->read('leading-whitespace.source')),
+ pht('Snippets with leading whitespace should be preserved.'));
+ $this->assertEqual(
+ $this->read('no-leading-whitespace.expect'),
+ (string)$this->highlight($this->read('no-leading-whitespace.source')),
+ pht('Snippets with no leading whitespace should be preserved.'));
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect
@@ -0,0 +1,12 @@
+<span class="o">{</span>
+ <span class="s">&quot;key&quot;</span><span class="o">:</span> <span class="mf">3.5</span><span class="o">,</span>
+ <span class="s">&quot;true&quot;</span><span class="o">:</span> <span class="k">true</span><span class="o">,</span>
+ <span class="s">&quot;false&quot;</span><span class="o">:</span> <span class="k">false</span><span class="o">,</span>
+ <span class="s">&quot;null&quot;</span><span class="o">:</span> <span class="k">null</span><span class="o">,</span>
+ <span class="s">&quot;list&quot;</span><span class="o">:</span> <span class="o">[</span><span class="mf">1</span><span class="o">,</span> <span class="mf">2</span><span class="o">,</span> <span class="mf">3</span><span class="o">],</span>
+ <span class="s">&quot;object&quot;</span><span class="o">:</span> <span class="o">{</span>
+ <span class="s">&quot;k1&quot;</span><span class="o">:</span> <span class="s">&quot;v1&quot;</span>
+ <span class="o">},</span>
+ <span class="s">&quot;numbers&quot;</span><span class="o">:</span> <span class="o">[</span><span class="mf">0</span>e<span class="mf">1</span><span class="o">,</span> <span class="mf">1</span>e<span class="mf">-1</span><span class="o">,</span> <span class="mf">-1</span>e<span class="mf">-1</span><span class="o">,</span> <span class="mf">-1</span>e+<span class="mf">1</span><span class="o">],</span>
+ <span class="s">&quot;</span><span class="k">\&quot;\u1234</span><span class="s">&#039;abc[]{}...&quot;</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test
@@ -0,0 +1,12 @@
+{
+ "key": 3.5,
+ "true": true,
+ "false": false,
+ "null": null,
+ "list": [1, 2, 3],
+ "object": {
+ "k1": "v1"
+ },
+ "numbers": [0e1, 1e-1, -1e-1, -1e+1],
+ "\"\u1234'abc[]{}..."
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect
@@ -0,0 +1,16 @@
+<span class="cp">&lt;?</span>
+
+<span class="c">// comment? comment! </span><span class="cp">?&gt;</span>
+
+data
+
+<span class="cp">&lt;?php</span>
+
+<span class="cp">__halt_compiler</span> <span class="cm">/* ! */</span> <span class="o">(</span> <span class="c">// )</span>
+<span class="o">)</span> <span class="cm">/* ;;;; */</span>
+
+<span class="o">;</span>
+
+data data
+&lt;?php
+data
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test
@@ -0,0 +1,16 @@
+<?
+
+// comment? comment! ?>
+
+data
+
+<?php
+
+__halt_compiler /* ! */ ( // )
+) /* ;;;; */
+
+;
+
+data data
+<?php
+data
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect
@@ -0,0 +1,5 @@
+<span class="k">public</span> <span class="k">function</span> <span class="no">f</span><span class="o">()</span> <span class="o">{</span>
+ <span class="nc" data-symbol-name="ExampleClass">ExampleClass</span><span class="o">::</span><span class="na" data-symbol-context="ExampleClass" data-symbol-name="EXAMPLE_CONSTANT">EXAMPLE_CONSTANT</span><span class="o">;</span>
+ <span class="nc" data-symbol-name="ExampleClass">ExampleClass</span><span class="o">::</span><span class="nf" data-symbol-context="ExampleClass" data-symbol-name="exampleMethod">exampleMethod</span><span class="o">();</span>
+ <span class="nf" data-symbol-name="example_function">example_function</span><span class="o">();</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test
@@ -0,0 +1,5 @@
+public function f() {
+ ExampleClass::EXAMPLE_CONSTANT;
+ ExampleClass::exampleMethod();
+ example_function();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect
@@ -0,0 +1,3 @@
+ <span class="k">foreach</span> <span class="o">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="o">)</span> <span class="o">{</span>
+ <span class="nf" data-symbol-name="z">z</span><span class="o">();</span>
+ <span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test
@@ -0,0 +1,3 @@
+ foreach ($x as $y) {
+ z();
+ }
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect
@@ -0,0 +1,3 @@
+<span class="k">foreach</span> <span class="o">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="o">)</span> <span class="o">{</span>
+ <span class="nf" data-symbol-name="z">z</span><span class="o">();</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test
@@ -0,0 +1,3 @@
+foreach ($x as $y) {
+ z();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect
@@ -0,0 +1,10 @@
+<span class="o">&lt;?php</span>
+
+<span class="k">class</span> <span data-symbol-name="C" class="nc">C</span> <span class="k">{</span>
+ <span class="k">public</span> <span class="k">function</span> <span class="nx">f</span><span class="k">(</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="D" class="nc">D</span><span class="k">::</span><span data-symbol-context="D" data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="nx">self</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="nx">parent</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="k">static</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="k">}</span>
+<span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source
@@ -0,0 +1,10 @@
+<?php
+
+class C {
+ public function f() {
+ D::X;
+ self::X;
+ parent::X;
+ static::X;
+ }
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect
@@ -0,0 +1,3 @@
+ <span class="k">foreach</span> <span class="k">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="z" class="nf">z</span><span class="k">(</span><span class="k">)</span><span class="k">;</span>
+ <span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source
@@ -0,0 +1,3 @@
+ foreach ($x as $y) {
+ z();
+ }
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect
@@ -0,0 +1,5 @@
+<span class="o">&lt;?php</span>
+
+<span class="c">/* this comment
+</span><span class="c">extends across
+</span><span class="c">multiple lines */</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source
@@ -0,0 +1,5 @@
+<?php
+
+/* this comment
+extends across
+multiple lines */
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect
@@ -0,0 +1,3 @@
+<span class="k">foreach</span> <span class="k">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="z" class="nf">z</span><span class="k">(</span><span class="k">)</span><span class="k">;</span>
+<span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source
@@ -0,0 +1,3 @@
+foreach ($x as $y) {
+ z();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect
@@ -0,0 +1,3 @@
+<span class="o">&lt;?php</span>
+<span class="c">// xyz
+</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source
@@ -0,0 +1,2 @@
+<?php
+// xyz
diff --git a/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php b/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php
@@ -0,0 +1,33 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighterEnginePygmentsFuture
+ extends FutureProxy {
+
+ private $source;
+ private $scrub;
+
+ public function __construct(Future $proxied, $source, $scrub = false) {
+ parent::__construct($proxied);
+ $this->source = $source;
+ $this->scrub = $scrub;
+ }
+
+ protected function didReceiveResult($result) {
+ list($err, $stdout, $stderr) = $result;
+
+ if (!$err && strlen($stdout)) {
+ // Strip off fluff Pygments adds.
+ $stdout = preg_replace(
+ '@^<div class="highlight"><pre>(.*)</pre></div>\s*$@s',
+ '\1',
+ $stdout);
+ if ($this->scrub) {
+ $stdout = preg_replace('/^.*\n/', '', $stdout);
+ }
+ return phutil_safe_html($stdout);
+ }
+
+ throw new PhutilSyntaxHighlighterException($stderr, $err);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php
@@ -0,0 +1,262 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighterFuture extends FutureProxy {
+
+ private $source;
+ private $scrub;
+
+ public function __construct(Future $proxied, $source, $scrub = false) {
+ parent::__construct($proxied);
+ $this->source = $source;
+ $this->scrub = $scrub;
+ }
+
+ protected function didReceiveResult($result) {
+ try {
+ return $this->applyXHPHighlight($result);
+ } catch (Exception $ex) {
+ // XHP can't highlight source that isn't syntactically valid. Fall back
+ // to the fragment lexer.
+ $source = ($this->scrub
+ ? preg_replace('/^.*\n/', '', $this->source)
+ : $this->source);
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPHPFragmentLexer())
+ ->setConfig('language', 'php')
+ ->getHighlightFuture($source)
+ ->resolve();
+ }
+ }
+
+ private function applyXHPHighlight($result) {
+
+ // We perform two passes here: one using the AST to find symbols we care
+ // about -- particularly, class names and function names. These are used
+ // in the crossreference stuff to link into Diffusion. After we've done our
+ // AST pass, we do a followup pass on the token stream to catch all the
+ // simple stuff like strings and comments.
+
+ $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
+ $this->source,
+ $result);
+
+ $root = $tree->getRootNode();
+
+ $tokens = $root->getTokens();
+ $interesting_symbols = $this->findInterestingSymbols($root);
+
+
+ if ($this->scrub) {
+ // If we're scrubbing, we prepended "<?php\n" to the text to force the
+ // highlighter to treat it as PHP source. Now, we need to remove that.
+
+ $ok = false;
+ if (count($tokens) >= 2) {
+ if ($tokens[0]->getTypeName() === 'T_OPEN_TAG') {
+ if ($tokens[1]->getTypeName() === 'T_WHITESPACE') {
+ $ok = true;
+ }
+ }
+ }
+
+ if (!$ok) {
+ throw new Exception(
+ pht(
+ 'Expected T_OPEN_TAG, T_WHITESPACE tokens at head of results '.
+ 'for highlighting parse of PHP snippet.'));
+ }
+
+ // Remove the "<?php".
+ unset($tokens[0]);
+
+ $value = $tokens[1]->getValue();
+ if ((strlen($value) < 1) || ($value[0] != "\n")) {
+ throw new Exception(
+ pht(
+ 'Expected "\\n" at beginning of T_WHITESPACE token at head of '.
+ 'tokens for highlighting parse of PHP snippet.'));
+ }
+
+ $value = substr($value, 1);
+ $tokens[1]->overwriteValue($value);
+ }
+
+ $out = array();
+ foreach ($tokens as $key => $token) {
+ $value = $token->getValue();
+ $class = null;
+ $multi = false;
+ $attrs = array();
+ if (isset($interesting_symbols[$key])) {
+ $sym = $interesting_symbols[$key];
+ $class = $sym[0];
+ $attrs['data-symbol-context'] = idx($sym, 'context');
+ $attrs['data-symbol-name'] = idx($sym, 'symbol');
+ } else {
+ switch ($token->getTypeName()) {
+ case 'T_WHITESPACE':
+ break;
+ case 'T_DOC_COMMENT':
+ $class = 'dc';
+ $multi = true;
+ break;
+ case 'T_COMMENT':
+ $class = 'c';
+ $multi = true;
+ break;
+ case 'T_CONSTANT_ENCAPSED_STRING':
+ case 'T_ENCAPSED_AND_WHITESPACE':
+ case 'T_INLINE_HTML':
+ $class = 's';
+ $multi = true;
+ break;
+ case 'T_VARIABLE':
+ $class = 'nv';
+ break;
+ case 'T_OPEN_TAG':
+ case 'T_OPEN_TAG_WITH_ECHO':
+ case 'T_CLOSE_TAG':
+ $class = 'o';
+ break;
+ case 'T_LNUMBER':
+ case 'T_DNUMBER':
+ $class = 'm';
+ break;
+ case 'T_STRING':
+ static $magic = array(
+ 'true' => true,
+ 'false' => true,
+ 'null' => true,
+ );
+ if (isset($magic[strtolower($value)])) {
+ $class = 'k';
+ break;
+ }
+ $class = 'nx';
+ break;
+ default:
+ $class = 'k';
+ break;
+ }
+ }
+
+ if ($class) {
+ $attrs['class'] = $class;
+ if ($multi) {
+ // If the token may have multiple lines in it, make sure each
+ // <span> crosses no more than one line so the lines can be put
+ // in a table, etc., later.
+ $value = phutil_split_lines($value, $retain_endings = true);
+ } else {
+ $value = array($value);
+ }
+ foreach ($value as $val) {
+ $out[] = phutil_tag('span', $attrs, $val);
+ }
+ } else {
+ $out[] = $value;
+ }
+ }
+
+ return phutil_implode_html('', $out);
+ }
+
+ private function findInterestingSymbols(XHPASTNode $root) {
+ // Class name symbols appear in:
+ // class X extends X implements X, X { ... }
+ // new X();
+ // $x instanceof X
+ // catch (X $x)
+ // function f(X $x)
+ // X::f();
+ // X::$m;
+ // X::CONST;
+
+ // These are PHP builtin tokens which can appear in a classname context.
+ // Don't link them since they don't go anywhere useful.
+ static $builtin_class_tokens = array(
+ 'self' => true,
+ 'parent' => true,
+ 'static' => true,
+ );
+
+ // Fortunately XHPAST puts all of these in a special node type so it's
+ // easy to find them.
+ $result_map = array();
+ $class_names = $root->selectDescendantsOfType('n_CLASS_NAME');
+ foreach ($class_names as $class_name) {
+ foreach ($class_name->getTokens() as $key => $token) {
+ if (isset($builtin_class_tokens[$token->getValue()])) {
+ // This is something like "self::method()".
+ continue;
+ }
+ $result_map[$key] = array(
+ 'nc', // "Name, Class"
+ 'symbol' => $class_name->getConcreteString(),
+ );
+ }
+ }
+
+ // Function name symbols appear in:
+ // f()
+
+ $function_calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
+ foreach ($function_calls as $call) {
+ $call = $call->getChildByIndex(0);
+ if ($call->getTypeName() == 'n_SYMBOL_NAME') {
+ // This is a normal function call, not some $f() shenanigans.
+ foreach ($call->getTokens() as $key => $token) {
+ $result_map[$key] = array(
+ 'nf', // "Name, Function"
+ 'symbol' => $call->getConcreteString(),
+ );
+ }
+ }
+ }
+
+ // Upon encountering $x->y, link y without context, since $x is unknown.
+
+ $prop_access = $root->selectDescendantsOfType('n_OBJECT_PROPERTY_ACCESS');
+ foreach ($prop_access as $access) {
+ $right = $access->getChildByIndex(1);
+ if ($right->getTypeName() == 'n_INDEX_ACCESS') {
+ // otherwise $x->y[0] doesn't get highlighted
+ $right = $right->getChildByIndex(0);
+ }
+ if ($right->getTypeName() == 'n_STRING') {
+ foreach ($right->getTokens() as $key => $token) {
+ $result_map[$key] = array(
+ 'na', // "Name, Attribute"
+ 'symbol' => $right->getConcreteString(),
+ );
+ }
+ }
+ }
+
+ // Upon encountering x::y, try to link y with context x.
+
+ $static_access = $root->selectDescendantsOfType('n_CLASS_STATIC_ACCESS');
+ foreach ($static_access as $access) {
+ $class = $access->getChildByIndex(0);
+ $right = $access->getChildByIndex(1);
+ if ($class->getTypeName() == 'n_CLASS_NAME' &&
+ ($right->getTypeName() == 'n_STRING' ||
+ $right->getTypeName() == 'n_VARIABLE')) {
+ $classname = head($class->getTokens())->getValue();
+ $result = array(
+ 'na',
+ 'symbol' => ltrim($right->getConcreteString(), '$'),
+ );
+ if (!isset($builtin_class_tokens[$classname])) {
+ $result['context'] = $classname;
+ }
+ foreach ($right->getTokens() as $key => $token) {
+ $result_map[$key] = $result;
+ }
+ }
+ }
+
+ return $result_map;
+ }
+
+}
diff --git a/src/infrastructure/parser/PhutilPygmentizeParser.php b/src/infrastructure/parser/PhutilPygmentizeParser.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/parser/PhutilPygmentizeParser.php
@@ -0,0 +1,83 @@
+<?php
+
+/**
+ * Parser that converts `pygmetize` output or similar HTML blocks from "class"
+ * attributes to "style" attributes.
+ */
+final class PhutilPygmentizeParser extends Phobject {
+
+ private $map = array();
+
+ public function setMap(array $map) {
+ $this->map = $map;
+ return $this;
+ }
+
+ public function getMap() {
+ return $this->map;
+ }
+
+ public function parse($block) {
+ $class_look = 'class="';
+ $class_len = strlen($class_look);
+
+ $class_start = null;
+
+ $map = $this->map;
+
+ $len = strlen($block);
+ $out = '';
+ $mode = 'text';
+ for ($ii = 0; $ii < $len; $ii++) {
+ $c = $block[$ii];
+ switch ($mode) {
+ case 'text':
+ // We're in general text between tags, and just passing characers
+ // through unmodified.
+ if ($c == '<') {
+ $mode = 'tag';
+ }
+ $out .= $c;
+ break;
+ case 'tag':
+ // We're inside a tag, and looking for `class="` so we can rewrite
+ // it.
+ if ($c == '>') {
+ $mode = 'text';
+ }
+ if ($c == 'c') {
+ if (!substr_compare($block, $class_look, $ii, $class_len)) {
+ $mode = 'class';
+ $ii += $class_len;
+ $class_start = $ii;
+ }
+ }
+
+ if ($mode != 'class') {
+ $out .= $c;
+ }
+ break;
+ case 'class':
+ // We're inside a `class="..."` tag, and looking for the ending quote
+ // so we can replace it.
+ if ($c == '"') {
+ $class = substr($block, $class_start, $ii - $class_start);
+
+ // If this class is present in the map, rewrite it into an inline
+ // style attribute.
+ if (isset($map[$class])) {
+ $out .= 'style="'.phutil_escape_html($map[$class]).'"';
+ } else {
+ $out .= 'class="'.$class.'"';
+ }
+
+ $mode = 'tag';
+ }
+ break;
+ }
+ }
+
+ return $out;
+ }
+
+}
diff --git a/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php b/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php
@@ -0,0 +1,43 @@
+<?php
+
+final class PhutilPygmentizeParserTestCase extends PhutilTestCase {
+
+ public function testPygmentizeParser() {
+ $this->tryParser(
+ '',
+ '',
+ array(),
+ pht('Empty'));
+
+ $this->tryParser(
+ '<span class="mi">1</span>',
+ '<span style="color: #ff0000">1</span>',
+ array(
+ 'mi' => 'color: #ff0000',
+ ),
+ pht('Simple'));
+
+ $this->tryParser(
+ '<span class="mi">1</span>',
+ '<span class="mi">1</span>',
+ array(),
+ pht('Missing Class'));
+
+ $this->tryParser(
+ '<span data-symbol-name="X" class="nc">X</span>',
+ '<span data-symbol-name="X" style="color: #ff0000">X</span>',
+ array(
+ 'nc' => 'color: #ff0000',
+ ),
+ pht('Extra Attribute'));
+ }
+
+ private function tryParser($input, $expect, array $map, $label) {
+ $actual = id(new PhutilPygmentizeParser())
+ ->setMap($map)
+ ->parse($input);
+
+ $this->assertEqual($expect, $actual, pht('Pygmentize Parser: %s', $label));
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Thu, Aug 28, 4:45 PM (4 d, 13 h ago)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/c2/qf/oikbkcadlsthkdcp
Default Alt Text
D20977.id49983.diff (141 KB)

Event Timeline

Phabricator · Built by Phacility