Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F18460634
D20977.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
144 KB
Referenced Files
None
Subscribers
None
D20977.diff
View Options
diff --git a/.arclint b/.arclint
--- a/.arclint
+++ b/.arclint
@@ -64,13 +64,13 @@
"text": {
"type": "text",
"exclude": [
- "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))"
+ "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))"
]
},
"text-without-length": {
"type": "text",
"include": [
- "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json))"
+ "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))"
],
"severity": {
"3": "disabled"
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -5591,6 +5591,7 @@
'PhrictionTransactionComment' => 'applications/phriction/storage/PhrictionTransactionComment.php',
'PhrictionTransactionEditor' => 'applications/phriction/editor/PhrictionTransactionEditor.php',
'PhrictionTransactionQuery' => 'applications/phriction/query/PhrictionTransactionQuery.php',
+ 'PhutilAPCKeyValueCache' => 'infrastructure/cache/PhutilAPCKeyValueCache.php',
'PhutilAmazonAuthAdapter' => 'applications/auth/adapter/PhutilAmazonAuthAdapter.php',
'PhutilAsanaAuthAdapter' => 'applications/auth/adapter/PhutilAsanaAuthAdapter.php',
'PhutilAuthAdapter' => 'applications/auth/adapter/PhutilAuthAdapter.php',
@@ -5620,7 +5621,15 @@
'PhutilCalendarRootNode' => 'applications/calendar/parser/data/PhutilCalendarRootNode.php',
'PhutilCalendarUserNode' => 'applications/calendar/parser/data/PhutilCalendarUserNode.php',
'PhutilCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilCodeSnippetContextFreeGrammar.php',
+ 'PhutilConsoleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php',
+ 'PhutilContextFreeGrammar' => 'infrastructure/lipsum/PhutilContextFreeGrammar.php',
+ 'PhutilDefaultSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php',
+ 'PhutilDefaultSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php',
+ 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php',
+ 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php',
+ 'PhutilDirectoryKeyValueCache' => 'infrastructure/cache/PhutilDirectoryKeyValueCache.php',
'PhutilDisqusAuthAdapter' => 'applications/auth/adapter/PhutilDisqusAuthAdapter.php',
+ 'PhutilDivinerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php',
'PhutilEmptyAuthAdapter' => 'applications/auth/adapter/PhutilEmptyAuthAdapter.php',
'PhutilFacebookAuthAdapter' => 'applications/auth/adapter/PhutilFacebookAuthAdapter.php',
'PhutilGitHubAuthAdapter' => 'applications/auth/adapter/PhutilGitHubAuthAdapter.php',
@@ -5630,19 +5639,38 @@
'PhutilICSParserTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSParserTestCase.php',
'PhutilICSWriter' => 'applications/calendar/parser/ics/PhutilICSWriter.php',
'PhutilICSWriterTestCase' => 'applications/calendar/parser/ics/__tests__/PhutilICSWriterTestCase.php',
+ 'PhutilInRequestKeyValueCache' => 'infrastructure/cache/PhutilInRequestKeyValueCache.php',
+ 'PhutilInvisibleSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php',
'PhutilJIRAAuthAdapter' => 'applications/auth/adapter/PhutilJIRAAuthAdapter.php',
+ 'PhutilJSONFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php',
'PhutilJavaCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilJavaCodeSnippetContextFreeGrammar.php',
+ 'PhutilKeyValueCache' => 'infrastructure/cache/PhutilKeyValueCache.php',
+ 'PhutilKeyValueCacheNamespace' => 'infrastructure/cache/PhutilKeyValueCacheNamespace.php',
+ 'PhutilKeyValueCacheProfiler' => 'infrastructure/cache/PhutilKeyValueCacheProfiler.php',
+ 'PhutilKeyValueCacheProxy' => 'infrastructure/cache/PhutilKeyValueCacheProxy.php',
+ 'PhutilKeyValueCacheStack' => 'infrastructure/cache/PhutilKeyValueCacheStack.php',
+ 'PhutilKeyValueCacheTestCase' => 'infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php',
'PhutilLDAPAuthAdapter' => 'applications/auth/adapter/PhutilLDAPAuthAdapter.php',
+ 'PhutilLexerSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php',
'PhutilLipsumContextFreeGrammar' => 'infrastructure/lipsum/PhutilLipsumContextFreeGrammar.php',
+ 'PhutilMarkupEngine' => 'infrastructure/markup/PhutilMarkupEngine.php',
+ 'PhutilMarkupTestCase' => 'infrastructure/markup/__tests__/PhutilMarkupTestCase.php',
+ 'PhutilMemcacheKeyValueCache' => 'infrastructure/cache/PhutilMemcacheKeyValueCache.php',
'PhutilOAuth1AuthAdapter' => 'applications/auth/adapter/PhutilOAuth1AuthAdapter.php',
'PhutilOAuthAuthAdapter' => 'applications/auth/adapter/PhutilOAuthAuthAdapter.php',
+ 'PhutilOnDiskKeyValueCache' => 'infrastructure/cache/PhutilOnDiskKeyValueCache.php',
'PhutilPHPCodeSnippetContextFreeGrammar' => 'infrastructure/lipsum/code/PhutilPHPCodeSnippetContextFreeGrammar.php',
+ 'PhutilPHPFragmentLexerHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php',
'PhutilPhabricatorAuthAdapter' => 'applications/auth/adapter/PhutilPhabricatorAuthAdapter.php',
'PhutilProseDiff' => 'infrastructure/diff/prose/PhutilProseDiff.php',
'PhutilProseDiffTestCase' => 'infrastructure/diff/prose/__tests__/PhutilProseDiffTestCase.php',
'PhutilProseDifferenceEngine' => 'infrastructure/diff/prose/PhutilProseDifferenceEngine.php',
+ 'PhutilPygmentizeParser' => 'infrastructure/parser/PhutilPygmentizeParser.php',
+ 'PhutilPygmentizeParserTestCase' => 'infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php',
+ 'PhutilPygmentsSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php',
'PhutilQsprintfInterface' => 'infrastructure/storage/xsprintf/PhutilQsprintfInterface.php',
'PhutilQueryString' => 'infrastructure/storage/xsprintf/PhutilQueryString.php',
+ 'PhutilRainbowSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php',
'PhutilRealNameContextFreeGrammar' => 'infrastructure/lipsum/PhutilRealNameContextFreeGrammar.php',
'PhutilRemarkupAnchorRule' => 'infrastructure/markup/markuprule/PhutilRemarkupAnchorRule.php',
'PhutilRemarkupBlockInterpreter' => 'infrastructure/markup/blockrule/PhutilRemarkupBlockInterpreter.php',
@@ -5678,6 +5706,9 @@
'PhutilRemarkupTableBlockRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTableBlockRule.php',
'PhutilRemarkupTestInterpreterRule' => 'infrastructure/markup/blockrule/PhutilRemarkupTestInterpreterRule.php',
'PhutilRemarkupUnderlineRule' => 'infrastructure/markup/markuprule/PhutilRemarkupUnderlineRule.php',
+ 'PhutilSafeHTML' => 'infrastructure/markup/PhutilSafeHTML.php',
+ 'PhutilSafeHTMLProducerInterface' => 'infrastructure/markup/PhutilSafeHTMLProducerInterface.php',
+ 'PhutilSafeHTMLTestCase' => 'infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php',
'PhutilSearchQueryCompiler' => 'applications/search/compiler/PhutilSearchQueryCompiler.php',
'PhutilSearchQueryCompilerSyntaxException' => 'applications/search/compiler/PhutilSearchQueryCompilerSyntaxException.php',
'PhutilSearchQueryCompilerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchQueryCompilerTestCase.php',
@@ -5685,9 +5716,18 @@
'PhutilSearchStemmer' => 'applications/search/compiler/PhutilSearchStemmer.php',
'PhutilSearchStemmerTestCase' => 'applications/search/compiler/__tests__/PhutilSearchStemmerTestCase.php',
'PhutilSlackAuthAdapter' => 'applications/auth/adapter/PhutilSlackAuthAdapter.php',
+ 'PhutilSprite' => 'aphront/sprite/PhutilSprite.php',
+ 'PhutilSpriteSheet' => 'aphront/sprite/PhutilSpriteSheet.php',
+ 'PhutilSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php',
+ 'PhutilSyntaxHighlighterEngine' => 'infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php',
+ 'PhutilSyntaxHighlighterException' => 'infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php',
+ 'PhutilTranslatedHTMLTestCase' => 'infrastructure/markup/__tests__/PhutilTranslatedHTMLTestCase.php',
'PhutilTwitchAuthAdapter' => 'applications/auth/adapter/PhutilTwitchAuthAdapter.php',
'PhutilTwitterAuthAdapter' => 'applications/auth/adapter/PhutilTwitterAuthAdapter.php',
'PhutilWordPressAuthAdapter' => 'applications/auth/adapter/PhutilWordPressAuthAdapter.php',
+ 'PhutilXHPASTSyntaxHighlighter' => 'infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php',
+ 'PhutilXHPASTSyntaxHighlighterFuture' => 'infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php',
+ 'PhutilXHPASTSyntaxHighlighterTestCase' => 'infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php',
'PolicyLockOptionType' => 'applications/policy/config/PolicyLockOptionType.php',
'PonderAddAnswerView' => 'applications/ponder/view/PonderAddAnswerView.php',
'PonderAnswer' => 'applications/ponder/storage/PonderAnswer.php',
@@ -5879,6 +5919,7 @@
'function' => array(
'celerity_generate_unique_node_id' => 'applications/celerity/api.php',
'celerity_get_resource_uri' => 'applications/celerity/api.php',
+ 'hsprintf' => 'infrastructure/markup/render.php',
'javelin_tag' => 'infrastructure/javelin/markup.php',
'phabricator_date' => 'view/viewutils.php',
'phabricator_datetime' => 'view/viewutils.php',
@@ -5890,6 +5931,12 @@
'phid_get_subtype' => 'applications/phid/utils.php',
'phid_get_type' => 'applications/phid/utils.php',
'phid_group_by_type' => 'applications/phid/utils.php',
+ 'phutil_escape_html' => 'infrastructure/markup/render.php',
+ 'phutil_escape_html_newlines' => 'infrastructure/markup/render.php',
+ 'phutil_implode_html' => 'infrastructure/markup/render.php',
+ 'phutil_safe_html' => 'infrastructure/markup/render.php',
+ 'phutil_tag' => 'infrastructure/markup/render.php',
+ 'phutil_tag_div' => 'infrastructure/markup/render.php',
'qsprintf' => 'infrastructure/storage/xsprintf/qsprintf.php',
'qsprintf_check_scalar_type' => 'infrastructure/storage/xsprintf/qsprintf.php',
'qsprintf_check_type' => 'infrastructure/storage/xsprintf/qsprintf.php',
@@ -12443,6 +12490,7 @@
'PhrictionTransactionComment' => 'PhabricatorApplicationTransactionComment',
'PhrictionTransactionEditor' => 'PhabricatorApplicationTransactionEditor',
'PhrictionTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
+ 'PhutilAPCKeyValueCache' => 'PhutilKeyValueCache',
'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilAuthAdapter' => 'Phobject',
@@ -12472,7 +12520,15 @@
'PhutilCalendarRootNode' => 'PhutilCalendarContainerNode',
'PhutilCalendarUserNode' => 'PhutilCalendarNode',
'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar',
+ 'PhutilConsoleSyntaxHighlighter' => 'Phobject',
+ 'PhutilContextFreeGrammar' => 'Phobject',
+ 'PhutilDefaultSyntaxHighlighter' => 'Phobject',
+ 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine',
+ 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy',
+ 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'PhutilTestCase',
+ 'PhutilDirectoryKeyValueCache' => 'PhutilKeyValueCache',
'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilDivinerSyntaxHighlighter' => 'Phobject',
'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter',
'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter',
@@ -12482,18 +12538,37 @@
'PhutilICSParserTestCase' => 'PhutilTestCase',
'PhutilICSWriter' => 'Phobject',
'PhutilICSWriterTestCase' => 'PhutilTestCase',
+ 'PhutilInRequestKeyValueCache' => 'PhutilKeyValueCache',
+ 'PhutilInvisibleSyntaxHighlighter' => 'Phobject',
'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter',
+ 'PhutilJSONFragmentLexerHighlighterTestCase' => 'PhutilTestCase',
'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar',
+ 'PhutilKeyValueCache' => 'Phobject',
+ 'PhutilKeyValueCacheNamespace' => 'PhutilKeyValueCacheProxy',
+ 'PhutilKeyValueCacheProfiler' => 'PhutilKeyValueCacheProxy',
+ 'PhutilKeyValueCacheProxy' => 'PhutilKeyValueCache',
+ 'PhutilKeyValueCacheStack' => 'PhutilKeyValueCache',
+ 'PhutilKeyValueCacheTestCase' => 'PhutilTestCase',
'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter',
+ 'PhutilLexerSyntaxHighlighter' => 'PhutilSyntaxHighlighter',
'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar',
+ 'PhutilMarkupEngine' => 'Phobject',
+ 'PhutilMarkupTestCase' => 'PhutilTestCase',
+ 'PhutilMemcacheKeyValueCache' => 'PhutilKeyValueCache',
'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter',
'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter',
+ 'PhutilOnDiskKeyValueCache' => 'PhutilKeyValueCache',
'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar',
+ 'PhutilPHPFragmentLexerHighlighterTestCase' => 'PhutilTestCase',
'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilProseDiff' => 'Phobject',
'PhutilProseDiffTestCase' => 'PhabricatorTestCase',
'PhutilProseDifferenceEngine' => 'Phobject',
+ 'PhutilPygmentizeParser' => 'Phobject',
+ 'PhutilPygmentizeParserTestCase' => 'PhutilTestCase',
+ 'PhutilPygmentsSyntaxHighlighter' => 'Phobject',
'PhutilQueryString' => 'Phobject',
+ 'PhutilRainbowSyntaxHighlighter' => 'Phobject',
'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar',
'PhutilRemarkupAnchorRule' => 'PhutilRemarkupRule',
'PhutilRemarkupBlockInterpreter' => 'Phobject',
@@ -12529,6 +12604,8 @@
'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule',
'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter',
'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule',
+ 'PhutilSafeHTML' => 'Phobject',
+ 'PhutilSafeHTMLTestCase' => 'PhutilTestCase',
'PhutilSearchQueryCompiler' => 'Phobject',
'PhutilSearchQueryCompilerSyntaxException' => 'Exception',
'PhutilSearchQueryCompilerTestCase' => 'PhutilTestCase',
@@ -12536,9 +12613,18 @@
'PhutilSearchStemmer' => 'Phobject',
'PhutilSearchStemmerTestCase' => 'PhutilTestCase',
'PhutilSlackAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilSprite' => 'Phobject',
+ 'PhutilSpriteSheet' => 'Phobject',
+ 'PhutilSyntaxHighlighter' => 'Phobject',
+ 'PhutilSyntaxHighlighterEngine' => 'Phobject',
+ 'PhutilSyntaxHighlighterException' => 'Exception',
+ 'PhutilTranslatedHTMLTestCase' => 'PhutilTestCase',
'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter',
'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter',
'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter',
+ 'PhutilXHPASTSyntaxHighlighter' => 'Phobject',
+ 'PhutilXHPASTSyntaxHighlighterFuture' => 'FutureProxy',
+ 'PhutilXHPASTSyntaxHighlighterTestCase' => 'PhutilTestCase',
'PolicyLockOptionType' => 'PhabricatorConfigJSONOptionType',
'PonderAddAnswerView' => 'AphrontView',
'PonderAnswer' => array(
diff --git a/src/aphront/sprite/PhutilSprite.php b/src/aphront/sprite/PhutilSprite.php
new file mode 100644
--- /dev/null
+++ b/src/aphront/sprite/PhutilSprite.php
@@ -0,0 +1,76 @@
+<?php
+
+/**
+ * NOTE: This is very new and unstable.
+ */
+final class PhutilSprite extends Phobject {
+
+ private $sourceFiles = array();
+ private $sourceX;
+ private $sourceY;
+ private $sourceW;
+ private $sourceH;
+ private $targetCSS;
+ private $spriteSheet;
+ private $name;
+
+ public function setName($name) {
+ $this->name = $name;
+ return $this;
+ }
+
+ public function getName() {
+ return $this->name;
+ }
+
+ public function setTargetCSS($target_css) {
+ $this->targetCSS = $target_css;
+ return $this;
+ }
+
+ public function getTargetCSS() {
+ return $this->targetCSS;
+ }
+
+ public function setSourcePosition($x, $y) {
+ $this->sourceX = $x;
+ $this->sourceY = $y;
+ return $this;
+ }
+
+ public function setSourceSize($w, $h) {
+ $this->sourceW = $w;
+ $this->sourceH = $h;
+ return $this;
+ }
+
+ public function getSourceH() {
+ return $this->sourceH;
+ }
+
+ public function getSourceW() {
+ return $this->sourceW;
+ }
+
+ public function getSourceY() {
+ return $this->sourceY;
+ }
+
+ public function getSourceX() {
+ return $this->sourceX;
+ }
+
+ public function setSourceFile($source_file, $scale = 1) {
+ $this->sourceFiles[$scale] = $source_file;
+ return $this;
+ }
+
+ public function getSourceFile($scale) {
+ if (empty($this->sourceFiles[$scale])) {
+ throw new Exception(pht("No source file for scale '%s'!", $scale));
+ }
+
+ return $this->sourceFiles[$scale];
+ }
+
+}
diff --git a/src/aphront/sprite/PhutilSpriteSheet.php b/src/aphront/sprite/PhutilSpriteSheet.php
new file mode 100644
--- /dev/null
+++ b/src/aphront/sprite/PhutilSpriteSheet.php
@@ -0,0 +1,385 @@
+<?php
+
+/**
+ * NOTE: This is very new and unstable.
+ */
+final class PhutilSpriteSheet extends Phobject {
+
+ const MANIFEST_VERSION = 1;
+
+ const TYPE_STANDARD = 'standard';
+ const TYPE_REPEAT_X = 'repeat-x';
+ const TYPE_REPEAT_Y = 'repeat-y';
+
+ private $sprites = array();
+ private $sources = array();
+ private $hashes = array();
+ private $cssHeader;
+ private $generated;
+ private $scales = array(1);
+ private $type = self::TYPE_STANDARD;
+ private $basePath;
+
+ private $css;
+ private $images;
+
+ public function addSprite(PhutilSprite $sprite) {
+ $this->generated = false;
+ $this->sprites[] = $sprite;
+ return $this;
+ }
+
+ public function setCSSHeader($header) {
+ $this->generated = false;
+ $this->cssHeader = $header;
+ return $this;
+ }
+
+ public function setScales(array $scales) {
+ $this->scales = array_values($scales);
+ return $this;
+ }
+
+ public function getScales() {
+ return $this->scales;
+ }
+
+ public function setSheetType($type) {
+ $this->type = $type;
+ return $this;
+ }
+
+ public function setBasePath($base_path) {
+ $this->basePath = $base_path;
+ return $this;
+ }
+
+ private function generate() {
+ if ($this->generated) {
+ return;
+ }
+
+ $multi_row = true;
+ $multi_col = true;
+ $margin_w = 1;
+ $margin_h = 1;
+
+ $type = $this->type;
+ switch ($type) {
+ case self::TYPE_STANDARD:
+ break;
+ case self::TYPE_REPEAT_X:
+ $multi_col = false;
+ $margin_w = 0;
+
+ $width = null;
+ foreach ($this->sprites as $sprite) {
+ if ($width === null) {
+ $width = $sprite->getSourceW();
+ } else if ($width !== $sprite->getSourceW()) {
+ throw new Exception(
+ pht(
+ "All sprites in a '%s' sheet must have the same width.",
+ 'repeat-x'));
+ }
+ }
+ break;
+ case self::TYPE_REPEAT_Y:
+ $multi_row = false;
+ $margin_h = 0;
+
+ $height = null;
+ foreach ($this->sprites as $sprite) {
+ if ($height === null) {
+ $height = $sprite->getSourceH();
+ } else if ($height !== $sprite->getSourceH()) {
+ throw new Exception(
+ pht(
+ "All sprites in a '%s' sheet must have the same height.",
+ 'repeat-y'));
+ }
+ }
+ break;
+ default:
+ throw new Exception(pht("Unknown sprite sheet type '%s'!", $type));
+ }
+
+
+ $css = array();
+ if ($this->cssHeader) {
+ $css[] = $this->cssHeader;
+ }
+
+ $out_w = 0;
+ $out_h = 0;
+
+ // Lay out the sprite sheet. We attempt to build a roughly square sheet
+ // so it's easier to manage, since 2000x20 is more cumbersome for humans
+ // to deal with than 200x200.
+ //
+ // To do this, we use a simple greedy algorithm, adding sprites one at a
+ // time. For each sprite, if the sheet is at least as wide as it is tall
+ // we create a new row. Otherwise, we try to add it to an existing row.
+ //
+ // This isn't optimal, but does a reasonable job in most cases and isn't
+ // too messy.
+
+ // Group the sprites by their sizes. We lay them out in the sheet as
+ // boxes, but then put them into the boxes in the order they were added
+ // so similar sprites end up nearby on the final sheet.
+ $boxes = array();
+ foreach (array_reverse($this->sprites) as $sprite) {
+ $s_w = $sprite->getSourceW() + $margin_w;
+ $s_h = $sprite->getSourceH() + $margin_h;
+ $boxes[$s_w][$s_h][] = $sprite;
+ }
+
+ $rows = array();
+ foreach ($this->sprites as $sprite) {
+ $s_w = $sprite->getSourceW() + $margin_w;
+ $s_h = $sprite->getSourceH() + $margin_h;
+
+ // Choose a row for this sprite.
+ $maybe = array();
+ foreach ($rows as $key => $row) {
+ if ($row['h'] < $s_h) {
+ // We can only add it to a row if the row is at least as tall as the
+ // sprite.
+ continue;
+ }
+ // We prefer rows which have the same height as the sprite, and then
+ // rows which aren't yet very wide.
+ $wasted_v = ($row['h'] - $s_h);
+ $wasted_h = ($row['w'] / $out_w);
+ $maybe[$key] = $wasted_v + $wasted_h;
+ }
+
+ $row_key = null;
+ if ($maybe && $multi_col) {
+ // If there were any candidate rows, pick the best one.
+ asort($maybe);
+ $row_key = head_key($maybe);
+ }
+
+ if ($row_key !== null && $multi_row) {
+ // If there's a candidate row, but adding the sprite to it would make
+ // the sprite wider than it is tall, create a new row instead. This
+ // generally keeps the sprite square-ish.
+ if ($rows[$row_key]['w'] + $s_w > $out_h) {
+ $row_key = null;
+ }
+ }
+
+ if ($row_key === null) {
+ // Add a new row.
+ $rows[] = array(
+ 'w' => 0,
+ 'h' => $s_h,
+ 'boxes' => array(),
+ );
+ $row_key = last_key($rows);
+ $out_h += $s_h;
+ }
+
+ // Add the sprite box to the row.
+ $row = $rows[$row_key];
+ $row['w'] += $s_w;
+ $row['boxes'][] = array($s_w, $s_h);
+ $rows[$row_key] = $row;
+
+ $out_w = max($row['w'], $out_w);
+ }
+
+ $images = array();
+ foreach ($this->scales as $scale) {
+ $img = imagecreatetruecolor($out_w * $scale, $out_h * $scale);
+ imagesavealpha($img, true);
+ imagefill($img, 0, 0, imagecolorallocatealpha($img, 0, 0, 0, 127));
+
+ $images[$scale] = $img;
+ }
+
+
+ // Put the shorter rows first. At the same height, put the wider rows first.
+ // This makes the resulting sheet more human-readable.
+ foreach ($rows as $key => $row) {
+ $rows[$key]['sort'] = $row['h'] + (1 - ($row['w'] / $out_w));
+ }
+ $rows = isort($rows, 'sort');
+
+ $pos_x = 0;
+ $pos_y = 0;
+ $rules = array();
+ foreach ($rows as $row) {
+ $max_h = 0;
+ foreach ($row['boxes'] as $box) {
+ $sprite = array_pop($boxes[$box[0]][$box[1]]);
+
+ foreach ($images as $scale => $img) {
+ $src = $this->loadSource($sprite, $scale);
+ imagecopy(
+ $img,
+ $src,
+ $scale * $pos_x, $scale * $pos_y,
+ $scale * $sprite->getSourceX(), $scale * $sprite->getSourceY(),
+ $scale * $sprite->getSourceW(), $scale * $sprite->getSourceH());
+ }
+
+ $rule = $sprite->getTargetCSS();
+ $cssx = (-$pos_x).'px';
+ $cssy = (-$pos_y).'px';
+
+ $rules[$sprite->getName()] = "{$rule} {\n".
+ " background-position: {$cssx} {$cssy};\n}";
+
+ $pos_x += $sprite->getSourceW() + $margin_w;
+ $max_h = max($max_h, $sprite->getSourceH());
+ }
+ $pos_x = 0;
+ $pos_y += $max_h + $margin_h;
+ }
+
+ // Generate CSS rules in input order.
+ foreach ($this->sprites as $sprite) {
+ $css[] = $rules[$sprite->getName()];
+ }
+
+ $this->images = $images;
+ $this->css = implode("\n\n", $css)."\n";
+ $this->generated = true;
+ }
+
+ public function generateImage($path, $scale = 1) {
+ $this->generate();
+ $this->log(pht("Writing sprite '%s'...", $path));
+ imagepng($this->images[$scale], $path);
+ return $this;
+ }
+
+ public function generateCSS($path) {
+ $this->generate();
+ $this->log(pht("Writing CSS '%s'...", $path));
+
+ $out = $this->css;
+ $out = str_replace('{X}', imagesx($this->images[1]), $out);
+ $out = str_replace('{Y}', imagesy($this->images[1]), $out);
+
+ Filesystem::writeFile($path, $out);
+ return $this;
+ }
+
+ public function needsRegeneration(array $manifest) {
+ return ($this->buildManifest() !== $manifest);
+ }
+
+ private function buildManifest() {
+ $output = array();
+ foreach ($this->sprites as $sprite) {
+ $output[$sprite->getName()] = array(
+ 'name' => $sprite->getName(),
+ 'rule' => $sprite->getTargetCSS(),
+ 'hash' => $this->loadSourceHash($sprite),
+ );
+ }
+
+ ksort($output);
+
+ $data = array(
+ 'version' => self::MANIFEST_VERSION,
+ 'sprites' => $output,
+ 'scales' => $this->scales,
+ 'header' => $this->cssHeader,
+ 'type' => $this->type,
+ );
+
+ return $data;
+ }
+
+ public function generateManifest($path) {
+ $data = $this->buildManifest();
+
+ $json = new PhutilJSON();
+ $data = $json->encodeFormatted($data);
+ Filesystem::writeFile($path, $data);
+ return $this;
+ }
+
+ private function log($message) {
+ echo $message."\n";
+ }
+
+ private function loadSourceHash(PhutilSprite $sprite) {
+ $inputs = array();
+
+ foreach ($this->scales as $scale) {
+ $file = $sprite->getSourceFile($scale);
+
+ // If two users have a project in different places, like:
+ //
+ // /home/alincoln/project
+ // /home/htaft/project
+ //
+ // ...we want to ignore the `/home/alincoln` part when hashing the sheet,
+ // since the sprites don't change when the project directory moves. If
+ // the base path is set, build the hashes using paths relative to the
+ // base path.
+
+ $file_key = $file;
+ if ($this->basePath) {
+ $file_key = Filesystem::readablePath($file, $this->basePath);
+ }
+
+ if (empty($this->hashes[$file_key])) {
+ $this->hashes[$file_key] = md5(Filesystem::readFile($file));
+ }
+
+ $inputs[] = $file_key;
+ $inputs[] = $this->hashes[$file_key];
+ }
+
+ $inputs[] = $sprite->getSourceX();
+ $inputs[] = $sprite->getSourceY();
+ $inputs[] = $sprite->getSourceW();
+ $inputs[] = $sprite->getSourceH();
+
+ return md5(implode(':', $inputs));
+ }
+
+ private function loadSource(PhutilSprite $sprite, $scale) {
+ $file = $sprite->getSourceFile($scale);
+ if (empty($this->sources[$file])) {
+ $data = Filesystem::readFile($file);
+ $image = imagecreatefromstring($data);
+ $this->sources[$file] = array(
+ 'image' => $image,
+ 'x' => imagesx($image),
+ 'y' => imagesy($image),
+ );
+ }
+
+ $s_w = $sprite->getSourceW() * $scale;
+ $i_w = $this->sources[$file]['x'];
+ if ($s_w > $i_w) {
+ throw new Exception(
+ pht(
+ "Sprite source for '%s' is too small (expected width %d, found %d).",
+ $file,
+ $s_w,
+ $i_w));
+ }
+
+ $s_h = $sprite->getSourceH() * $scale;
+ $i_h = $this->sources[$file]['y'];
+ if ($s_h > $i_h) {
+ throw new Exception(
+ pht(
+ "Sprite source for '%s' is too small (expected height %d, found %d).",
+ $file,
+ $s_h,
+ $i_h));
+ }
+
+ return $this->sources[$file]['image'];
+ }
+
+}
diff --git a/src/applications/differential/parser/DifferentialChangesetParser.php b/src/applications/differential/parser/DifferentialChangesetParser.php
--- a/src/applications/differential/parser/DifferentialChangesetParser.php
+++ b/src/applications/differential/parser/DifferentialChangesetParser.php
@@ -592,7 +592,7 @@
$result = $text;
if (isset($intra[$key])) {
- $result = ArcanistDiffUtils::applyIntralineDiff(
+ $result = PhabricatorDifferenceEngine::applyIntralineDiff(
$result,
$intra[$key]);
}
diff --git a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php
--- a/src/applications/files/document/PhabricatorJupyterDocumentEngine.php
+++ b/src/applications/files/document/PhabricatorJupyterDocumentEngine.php
@@ -129,11 +129,11 @@
$v_segments[] = $v_segment;
}
- $usource = ArcanistDiffUtils::applyIntralineDiff(
+ $usource = PhabricatorDifferenceEngine::applyIntralineDiff(
$udisplay,
$u_segments);
- $vsource = ArcanistDiffUtils::applyIntralineDiff(
+ $vsource = PhabricatorDifferenceEngine::applyIntralineDiff(
$vdisplay,
$v_segments);
diff --git a/src/infrastructure/cache/PhutilAPCKeyValueCache.php b/src/infrastructure/cache/PhutilAPCKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilAPCKeyValueCache.php
@@ -0,0 +1,97 @@
+<?php
+
+/**
+ * Interface to the APC key-value cache. This is a very high-performance cache
+ * which is local to the current machine.
+ */
+final class PhutilAPCKeyValueCache extends PhutilKeyValueCache {
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return (function_exists('apc_fetch') || function_exists('apcu_fetch')) &&
+ ini_get('apc.enabled') &&
+ (ini_get('apc.enable_cli') || php_sapi_name() != 'cli');
+ }
+
+ public function getKeys(array $keys, $ttl = null) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ $results = array();
+ $fetched = false;
+ foreach ($keys as $key) {
+ if ($is_apcu) {
+ $result = apcu_fetch($key, $fetched);
+ } else {
+ $result = apc_fetch($key, $fetched);
+ }
+
+ if ($fetched) {
+ $results[$key] = $result;
+ }
+ }
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ // NOTE: Although modern APC supports passing an array to `apc_store()`,
+ // it is not supported by older version of APC or by HPHP.
+
+ foreach ($keys as $key => $value) {
+ if ($is_apcu) {
+ apcu_store($key, $value, $ttl);
+ } else {
+ apc_store($key, $value, $ttl);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ foreach ($keys as $key) {
+ if ($is_apcu) {
+ apcu_delete($key);
+ } else {
+ apc_delete($key);
+ }
+ }
+
+ return $this;
+ }
+
+ public function destroyCache() {
+ static $is_apcu;
+ if ($is_apcu === null) {
+ $is_apcu = self::isAPCu();
+ }
+
+ if ($is_apcu) {
+ apcu_clear_cache();
+ } else {
+ apc_clear_cache('user');
+ }
+
+ return $this;
+ }
+
+ private static function isAPCu() {
+ return function_exists('apcu_fetch');
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilDirectoryKeyValueCache.php
@@ -0,0 +1,244 @@
+<?php
+
+/**
+ * Interface to a directory-based disk cache. Storage persists across requests.
+ *
+ * This cache is very very slow, and most suitable for command line scripts
+ * which need to build large caches derived from sources like working copies
+ * (for example, Diviner). This cache performs better for large amounts of
+ * data than @{class:PhutilOnDiskKeyValueCache} because each key is serialized
+ * individually, but this comes at the cost of having even slower reads and
+ * writes.
+ *
+ * In addition to having slow reads and writes, this entire cache locks for
+ * any read or write activity.
+ *
+ * Keys for this cache treat the character "/" specially, and encode it as
+ * a new directory on disk. This can help keep the cache organized and keep the
+ * number of items in any single directory under control, by using keys like
+ * "ab/cd/efghijklmn".
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ * @task storage Cache Storage
+ */
+final class PhutilDirectoryKeyValueCache extends PhutilKeyValueCache {
+
+ private $lock;
+ private $cacheDirectory;
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+
+ public function getKeys(array $keys) {
+ $this->validateKeys($keys);
+
+ try {
+ $this->lockCache();
+ } catch (PhutilLockException $ex) {
+ return array();
+ }
+
+ $now = time();
+
+ $results = array();
+ foreach ($keys as $key) {
+ $key_file = $this->getKeyFile($key);
+ try {
+ $data = Filesystem::readFile($key_file);
+ } catch (FilesystemException $ex) {
+ continue;
+ }
+
+ $data = unserialize($data);
+ if (!$data) {
+ continue;
+ }
+
+ if (isset($data['ttl']) && $data['ttl'] < $now) {
+ continue;
+ }
+
+ $results[$key] = $data['value'];
+ }
+
+ $this->unlockCache();
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ $this->validateKeys(array_keys($keys));
+
+ $this->lockCache(15);
+
+ if ($ttl) {
+ $ttl_epoch = time() + $ttl;
+ } else {
+ $ttl_epoch = null;
+ }
+
+ foreach ($keys as $key => $value) {
+ $dict = array(
+ 'value' => $value,
+ );
+ if ($ttl_epoch) {
+ $dict['ttl'] = $ttl_epoch;
+ }
+
+ try {
+ $key_file = $this->getKeyFile($key);
+ $key_dir = dirname($key_file);
+ if (!Filesystem::pathExists($key_dir)) {
+ Filesystem::createDirectory(
+ $key_dir,
+ $mask = 0755,
+ $recursive = true);
+ }
+
+ $new_file = $key_file.'.new';
+ Filesystem::writeFile($new_file, serialize($dict));
+ Filesystem::rename($new_file, $key_file);
+ } catch (FilesystemException $ex) {
+ phlog($ex);
+ }
+ }
+
+ $this->unlockCache();
+
+ return $this;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $this->validateKeys($keys);
+
+ $this->lockCache(15);
+
+ foreach ($keys as $key) {
+ $path = $this->getKeyFile($key);
+ Filesystem::remove($path);
+
+ // If removing this key leaves the directory empty, clean it up. Then
+ // clean up any empty parent directories.
+ $path = dirname($path);
+ do {
+ if (!Filesystem::isDescendant($path, $this->getCacheDirectory())) {
+ break;
+ }
+ if (Filesystem::listDirectory($path, true)) {
+ break;
+ }
+ Filesystem::remove($path);
+ $path = dirname($path);
+ } while (true);
+ }
+
+ $this->unlockCache();
+
+ return $this;
+ }
+
+
+ public function destroyCache() {
+ Filesystem::remove($this->getCacheDirectory());
+ return $this;
+ }
+
+
+/* -( Cache Storage )------------------------------------------------------ */
+
+
+ /**
+ * @task storage
+ */
+ public function setCacheDirectory($directory) {
+ $this->cacheDirectory = rtrim($directory, '/').'/';
+ return $this;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getCacheDirectory() {
+ if (!$this->cacheDirectory) {
+ throw new PhutilInvalidStateException('setCacheDirectory');
+ }
+ return $this->cacheDirectory;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getKeyFile($key) {
+ // Colon is a drive separator on Windows.
+ $key = str_replace(':', '_', $key);
+
+ // NOTE: We add ".cache" to each file so we don't get a collision if you
+ // set the keys "a" and "a/b". Without ".cache", the file "a" would need
+ // to be both a file and a directory.
+ return $this->getCacheDirectory().$key.'.cache';
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function validateKeys(array $keys) {
+ foreach ($keys as $key) {
+ // NOTE: Use of "." is reserved for ".lock", "key.new" and "key.cache".
+ // Use of "_" is reserved for converting ":".
+ if (!preg_match('@^[a-zA-Z0-9/:-]+$@', $key)) {
+ throw new Exception(
+ pht(
+ "Invalid key '%s': directory caches may only contain letters, ".
+ "numbers, hyphen, colon and slash.",
+ $key));
+ }
+ }
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function lockCache($wait = 0) {
+ if ($this->lock) {
+ throw new Exception(
+ pht(
+ 'Trying to %s with a lock!',
+ __FUNCTION__.'()'));
+ }
+
+ if (!Filesystem::pathExists($this->getCacheDirectory())) {
+ Filesystem::createDirectory($this->getCacheDirectory(), 0755, true);
+ }
+
+ $lock = PhutilFileLock::newForPath($this->getCacheDirectory().'.lock');
+ $lock->lock($wait);
+
+ $this->lock = $lock;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function unlockCache() {
+ if (!$this->lock) {
+ throw new PhutilInvalidStateException('lockCache');
+ }
+
+ $this->lock->unlock();
+ $this->lock = null;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilInRequestKeyValueCache.php b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilInRequestKeyValueCache.php
@@ -0,0 +1,118 @@
+<?php
+
+/**
+ * Key-value cache implemented in the current request. All storage is local
+ * to this request (i.e., the current page) and destroyed after the request
+ * exits. This means the first request to this cache for a given key on a page
+ * will ALWAYS miss.
+ *
+ * This cache exists mostly to support unit tests. In a well-designed
+ * applications, you generally should not be fetching the same data over and
+ * over again in one request, so this cache should be of limited utility.
+ * If using this cache improves application performance, especially if it
+ * improves it significantly, it may indicate an architectural problem in your
+ * application.
+ */
+final class PhutilInRequestKeyValueCache extends PhutilKeyValueCache {
+
+ private $cache = array();
+ private $ttl = array();
+ private $limit = 0;
+
+
+ /**
+ * Set a limit on the number of keys this cache may contain.
+ *
+ * When too many keys are inserted, the oldest keys are removed from the
+ * cache. Setting a limit of `0` disables the cache.
+ *
+ * @param int Maximum number of items to store in the cache.
+ * @return this
+ */
+ public function setLimit($limit) {
+ $this->limit = $limit;
+ return $this;
+ }
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+ public function getKeys(array $keys) {
+ $results = array();
+ $now = time();
+ foreach ($keys as $key) {
+ if (!isset($this->cache[$key]) && !array_key_exists($key, $this->cache)) {
+ continue;
+ }
+ if (isset($this->ttl[$key]) && ($this->ttl[$key] < $now)) {
+ continue;
+ }
+ $results[$key] = $this->cache[$key];
+ }
+
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+
+ foreach ($keys as $key => $value) {
+ $this->cache[$key] = $value;
+ }
+
+ if ($ttl) {
+ $end = time() + $ttl;
+ foreach ($keys as $key => $value) {
+ $this->ttl[$key] = $end;
+ }
+ } else {
+ foreach ($keys as $key => $value) {
+ unset($this->ttl[$key]);
+ }
+ }
+
+ if ($this->limit) {
+ $count = count($this->cache);
+ if ($count > $this->limit) {
+ $remove = array();
+ foreach ($this->cache as $key => $value) {
+ $remove[] = $key;
+
+ $count--;
+ if ($count <= $this->limit) {
+ break;
+ }
+ }
+
+ $this->deleteKeys($remove);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ foreach ($keys as $key) {
+ unset($this->cache[$key]);
+ unset($this->ttl[$key]);
+ }
+
+ return $this;
+ }
+
+ public function getAllKeys() {
+ return $this->cache;
+ }
+
+ public function destroyCache() {
+ $this->cache = array();
+ $this->ttl = array();
+
+ return $this;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCache.php b/src/infrastructure/cache/PhutilKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCache.php
@@ -0,0 +1,121 @@
+<?php
+
+/**
+ * Interface to a key-value cache like Memcache or APC. This class provides a
+ * uniform interface to multiple different key-value caches and integration
+ * with PhutilServiceProfiler.
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ */
+abstract class PhutilKeyValueCache extends Phobject {
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ /**
+ * Determine if the cache is available. For example, the APC cache tests if
+ * APC is installed. If this method returns false, the cache is not
+ * operational and can not be used.
+ *
+ * @return bool True if the cache can be used.
+ * @task kvimpl
+ */
+ public function isAvailable() {
+ return false;
+ }
+
+
+ /**
+ * Get a single key from cache. See @{method:getKeys} to get multiple keys at
+ * once.
+ *
+ * @param string Key to retrieve.
+ * @param wild Optional value to return if the key is not found. By
+ * default, returns null.
+ * @return wild Cache value (on cache hit) or default value (on cache
+ * miss).
+ * @task kvimpl
+ */
+ final public function getKey($key, $default = null) {
+ $map = $this->getKeys(array($key));
+ return idx($map, $key, $default);
+ }
+
+
+ /**
+ * Set a single key in cache. See @{method:setKeys} to set multiple keys at
+ * once.
+ *
+ * See @{method:setKeys} for a description of TTLs.
+ *
+ * @param string Key to set.
+ * @param wild Value to set.
+ * @param int|null Optional TTL.
+ * @return this
+ * @task kvimpl
+ */
+ final public function setKey($key, $value, $ttl = null) {
+ return $this->setKeys(array($key => $value), $ttl);
+ }
+
+
+ /**
+ * Delete a key from the cache. See @{method:deleteKeys} to delete multiple
+ * keys at once.
+ *
+ * @param string Key to delete.
+ * @return this
+ * @task kvimpl
+ */
+ final public function deleteKey($key) {
+ return $this->deleteKeys(array($key));
+ }
+
+
+ /**
+ * Get data from the cache.
+ *
+ * @param list<string> List of cache keys to retrieve.
+ * @return dict<string, wild> Dictionary of keys that were found in the
+ * cache. Keys not present in the cache are
+ * omitted, so you can detect a cache miss.
+ * @task kvimpl
+ */
+ abstract public function getKeys(array $keys);
+
+
+ /**
+ * Put data into the key-value cache.
+ *
+ * With a TTL ("time to live"), the cache will automatically delete the key
+ * after a specified number of seconds. By default, there is no expiration
+ * policy and data will persist in cache indefinitely.
+ *
+ * @param dict<string, wild> Map of cache keys to values.
+ * @param int|null TTL for cache keys, in seconds.
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function setKeys(array $keys, $ttl = null);
+
+
+ /**
+ * Delete a list of keys from the cache.
+ *
+ * @param list<string> List of keys to delete.
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function deleteKeys(array $keys);
+
+
+ /**
+ * Completely destroy all data in the cache.
+ *
+ * @return this
+ * @task kvimpl
+ */
+ abstract public function destroyCache();
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheNamespace.php
@@ -0,0 +1,65 @@
+<?php
+
+final class PhutilKeyValueCacheNamespace extends PhutilKeyValueCacheProxy {
+
+ private $namespace;
+
+ public function setNamespace($namespace) {
+ if (strpos($namespace, ':') !== false) {
+ throw new Exception(pht("Namespace can't contain colons."));
+ }
+
+ $this->namespace = $namespace.':';
+
+ return $this;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ return parent::setKeys(array_combine(
+ $this->prefixKeys(array_keys($keys)),
+ $keys), $ttl);
+ }
+
+ public function getKeys(array $keys) {
+ $results = parent::getKeys($this->prefixKeys($keys));
+
+ if (!$results) {
+ return array();
+ }
+
+ return array_combine(
+ $this->unprefixKeys(array_keys($results)),
+ $results);
+ }
+
+ public function deleteKeys(array $keys) {
+ return parent::deleteKeys($this->prefixKeys($keys));
+ }
+
+ private function prefixKeys(array $keys) {
+ if ($this->namespace == null) {
+ throw new Exception(pht('Namespace not set.'));
+ }
+
+ $prefixed_keys = array();
+ foreach ($keys as $key) {
+ $prefixed_keys[] = $this->namespace.$key;
+ }
+
+ return $prefixed_keys;
+ }
+
+ private function unprefixKeys(array $keys) {
+ if ($this->namespace == null) {
+ throw new Exception(pht('Namespace not set.'));
+ }
+
+ $unprefixed_keys = array();
+ foreach ($keys as $key) {
+ $unprefixed_keys[] = substr($key, strlen($this->namespace));
+ }
+
+ return $unprefixed_keys;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheProfiler.php
@@ -0,0 +1,108 @@
+<?php
+
+final class PhutilKeyValueCacheProfiler extends PhutilKeyValueCacheProxy {
+
+ private $profiler;
+ private $name;
+
+ public function setName($name) {
+ $this->name = $name;
+ return $this;
+ }
+
+ public function getName() {
+ return $this->name;
+ }
+
+ /**
+ * Set a profiler for cache operations.
+ *
+ * @param PhutilServiceProfiler Service profiler.
+ * @return this
+ * @task kvimpl
+ */
+ public function setProfiler(PhutilServiceProfiler $profiler) {
+ $this->profiler = $profiler;
+ return $this;
+ }
+
+
+ /**
+ * Get the current profiler.
+ *
+ * @return PhutilServiceProfiler|null Profiler, or null if none is set.
+ * @task kvimpl
+ */
+ public function getProfiler() {
+ return $this->profiler;
+ }
+
+
+ public function getKeys(array $keys) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-get',
+ 'name' => $this->getName(),
+ 'keys' => $keys,
+ ));
+ }
+
+ $results = parent::getKeys($keys);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall(
+ $call_id,
+ array(
+ 'hits' => array_keys($results),
+ ));
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-set',
+ 'name' => $this->getName(),
+ 'keys' => array_keys($keys),
+ 'ttl' => $ttl,
+ ));
+ }
+
+ $result = parent::setKeys($keys, $ttl);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall($call_id, array());
+ }
+
+ return $result;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $call_id = null;
+ if ($this->getProfiler()) {
+ $call_id = $this->getProfiler()->beginServiceCall(
+ array(
+ 'type' => 'kvcache-del',
+ 'name' => $this->getName(),
+ 'keys' => $keys,
+ ));
+ }
+
+ $result = parent::deleteKeys($keys);
+
+ if ($call_id !== null) {
+ $this->getProfiler()->endServiceCall($call_id, array());
+ }
+
+ return $result;
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheProxy.php b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheProxy.php
@@ -0,0 +1,45 @@
+<?php
+
+abstract class PhutilKeyValueCacheProxy extends PhutilKeyValueCache {
+
+ private $proxy;
+
+ final public function __construct(PhutilKeyValueCache $proxy) {
+ $this->proxy = $proxy;
+ }
+
+ final protected function getProxy() {
+ return $this->proxy;
+ }
+
+ public function isAvailable() {
+ return $this->getProxy()->isAvailable();
+ }
+
+
+ public function getKeys(array $keys) {
+ return $this->getProxy()->getKeys($keys);
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ return $this->getProxy()->setKeys($keys, $ttl);
+ }
+
+
+ public function deleteKeys(array $keys) {
+ return $this->getProxy()->deleteKeys($keys);
+ }
+
+
+ public function destroyCache() {
+ return $this->getProxy()->destroyCache();
+ }
+
+ public function __call($method, array $arguments) {
+ return call_user_func_array(
+ array($this->getProxy(), $method),
+ $arguments);
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilKeyValueCacheStack.php b/src/infrastructure/cache/PhutilKeyValueCacheStack.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilKeyValueCacheStack.php
@@ -0,0 +1,131 @@
+<?php
+
+/**
+ * Stacks multiple caches on top of each other, with readthrough semantics:
+ *
+ * - For reads, we try each cache in order until we find all the keys.
+ * - For writes, we set the keys in each cache.
+ *
+ * @task config Configuring the Stack
+ */
+final class PhutilKeyValueCacheStack extends PhutilKeyValueCache {
+
+
+ /**
+ * Forward list of caches in the stack (from the nearest cache to the farthest
+ * cache).
+ */
+ private $cachesForward;
+
+
+ /**
+ * Backward list of caches in the stack (from the farthest cache to the
+ * nearest cache).
+ */
+ private $cachesBackward;
+
+
+ /**
+ * TTL to use for any writes which are side effects of the next read
+ * operation.
+ */
+ private $nextTTL;
+
+
+/* -( Configuring the Stack )---------------------------------------------- */
+
+
+ /**
+ * Set the caches which comprise this stack.
+ *
+ * @param list<PhutilKeyValueCache> Ordered list of key-value caches.
+ * @return this
+ * @task config
+ */
+ public function setCaches(array $caches) {
+ assert_instances_of($caches, 'PhutilKeyValueCache');
+ $this->cachesForward = $caches;
+ $this->cachesBackward = array_reverse($caches);
+
+ return $this;
+ }
+
+
+ /**
+ * Set the readthrough TTL for the next cache operation. The TTL applies to
+ * any keys set by the next call to @{method:getKey} or @{method:getKeys},
+ * and is reset after the call finishes.
+ *
+ * // If this causes any caches to fill, they'll fill with a 15-second TTL.
+ * $stack->setNextTTL(15)->getKey('porcupine');
+ *
+ * // TTL does not persist; this will use no TTL.
+ * $stack->getKey('hedgehog');
+ *
+ * @param int TTL in seconds.
+ * @return this
+ *
+ * @task config
+ */
+ public function setNextTTL($ttl) {
+ $this->nextTTL = $ttl;
+ return $this;
+ }
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function getKeys(array $keys) {
+
+ $remaining = array_fuse($keys);
+ $results = array();
+ $missed = array();
+
+ try {
+ foreach ($this->cachesForward as $cache) {
+ $result = $cache->getKeys($remaining);
+ $remaining = array_diff_key($remaining, $result);
+ $results += $result;
+ if (!$remaining) {
+ while ($cache = array_pop($missed)) {
+ // TODO: This sets too many results in the closer caches, although
+ // it probably isn't a big deal in most cases; normally we're just
+ // filling the request cache.
+ $cache->setKeys($result, $this->nextTTL);
+ }
+ break;
+ }
+ $missed[] = $cache;
+ }
+ $this->nextTTL = null;
+ } catch (Exception $ex) {
+ $this->nextTTL = null;
+ throw $ex;
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->setKeys($keys, $ttl);
+ }
+ }
+
+
+ public function deleteKeys(array $keys) {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->deleteKeys($keys);
+ }
+ }
+
+
+ public function destroyCache() {
+ foreach ($this->cachesBackward as $cache) {
+ $cache->destroyCache();
+ }
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilMemcacheKeyValueCache.php
@@ -0,0 +1,153 @@
+<?php
+
+/**
+ * @task memcache Managing Memcache
+ */
+final class PhutilMemcacheKeyValueCache extends PhutilKeyValueCache {
+
+ private $servers = array();
+ private $connections = array();
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return function_exists('memcache_pconnect');
+ }
+
+ public function getKeys(array $keys) {
+ $buckets = $this->bucketKeys($keys);
+ $results = array();
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+ $result = $conn->get($bucket_keys);
+ if (!$result) {
+ // If the call fails, treat it as a miss on all keys.
+ $result = array();
+ }
+
+ $results += $result;
+ }
+
+ return $results;
+ }
+
+ public function setKeys(array $keys, $ttl = null) {
+ $buckets = $this->bucketKeys(array_keys($keys));
+
+ // Memcache interprets TTLs as:
+ //
+ // - Seconds from now, for values from 1 to 2592000 (30 days).
+ // - Epoch timestamp, for values larger than 2592000.
+ //
+ // We support only relative TTLs, so convert excessively large relative
+ // TTLs into epoch TTLs.
+ if ($ttl > 2592000) {
+ $effective_ttl = time() + $ttl;
+ } else {
+ $effective_ttl = $ttl;
+ }
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+
+ foreach ($bucket_keys as $key) {
+ $conn->set($key, $keys[$key], 0, $effective_ttl);
+ }
+ }
+
+ return $this;
+ }
+
+ public function deleteKeys(array $keys) {
+ $buckets = $this->bucketKeys($keys);
+
+ foreach ($buckets as $bucket => $bucket_keys) {
+ $conn = $this->getConnection($bucket);
+ foreach ($bucket_keys as $key) {
+ $conn->delete($key);
+ }
+ }
+
+ return $this;
+ }
+
+ public function destroyCache() {
+ foreach ($this->servers as $key => $spec) {
+ $this->getConnection($key)->flush();
+ }
+ return $this;
+ }
+
+
+/* -( Managing Memcache )-------------------------------------------------- */
+
+
+ /**
+ * Set available memcache servers. For example:
+ *
+ * $cache->setServers(
+ * array(
+ * array(
+ * 'host' => '10.0.0.20',
+ * 'port' => 11211,
+ * ),
+ * array(
+ * 'host' => '10.0.0.21',
+ * 'port' => 11211,
+ * ),
+ * ));
+ *
+ * @param list<dict> List of server specifications.
+ * @return this
+ * @task memcache
+ */
+ public function setServers(array $servers) {
+ $this->servers = array_values($servers);
+ return $this;
+ }
+
+ private function bucketKeys(array $keys) {
+ $buckets = array();
+ $n = count($this->servers);
+
+ if (!$n) {
+ throw new PhutilInvalidStateException('setServers');
+ }
+
+ foreach ($keys as $key) {
+ $bucket = (int)((crc32($key) & 0x7FFFFFFF) % $n);
+ $buckets[$bucket][] = $key;
+ }
+
+ return $buckets;
+ }
+
+
+ /**
+ * @phutil-external-symbol function memcache_pconnect
+ */
+ private function getConnection($server) {
+ if (empty($this->connections[$server])) {
+ $spec = $this->servers[$server];
+ $host = $spec['host'];
+ $port = $spec['port'];
+
+ $conn = memcache_pconnect($host, $spec['port']);
+
+ if (!$conn) {
+ throw new Exception(
+ pht(
+ 'Unable to connect to memcache server (%s:%d)!',
+ $host,
+ $port));
+ }
+
+ $this->connections[$server] = $conn;
+ }
+ return $this->connections[$server];
+ }
+
+}
diff --git a/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/PhutilOnDiskKeyValueCache.php
@@ -0,0 +1,205 @@
+<?php
+
+/**
+ * Interface to a disk cache. Storage persists across requests.
+ *
+ * This cache is very slow compared to caches like APC. It is intended as a
+ * specialized alternative to APC when APC is not available.
+ *
+ * This is a highly specialized cache and not appropriate for use as a
+ * generalized key-value cache for arbitrary application data.
+ *
+ * Also note that reading and writing keys from the cache currently involves
+ * loading and saving the entire cache, no matter how little data you touch.
+ *
+ * @task kvimpl Key-Value Cache Implementation
+ * @task storage Cache Storage
+ */
+final class PhutilOnDiskKeyValueCache extends PhutilKeyValueCache {
+
+ private $cache = array();
+ private $cacheFile;
+ private $lock;
+ private $wait = 0;
+
+
+/* -( Key-Value Cache Implementation )------------------------------------- */
+
+
+ public function isAvailable() {
+ return true;
+ }
+
+
+ /**
+ * Set duration (in seconds) to wait for the file lock.
+ */
+ public function setWait($wait) {
+ $this->wait = $wait;
+ return $this;
+ }
+
+ public function getKeys(array $keys) {
+ $now = time();
+
+ $results = array();
+ $reloaded = false;
+ foreach ($keys as $key) {
+
+ // Try to read the value from cache. If we miss, load (or reload) the
+ // cache.
+
+ while (true) {
+ if (isset($this->cache[$key])) {
+ $val = $this->cache[$key];
+ if (empty($val['ttl']) || $val['ttl'] >= $now) {
+ $results[$key] = $val['val'];
+ break;
+ }
+ }
+
+ if ($reloaded) {
+ break;
+ }
+
+ $this->loadCache($hold_lock = false);
+ $reloaded = true;
+ }
+ }
+
+ return $results;
+ }
+
+
+ public function setKeys(array $keys, $ttl = null) {
+ if ($ttl) {
+ $ttl_epoch = time() + $ttl;
+ } else {
+ $ttl_epoch = null;
+ }
+
+ $dicts = array();
+ foreach ($keys as $key => $value) {
+ $dict = array(
+ 'val' => $value,
+ );
+ if ($ttl_epoch) {
+ $dict['ttl'] = $ttl_epoch;
+ }
+ $dicts[$key] = $dict;
+ }
+
+ $this->loadCache($hold_lock = true);
+ foreach ($dicts as $key => $dict) {
+ $this->cache[$key] = $dict;
+ }
+ $this->saveCache();
+
+ return $this;
+ }
+
+
+ public function deleteKeys(array $keys) {
+ $this->loadCache($hold_lock = true);
+ foreach ($keys as $key) {
+ unset($this->cache[$key]);
+ }
+ $this->saveCache();
+
+ return $this;
+ }
+
+
+ public function destroyCache() {
+ Filesystem::remove($this->getCacheFile());
+ return $this;
+ }
+
+
+/* -( Cache Storage )------------------------------------------------------ */
+
+
+ /**
+ * @task storage
+ */
+ public function setCacheFile($file) {
+ $this->cacheFile = $file;
+ return $this;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function loadCache($hold_lock) {
+ if ($this->lock) {
+ throw new Exception(
+ pht(
+ 'Trying to %s with a lock!',
+ __FUNCTION__.'()'));
+ }
+
+ $lock = PhutilFileLock::newForPath($this->getCacheFile().'.lock');
+ try {
+ $lock->lock($this->wait);
+ } catch (PhutilLockException $ex) {
+ if ($hold_lock) {
+ throw $ex;
+ } else {
+ $this->cache = array();
+ return;
+ }
+ }
+
+ try {
+ $this->cache = array();
+ if (Filesystem::pathExists($this->getCacheFile())) {
+ $cache = unserialize(Filesystem::readFile($this->getCacheFile()));
+ if ($cache) {
+ $this->cache = $cache;
+ }
+ }
+ } catch (Exception $ex) {
+ $lock->unlock();
+ throw $ex;
+ }
+
+ if ($hold_lock) {
+ $this->lock = $lock;
+ } else {
+ $lock->unlock();
+ }
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function saveCache() {
+ if (!$this->lock) {
+ throw new PhutilInvalidStateException('loadCache');
+ }
+
+ // We're holding a lock so we're safe to do a write to a well-known file.
+ // Write to the same directory as the cache so the rename won't imply a
+ // copy across volumes.
+ $new = $this->getCacheFile().'.new';
+ Filesystem::writeFile($new, serialize($this->cache));
+ Filesystem::rename($new, $this->getCacheFile());
+
+ $this->lock->unlock();
+ $this->lock = null;
+ }
+
+
+ /**
+ * @task storage
+ */
+ private function getCacheFile() {
+ if (!$this->cacheFile) {
+ throw new PhutilInvalidStateException('setCacheFile');
+ }
+ return $this->cacheFile;
+ }
+
+}
diff --git a/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/cache/__tests__/PhutilKeyValueCacheTestCase.php
@@ -0,0 +1,267 @@
+<?php
+
+final class PhutilKeyValueCacheTestCase extends PhutilTestCase {
+
+ public function testInRequestCache() {
+ $cache = new PhutilInRequestKeyValueCache();
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testInRequestCacheLimit() {
+ $cache = new PhutilInRequestKeyValueCache();
+ $cache->setLimit(4);
+
+ $cache->setKey(1, 1);
+ $cache->setKey(2, 2);
+ $cache->setKey(3, 3);
+ $cache->setKey(4, 4);
+
+ $this->assertEqual(
+ array(
+ 1 => 1,
+ 2 => 2,
+ 3 => 3,
+ 4 => 4,
+ ),
+ $cache->getAllKeys());
+
+
+ $cache->setKey(5, 5);
+
+ $this->assertEqual(
+ array(
+ 2 => 2,
+ 3 => 3,
+ 4 => 4,
+ 5 => 5,
+ ),
+ $cache->getAllKeys());
+ }
+
+ public function testOnDiskCache() {
+ $cache = new PhutilOnDiskKeyValueCache();
+ $cache->setCacheFile(new TempFile());
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testAPCCache() {
+ $cache = new PhutilAPCKeyValueCache();
+ if (!$cache->isAvailable()) {
+ $this->assertSkipped(pht('Cache not available.'));
+ }
+ $this->doCacheTest($cache);
+ }
+
+ public function testDirectoryCache() {
+ $cache = new PhutilDirectoryKeyValueCache();
+
+ $dir = Filesystem::createTemporaryDirectory();
+ $cache->setCacheDirectory($dir);
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testDirectoryCacheSpecialDirectoryRules() {
+ $cache = new PhutilDirectoryKeyValueCache();
+
+ $dir = Filesystem::createTemporaryDirectory();
+ $dir = $dir.'/dircache/';
+ $cache->setCacheDirectory($dir);
+
+ $cache->setKey('a', 1);
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a.cache'));
+
+ $cache->setKey('a/b', 1);
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a/'));
+ $this->assertEqual(true, Filesystem::pathExists($dir.'/a/b.cache'));
+
+ $cache->deleteKey('a/b');
+ $this->assertEqual(false, Filesystem::pathExists($dir.'/a/'));
+ $this->assertEqual(false, Filesystem::pathExists($dir.'/a/b.cache'));
+
+ $cache->destroyCache();
+ $this->assertEqual(false, Filesystem::pathExists($dir));
+ }
+
+ public function testNamespaceCache() {
+ $namespace = 'namespace'.mt_rand();
+ $in_request_cache = new PhutilInRequestKeyValueCache();
+ $cache = new PhutilKeyValueCacheNamespace($in_request_cache);
+ $cache->setNamespace($namespace);
+
+ $test_info = get_class($cache);
+ $keys = array(
+ 'key1' => mt_rand(),
+ 'key2' => '',
+ 'key3' => 'Phabricator',
+ );
+ $cache->setKeys($keys);
+ $cached_keys = $in_request_cache->getAllKeys();
+
+ foreach ($keys as $key => $value) {
+ $cached_key = $namespace.':'.$key;
+
+ $this->assertTrue(
+ isset($cached_keys[$cached_key]),
+ $test_info);
+
+ $this->assertEqual(
+ $value,
+ $cached_keys[$cached_key],
+ $test_info);
+ }
+
+ $cache->destroyCache();
+
+ $this->doCacheTest($cache);
+ $cache->destroyCache();
+ }
+
+ public function testCacheStack() {
+ $req_cache = new PhutilInRequestKeyValueCache();
+ $disk_cache = new PhutilOnDiskKeyValueCache();
+ $disk_cache->setCacheFile(new TempFile());
+ $apc_cache = new PhutilAPCKeyValueCache();
+
+ $stack = array(
+ $req_cache,
+ $disk_cache,
+ );
+
+ if ($apc_cache->isAvailable()) {
+ $stack[] = $apc_cache;
+ }
+
+ $cache = new PhutilKeyValueCacheStack();
+ $cache->setCaches($stack);
+
+ $this->doCacheTest($cache);
+
+ $disk_cache->destroyCache();
+ $req_cache->destroyCache();
+ }
+
+ private function doCacheTest(PhutilKeyValueCache $cache) {
+ $key1 = 'test:'.mt_rand();
+ $key2 = 'test:'.mt_rand();
+
+ $default = 'cache-miss';
+ $value1 = 'cache-hit1';
+ $value2 = 'cache-hit2';
+
+ $test_info = get_class($cache);
+
+ // Test that we miss correctly on missing values.
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can set individual keys.
+
+ $cache->setKey($key1, $value1);
+ $this->assertEqual(
+ $value1,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ $key1 => $value1,
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can delete individual keys.
+
+ $cache->deleteKey($key1);
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+
+ // Test that we can set multiple keys.
+
+ $cache->setKeys(
+ array(
+ $key1 => $value1,
+ $key2 => $value2,
+ ));
+
+ $this->assertEqual(
+ $value1,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ $key1 => $value1,
+ $key2 => $value2,
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // Test that we can delete multiple keys.
+
+ $cache->deleteKeys(array($key1, $key2));
+
+ $this->assertEqual(
+ $default,
+ $cache->getKey($key1, $default),
+ $test_info);
+ $this->assertEqual(
+ array(
+ ),
+ $cache->getKeys(array($key1, $key2)),
+ $test_info);
+
+
+ // NOTE: The TTL tests are necessarily slow (we must sleep() through the
+ // TTLs) and do not work with APC (it does not TTL until the next request)
+ // so they're disabled by default. If you're developing the cache stack,
+ // it may be useful to run them.
+
+ return;
+
+ // Test that keys expire when they TTL.
+
+ $cache->setKey($key1, $value1, 1);
+ $cache->setKey($key2, $value2, 5);
+
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ $this->assertEqual($value2, $cache->getKey($key2, $default));
+
+ sleep(2);
+
+ $this->assertEqual($default, $cache->getKey($key1, $default));
+ $this->assertEqual($value2, $cache->getKey($key2, $default));
+
+
+ // Test that setting a 0 TTL overwrites a nonzero TTL.
+
+ $cache->setKey($key1, $value1, 1);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ $cache->setKey($key1, $value1, 0);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ sleep(2);
+ $this->assertEqual($value1, $cache->getKey($key1, $default));
+ }
+
+}
diff --git a/src/infrastructure/diff/PhabricatorDifferenceEngine.php b/src/infrastructure/diff/PhabricatorDifferenceEngine.php
--- a/src/infrastructure/diff/PhabricatorDifferenceEngine.php
+++ b/src/infrastructure/diff/PhabricatorDifferenceEngine.php
@@ -169,4 +169,93 @@
return $corpus;
}
+ public static function applyIntralineDiff($str, $intra_stack) {
+ $buf = '';
+ $p = $s = $e = 0; // position, start, end
+ $highlight = $tag = $ent = false;
+ $highlight_o = '<span class="bright">';
+ $highlight_c = '</span>';
+
+ $depth_in = '<span class="depth-in">';
+ $depth_out = '<span class="depth-out">';
+
+ $is_html = false;
+ if ($str instanceof PhutilSafeHTML) {
+ $is_html = true;
+ $str = $str->getHTMLContent();
+ }
+
+ $n = strlen($str);
+ for ($i = 0; $i < $n; $i++) {
+
+ if ($p == $e) {
+ do {
+ if (empty($intra_stack)) {
+ $buf .= substr($str, $i);
+ break 2;
+ }
+ $stack = array_shift($intra_stack);
+ $s = $e;
+ $e += $stack[1];
+ } while ($stack[0] === 0);
+
+ switch ($stack[0]) {
+ case '>':
+ $open_tag = $depth_in;
+ break;
+ case '<':
+ $open_tag = $depth_out;
+ break;
+ default:
+ $open_tag = $highlight_o;
+ break;
+ }
+ }
+
+ if (!$highlight && !$tag && !$ent && $p == $s) {
+ $buf .= $open_tag;
+ $highlight = true;
+ }
+
+ if ($str[$i] == '<') {
+ $tag = true;
+ if ($highlight) {
+ $buf .= $highlight_c;
+ }
+ }
+
+ if (!$tag) {
+ if ($str[$i] == '&') {
+ $ent = true;
+ }
+ if ($ent && $str[$i] == ';') {
+ $ent = false;
+ }
+ if (!$ent) {
+ $p++;
+ }
+ }
+
+ $buf .= $str[$i];
+
+ if ($tag && $str[$i] == '>') {
+ $tag = false;
+ if ($highlight) {
+ $buf .= $open_tag;
+ }
+ }
+
+ if ($highlight && ($p == $e || $i == $n - 1)) {
+ $buf .= $highlight_c;
+ $highlight = false;
+ }
+ }
+
+ if ($is_html) {
+ return phutil_safe_html($buf);
+ }
+
+ return $buf;
+ }
+
}
diff --git a/src/infrastructure/lipsum/PhutilContextFreeGrammar.php b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/lipsum/PhutilContextFreeGrammar.php
@@ -0,0 +1,93 @@
+<?php
+
+/**
+ * Generate nonsense test data according to a context-free grammar definition.
+ */
+abstract class PhutilContextFreeGrammar extends Phobject {
+
+ private $limit = 65535;
+
+ abstract protected function getRules();
+
+ public function generateSeveral($count, $implode = ' ') {
+ $paragraph = array();
+ for ($ii = 0; $ii < $count; $ii++) {
+ $paragraph[$ii] = $this->generate();
+ }
+ return implode($implode, $paragraph);
+ }
+
+ public function generate() {
+ $count = 0;
+ $rules = $this->getRules();
+ return $this->applyRules('[start]', $count, $rules);
+ }
+
+ final protected function applyRules($input, &$count, array $rules) {
+ if (++$count > $this->limit) {
+ throw new Exception(pht('Token replacement count exceeded limit!'));
+ }
+
+ $matches = null;
+ preg_match_all('/(\\[[^\\]]+\\])/', $input, $matches, PREG_OFFSET_CAPTURE);
+
+ foreach (array_reverse($matches[1]) as $token_spec) {
+ list($token, $offset) = $token_spec;
+ $token_name = substr($token, 1, -1);
+ $options = array();
+
+ if (($name_end = strpos($token_name, ','))) {
+ $options_parser = new PhutilSimpleOptions();
+ $options = $options_parser->parse($token_name);
+ $token_name = substr($token_name, 0, $name_end);
+ }
+
+ if (empty($rules[$token_name])) {
+ throw new Exception(pht("Invalid token '%s' in grammar.", $token_name));
+ }
+
+ $key = array_rand($rules[$token_name]);
+ $replacement = $this->applyRules($rules[$token_name][$key],
+ $count, $rules);
+
+ if (isset($options['indent'])) {
+ if (is_numeric($options['indent'])) {
+ $replacement = self::strPadLines($replacement, $options['indent']);
+ } else {
+ $replacement = self::strPadLines($replacement);
+ }
+ }
+ if (isset($options['trim'])) {
+ switch ($options['trim']) {
+ case 'left':
+ $replacement = ltrim($replacement);
+ break;
+ case 'right':
+ $replacement = rtrim($replacement);
+ break;
+ default:
+ case 'both':
+ $replacement = trim($replacement);
+ break;
+ }
+ }
+ if (isset($options['block'])) {
+ $replacement = "\n".$replacement."\n";
+ }
+
+ $input = substr_replace($input, $replacement, $offset, strlen($token));
+ }
+
+ return $input;
+ }
+
+ private static function strPadLines($text, $num_spaces = 2) {
+ $text_lines = phutil_split_lines($text);
+ foreach ($text_lines as $linenr => $line) {
+ $text_lines[$linenr] = str_repeat(' ', $num_spaces).$line;
+ }
+
+ return implode('', $text_lines);
+ }
+
+}
diff --git a/src/infrastructure/markup/PhutilMarkupEngine.php b/src/infrastructure/markup/PhutilMarkupEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilMarkupEngine.php
@@ -0,0 +1,32 @@
+<?php
+
+abstract class PhutilMarkupEngine extends Phobject {
+
+ /**
+ * Set a configuration parameter which the engine can read to customize how
+ * the text is marked up. This is a generic interface; consult the
+ * documentation for specific rules and blocks for what options are available
+ * for configuration.
+ *
+ * @param string Key to set in the configuration dictionary.
+ * @param string Value to set.
+ * @return this
+ */
+ abstract public function setConfig($key, $value);
+
+ /**
+ * After text has been marked up with @{method:markupText}, you can retrieve
+ * any metadata the markup process generated by calling this method. This is
+ * a generic interface that allows rules to export extra information about
+ * text; consult the documentation for specific rules and blocks to see what
+ * metadata may be available in your configuration.
+ *
+ * @param string Key to retrieve from metadata.
+ * @param mixed Default value to return if the key is not available.
+ * @return mixed Metadata property, or default value.
+ */
+ abstract public function getTextMetadata($key, $default = null);
+
+ abstract public function markupText($text);
+
+}
diff --git a/src/infrastructure/markup/PhutilSafeHTML.php b/src/infrastructure/markup/PhutilSafeHTML.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilSafeHTML.php
@@ -0,0 +1,44 @@
+<?php
+
+final class PhutilSafeHTML extends Phobject {
+
+ private $content;
+
+ public function __construct($content) {
+ $this->content = (string)$content;
+ }
+
+ public function __toString() {
+ return $this->content;
+ }
+
+ public function getHTMLContent() {
+ return $this->content;
+ }
+
+ public function appendHTML($html /* , ... */) {
+ foreach (func_get_args() as $html) {
+ $this->content .= phutil_escape_html($html);
+ }
+ return $this;
+ }
+
+ public static function applyFunction($function, $string /* , ... */) {
+ $args = func_get_args();
+ array_shift($args);
+ $args = array_map('phutil_escape_html', $args);
+ return new PhutilSafeHTML(call_user_func_array($function, $args));
+ }
+
+// Requires http://pecl.php.net/operator.
+
+ public function __concat($html) {
+ $clone = clone $this;
+ return $clone->appendHTML($html);
+ }
+
+ public function __assign_concat($html) {
+ return $this->appendHTML($html);
+ }
+
+}
diff --git a/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/PhutilSafeHTMLProducerInterface.php
@@ -0,0 +1,12 @@
+<?php
+
+/**
+ * Implement this interface to mark an object as capable of producing a
+ * PhutilSafeHTML representation. This is primarily useful for building
+ * renderable HTML views.
+ */
+interface PhutilSafeHTMLProducerInterface {
+
+ public function producePhutilSafeHTML();
+
+}
diff --git a/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php b/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/__tests__/PhutilMarkupTestCase.php
@@ -0,0 +1,223 @@
+<?php
+
+final class PhutilMarkupTestCase extends PhutilTestCase {
+
+ public function testTagDefaults() {
+ $this->assertEqual(
+ (string)phutil_tag('br'),
+ (string)phutil_tag('br', array()));
+
+ $this->assertEqual(
+ (string)phutil_tag('br', array()),
+ (string)phutil_tag('br', array(), null));
+ }
+
+ public function testTagEmpty() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br', array(), null));
+
+ $this->assertEqual(
+ '<div></div>',
+ (string)phutil_tag('div', array(), null));
+
+ $this->assertEqual(
+ '<div></div>',
+ (string)phutil_tag('div', array(), ''));
+ }
+
+ public function testTagBasics() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br'));
+
+ $this->assertEqual(
+ '<div>y</div>',
+ (string)phutil_tag('div', array(), 'y'));
+ }
+
+ public function testTagAttributes() {
+ $this->assertEqual(
+ '<div u="v">y</div>',
+ (string)phutil_tag('div', array('u' => 'v'), 'y'));
+
+ $this->assertEqual(
+ '<br u="v" />',
+ (string)phutil_tag('br', array('u' => 'v')));
+ }
+
+ public function testTagEscapes() {
+ $this->assertEqual(
+ '<br u="<" />',
+ (string)phutil_tag('br', array('u' => '<')));
+
+ $this->assertEqual(
+ '<div><br /></div>',
+ (string)phutil_tag('div', array(), phutil_tag('br')));
+ }
+
+ public function testTagNullAttribute() {
+ $this->assertEqual(
+ '<br />',
+ (string)phutil_tag('br', array('y' => null)));
+ }
+
+ public function testTagJavascriptProtocolRejection() {
+ $hrefs = array(
+ 'javascript:alert(1)' => true,
+ 'JAVASCRIPT:alert(2)' => true,
+
+ // NOTE: When interpreted as a URI, this is dropped because of leading
+ // whitespace.
+ ' javascript:alert(3)' => array(true, false),
+ '/' => false,
+ '/path/to/stuff/' => false,
+ '' => false,
+ 'http://example.com/' => false,
+ '#' => false,
+ 'javascript://anything' => true,
+
+ // Chrome 33 and IE11, at a minimum, treat this as Javascript.
+ "javascript\n:alert(4)" => true,
+
+ // Opera currently accepts a variety of unicode spaces. This test case
+ // has a smattering of them.
+ "\xE2\x80\x89javascript:" => true,
+ "javascript\xE2\x80\x89:" => true,
+ "\xE2\x80\x84javascript:" => true,
+ "javascript\xE2\x80\x84:" => true,
+
+ // Because we're aggressive, all of unicode should trigger detection
+ // by default.
+ "\xE2\x98\x83javascript:" => true,
+ "javascript\xE2\x98\x83:" => true,
+ "\xE2\x98\x83javascript\xE2\x98\x83:" => true,
+
+ // We're aggressive about this, so we'll intentionally raise false
+ // positives in these cases.
+ 'javascript~:alert(5)' => true,
+ '!!!javascript!!!!:alert(6)' => true,
+
+ // However, we should raise true negatives in these slightly more
+ // reasonable cases.
+ 'javascript/:docs.html' => false,
+ 'javascripts:x.png' => false,
+ 'COOLjavascript:page' => false,
+ '/javascript:alert(1)' => false,
+ );
+
+ foreach (array(true, false) as $use_uri) {
+ foreach ($hrefs as $href => $expect) {
+ if (is_array($expect)) {
+ $expect = ($use_uri ? $expect[1] : $expect[0]);
+ }
+
+ if ($use_uri) {
+ $href_value = new PhutilURI($href);
+ } else {
+ $href_value = $href;
+ }
+
+ $caught = null;
+ try {
+ phutil_tag('a', array('href' => $href_value), 'click for candy');
+ } catch (Exception $ex) {
+ $caught = $ex;
+ }
+
+ $desc = pht(
+ 'Unexpected result for "%s". <uri = %s, expect exception = %s>',
+ $href,
+ $use_uri ? pht('Yes') : pht('No'),
+ $expect ? pht('Yes') : pht('No'));
+
+ $this->assertEqual(
+ $expect,
+ $caught instanceof Exception,
+ $desc);
+ }
+ }
+ }
+
+ public function testURIEscape() {
+ $this->assertEqual(
+ '%2B/%20%3F%23%26%3A%21xyz%25',
+ phutil_escape_uri('+/ ?#&:!xyz%'));
+ }
+
+ public function testURIPathComponentEscape() {
+ $this->assertEqual(
+ 'a%252Fb',
+ phutil_escape_uri_path_component('a/b'));
+
+ $str = '';
+ for ($ii = 0; $ii <= 255; $ii++) {
+ $str .= chr($ii);
+ }
+
+ $this->assertEqual(
+ $str,
+ phutil_unescape_uri_path_component(
+ rawurldecode( // Simulates webserver.
+ phutil_escape_uri_path_component($str))));
+ }
+
+ public function testHsprintf() {
+ $this->assertEqual(
+ '<div><3</div>',
+ (string)hsprintf('<div>%s</div>', '<3'));
+ }
+
+ public function testAppendHTML() {
+ $html = phutil_tag('hr');
+ $html->appendHTML(phutil_tag('br'), '<evil>');
+ $this->assertEqual('<hr /><br /><evil>', $html->getHTMLContent());
+ }
+
+ public function testArrayEscaping() {
+ $this->assertEqual(
+ '<div><div></div>',
+ phutil_escape_html(
+ array(
+ hsprintf('<div>'),
+ array(
+ array(
+ '<',
+ array(
+ 'd',
+ array(
+ array(
+ hsprintf('i'),
+ ),
+ 'v',
+ ),
+ ),
+ array(
+ array(
+ '>',
+ ),
+ ),
+ ),
+ ),
+ hsprintf('</div>'),
+ )));
+
+ $this->assertEqual(
+ '<div><br /><hr /><wbr /></div>',
+ phutil_tag(
+ 'div',
+ array(),
+ array(
+ array(
+ array(
+ phutil_tag('br'),
+ array(
+ phutil_tag('hr'),
+ ),
+ phutil_tag('wbr'),
+ ),
+ ),
+ ))->getHTMLContent());
+ }
+
+}
diff --git a/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/__tests__/PhutilSafeHTMLTestCase.php
@@ -0,0 +1,19 @@
+<?php
+
+final class PhutilSafeHTMLTestCase extends PhutilTestCase {
+
+ public function testOperator() {
+ if (!extension_loaded('operator')) {
+ $this->assertSkipped(pht('Operator extension not available.'));
+ }
+
+ $a = phutil_tag('a');
+ $ab = $a.phutil_tag('b');
+ $this->assertEqual('<a></a><b></b>', $ab->getHTMLContent());
+ $this->assertEqual('<a></a>', $a->getHTMLContent());
+
+ $a .= phutil_tag('a');
+ $this->assertEqual('<a></a><a></a>', $a->getHTMLContent());
+ }
+
+}
diff --git a/src/infrastructure/markup/__tests__/PhutilTranslatedHTMLTestCase.php b/src/infrastructure/markup/__tests__/PhutilTranslatedHTMLTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/__tests__/PhutilTranslatedHTMLTestCase.php
@@ -0,0 +1,68 @@
+<?php
+
+final class PhutilTranslatedHTMLTestCase extends PhutilTestCase {
+
+ public function testHTMLTranslations() {
+ $string = '%s awoke <strong>suddenly</strong> at %s.';
+ $when = '<4 AM>';
+
+ $translator = $this->newTranslator('en_US');
+
+ // When no components are HTML, everything is treated as a string.
+ $who = '<span>Abraham</span>';
+ $translation = $translator->translate(
+ $string,
+ $who,
+ $when);
+ $this->assertEqual(
+ 'string',
+ gettype($translation));
+ $this->assertEqual(
+ '<span>Abraham</span> awoke <strong>suddenly</strong> at <4 AM>.',
+ $translation);
+
+ // When at least one component is HTML, everything is treated as HTML.
+ $who = phutil_tag('span', array(), 'Abraham');
+ $translation = $translator->translate(
+ $string,
+ $who,
+ $when);
+ $this->assertTrue($translation instanceof PhutilSafeHTML);
+ $this->assertEqual(
+ '<span>Abraham</span> awoke <strong>suddenly</strong> at <4 AM>.',
+ $translation->getHTMLContent());
+
+ $translation = $translator->translate(
+ $string,
+ $who,
+ new PhutilNumber(1383930802));
+ $this->assertEqual(
+ '<span>Abraham</span> awoke <strong>suddenly</strong> at 1,383,930,802.',
+ $translation->getHTMLContent());
+
+ // In this translation, we have no alternatives for the first conversion.
+ $translator->setTranslations(
+ array(
+ 'Run the command %s %d time(s).' => array(
+ array(
+ 'Run the command %s once.',
+ 'Run the command %s %d times.',
+ ),
+ ),
+ ));
+
+ $this->assertEqual(
+ 'Run the command <tt>ls</tt> 123 times.',
+ (string)$translator->translate(
+ 'Run the command %s %d time(s).',
+ hsprintf('<tt>%s</tt>', 'ls'),
+ 123));
+ }
+
+ private function newTranslator($locale_code) {
+ $locale = PhutilLocale::loadLocale($locale_code);
+ return id(new PhutilTranslator())
+ ->setLocale($locale);
+ }
+
+}
diff --git a/src/infrastructure/markup/render.php b/src/infrastructure/markup/render.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/render.php
@@ -0,0 +1,183 @@
+<?php
+
+/**
+ * Render an HTML tag in a way that treats user content as unsafe by default.
+ *
+ * Tag rendering has some special logic which implements security features:
+ *
+ * - When rendering `<a>` tags, if the `rel` attribute is not specified, it
+ * is interpreted as `rel="noreferrer"`.
+ * - When rendering `<a>` tags, the `href` attribute may not begin with
+ * `javascript:`.
+ *
+ * These special cases can not be disabled.
+ *
+ * IMPORTANT: The `$tag` attribute and the keys of the `$attributes` array are
+ * trusted blindly, and not escaped. You should not pass user data in these
+ * parameters.
+ *
+ * @param string The name of the tag, like `a` or `div`.
+ * @param map<string, string> A map of tag attributes.
+ * @param wild Content to put in the tag.
+ * @return PhutilSafeHTML Tag object.
+ */
+function phutil_tag($tag, array $attributes = array(), $content = null) {
+ // If the `href` attribute is present, make sure it is not a "javascript:"
+ // URI. We never permit these.
+ if (!empty($attributes['href'])) {
+ // This might be a URI object, so cast it to a string.
+ $href = (string)$attributes['href'];
+
+ if (isset($href[0])) {
+ // Block 'javascript:' hrefs at the tag level: no well-designed
+ // application should ever use them, and they are a potent attack vector.
+
+ // This function is deep in the core and performance sensitive, so we're
+ // doing a cheap version of this test first to avoid calling preg_match()
+ // on URIs which begin with '/' or `#`. These cover essentially all URIs
+ // in Phabricator.
+ if (($href[0] !== '/') && ($href[0] !== '#')) {
+ // Chrome 33 and IE 11 both interpret "javascript\n:" as a Javascript
+ // URI, and all browsers interpret " javascript:" as a Javascript URI,
+ // so be aggressive about looking for "javascript:" in the initial
+ // section of the string.
+
+ $normalized_href = preg_replace('([^a-z0-9/:]+)i', '', $href);
+ if (preg_match('/^javascript:/i', $normalized_href)) {
+ throw new Exception(
+ pht(
+ "Attempting to render a tag with an '%s' attribute that begins ".
+ "with '%s'. This is either a serious security concern or a ".
+ "serious architecture concern. Seek urgent remedy.",
+ 'href',
+ 'javascript:'));
+ }
+ }
+ }
+ }
+
+ // For tags which can't self-close, treat null as the empty string -- for
+ // example, always render `<div></div>`, never `<div />`.
+ static $self_closing_tags = array(
+ 'area' => true,
+ 'base' => true,
+ 'br' => true,
+ 'col' => true,
+ 'command' => true,
+ 'embed' => true,
+ 'frame' => true,
+ 'hr' => true,
+ 'img' => true,
+ 'input' => true,
+ 'keygen' => true,
+ 'link' => true,
+ 'meta' => true,
+ 'param' => true,
+ 'source' => true,
+ 'track' => true,
+ 'wbr' => true,
+ );
+
+ $attr_string = '';
+ foreach ($attributes as $k => $v) {
+ if ($v === null) {
+ continue;
+ }
+ $v = phutil_escape_html($v);
+ $attr_string .= ' '.$k.'="'.$v.'"';
+ }
+
+ if ($content === null) {
+ if (isset($self_closing_tags[$tag])) {
+ return new PhutilSafeHTML('<'.$tag.$attr_string.' />');
+ } else {
+ $content = '';
+ }
+ } else {
+ $content = phutil_escape_html($content);
+ }
+
+ return new PhutilSafeHTML('<'.$tag.$attr_string.'>'.$content.'</'.$tag.'>');
+}
+
+function phutil_tag_div($class, $content = null) {
+ return phutil_tag('div', array('class' => $class), $content);
+}
+
+function phutil_escape_html($string) {
+ if ($string instanceof PhutilSafeHTML) {
+ return $string;
+ } else if ($string instanceof PhutilSafeHTMLProducerInterface) {
+ $result = $string->producePhutilSafeHTML();
+ if ($result instanceof PhutilSafeHTML) {
+ return phutil_escape_html($result);
+ } else if (is_array($result)) {
+ return phutil_escape_html($result);
+ } else if ($result instanceof PhutilSafeHTMLProducerInterface) {
+ return phutil_escape_html($result);
+ } else {
+ try {
+ assert_stringlike($result);
+ return phutil_escape_html((string)$result);
+ } catch (Exception $ex) {
+ throw new Exception(
+ pht(
+ "Object (of class '%s') implements %s but did not return anything ".
+ "renderable from %s.",
+ get_class($string),
+ 'PhutilSafeHTMLProducerInterface',
+ 'producePhutilSafeHTML()'));
+ }
+ }
+ } else if (is_array($string)) {
+ $result = '';
+ foreach ($string as $item) {
+ $result .= phutil_escape_html($item);
+ }
+ return $result;
+ }
+
+ return htmlspecialchars($string, ENT_QUOTES, 'UTF-8');
+}
+
+function phutil_escape_html_newlines($string) {
+ return PhutilSafeHTML::applyFunction('nl2br', $string);
+}
+
+/**
+ * Mark string as safe for use in HTML.
+ */
+function phutil_safe_html($string) {
+ if ($string == '') {
+ return $string;
+ } else if ($string instanceof PhutilSafeHTML) {
+ return $string;
+ } else {
+ return new PhutilSafeHTML($string);
+ }
+}
+
+/**
+ * HTML safe version of `implode()`.
+ */
+function phutil_implode_html($glue, array $pieces) {
+ $glue = phutil_escape_html($glue);
+
+ foreach ($pieces as $k => $piece) {
+ $pieces[$k] = phutil_escape_html($piece);
+ }
+
+ return phutil_safe_html(implode($glue, $pieces));
+}
+
+/**
+ * Format a HTML code. This function behaves like `sprintf()`, except that all
+ * the normal conversions (like %s) will be properly escaped.
+ */
+function hsprintf($html /* , ... */) {
+ $args = func_get_args();
+ array_shift($args);
+ return new PhutilSafeHTML(
+ vsprintf($html, array_map('phutil_escape_html', $args)));
+}
+
diff --git a/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php
@@ -0,0 +1,115 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighterEngine
+ extends PhutilSyntaxHighlighterEngine {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getLanguageFromFilename($filename) {
+ static $default_map = array(
+ // All files which have file extensions that we haven't already matched
+ // map to their extensions.
+ '@\\.([^./]+)$@' => 1,
+ );
+
+ $maps = array();
+ if (!empty($this->config['filename.map'])) {
+ $maps[] = $this->config['filename.map'];
+ }
+ $maps[] = $default_map;
+
+ foreach ($maps as $map) {
+ foreach ($map as $regexp => $lang) {
+ $matches = null;
+ if (preg_match($regexp, $filename, $matches)) {
+ if (is_numeric($lang)) {
+ return idx($matches, $lang);
+ } else {
+ return $lang;
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public function getHighlightFuture($language, $source) {
+ if ($language === null) {
+ $language = PhutilLanguageGuesser::guessLanguage($source);
+ }
+
+ $have_pygments = !empty($this->config['pygments.enabled']);
+
+ if ($language == 'php' && PhutilXHPASTBinary::isAvailable()) {
+ return id(new PhutilXHPASTSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'console') {
+ return id(new PhutilConsoleSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'diviner' || $language == 'remarkup') {
+ return id(new PhutilDivinerSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'rainbow') {
+ return id(new PhutilRainbowSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'php') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPHPFragmentLexer())
+ ->setConfig('language', 'php')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'py' || $language == 'python') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPythonFragmentLexer())
+ ->setConfig('language', 'py')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'java') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilJavaFragmentLexer())
+ ->setConfig('language', 'java')
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'json') {
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilJSONFragmentLexer())
+ ->getHighlightFuture($source);
+ }
+
+ if ($language == 'invisible') {
+ return id(new PhutilInvisibleSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ // Don't invoke Pygments for plain text, since it's expensive and has
+ // no effect.
+ if ($language !== 'text' && $language !== 'txt') {
+ if ($have_pygments) {
+ return id(new PhutilPygmentsSyntaxHighlighter())
+ ->setConfig('language', $language)
+ ->getHighlightFuture($source);
+ }
+ }
+
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/PhutilSyntaxHighlighterEngine.php
@@ -0,0 +1,19 @@
+<?php
+
+abstract class PhutilSyntaxHighlighterEngine extends Phobject {
+
+ abstract public function setConfig($key, $value);
+ abstract public function getHighlightFuture($language, $source);
+ abstract public function getLanguageFromFilename($filename);
+
+ final public function highlightSource($language, $source) {
+ try {
+ return $this->getHighlightFuture($language, $source)->resolve();
+ } catch (PhutilSyntaxHighlighterException $ex) {
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source)
+ ->resolve();
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php
@@ -0,0 +1,28 @@
+<?php
+
+/**
+ * Test cases for @{class:PhutilDefaultSyntaxHighlighterEngine}.
+ */
+final class PhutilDefaultSyntaxHighlighterEngineTestCase
+ extends PhutilTestCase {
+
+ public function testFilenameGreediness() {
+ $names = array(
+ 'x.php' => 'php',
+ '/x.php' => 'php',
+ 'x.y.php' => 'php',
+ '/x.y/z.php' => 'php',
+ '/x.php/' => null,
+ );
+
+ $engine = new PhutilDefaultSyntaxHighlighterEngine();
+ foreach ($names as $path => $language) {
+ $detect = $engine->getLanguageFromFilename($path);
+ $this->assertEqual(
+ $language,
+ $detect,
+ pht('Language detect for %s', $path));
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php
@@ -0,0 +1,51 @@
+<?php
+
+/**
+ * Simple syntax highlighter for console output. We just try to highlight the
+ * commands so it's easier to follow transcripts.
+ */
+final class PhutilConsoleSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $in_command = false;
+ $lines = explode("\n", $source);
+ foreach ($lines as $key => $line) {
+ $matches = null;
+
+ // Parse commands like this:
+ //
+ // some/path/ $ ./bin/example # Do things
+ //
+ // ...into path, command, and comment components.
+
+ $pattern =
+ '@'.
+ ($in_command ? '()(.*?)' : '^(\S+[\\\\/] )?([$] .*?)').
+ '(#.*|\\\\)?$@';
+
+ if (preg_match($pattern, $line, $matches)) {
+ $lines[$key] = hsprintf(
+ '%s<span class="gp">%s</span>%s',
+ $matches[1],
+ $matches[2],
+ (!empty($matches[3])
+ ? hsprintf('<span class="k">%s</span>', $matches[3])
+ : ''));
+ $in_command = (idx($matches, 3) == '\\');
+ } else {
+ $lines[$key] = hsprintf('<span class="go">%s</span>', $line);
+ }
+ }
+ $lines = phutil_implode_html("\n", $lines);
+
+ return new ImmediateFuture($lines);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php
@@ -0,0 +1,14 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighter extends Phobject {
+
+ public function setConfig($key, $value) {
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $result = hsprintf('%s', $source);
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php
@@ -0,0 +1,81 @@
+<?php
+
+/**
+ * Simple syntax highlighter for the ".diviner" format, which is just Remarkup
+ * with a specific ruleset. This should also work alright for Remarkup.
+ */
+final class PhutilDivinerSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+ private $replaceClass;
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $source = phutil_escape_html($source);
+
+ // This highlighter isn't perfect but tries to do an okay job at getting
+ // some of the basics at least. There's lots of room for improvement.
+
+ $blocks = explode("\n\n", $source);
+ foreach ($blocks as $key => $block) {
+ if (preg_match('/^[^ ](?! )/m', $block)) {
+ $blocks[$key] = $this->highlightBlock($block);
+ }
+ }
+ $source = implode("\n\n", $blocks);
+
+ $source = phutil_safe_html($source);
+ return new ImmediateFuture($source);
+ }
+
+ private function highlightBlock($source) {
+ // Highlight "@{class:...}" links to other documentation pages.
+ $source = $this->highlightPattern('/@{([\w@]+?):([^}]+?)}/', $source, 'nc');
+
+ // Highlight "@title", "@group", etc.
+ $source = $this->highlightPattern('/^@(\w+)/m', $source, 'k');
+
+ // Highlight bold, italic and monospace.
+ $source = $this->highlightPattern('@\\*\\*(.+?)\\*\\*@s', $source, 's');
+ $source = $this->highlightPattern('@(?<!:)//(.+?)//@s', $source, 's');
+ $source = $this->highlightPattern(
+ '@##([\s\S]+?)##|\B`(.+?)`\B@',
+ $source,
+ 's');
+
+ // Highlight stuff that looks like headers.
+ $source = $this->highlightPattern('/^=(.*)$/m', $source, 'nv');
+
+ return $source;
+ }
+
+ private function highlightPattern($regexp, $source, $class) {
+ $this->replaceClass = $class;
+ $source = preg_replace_callback(
+ $regexp,
+ array($this, 'replacePattern'),
+ $source);
+
+ return $source;
+ }
+
+ public function replacePattern($matches) {
+
+ // NOTE: The goal here is to make sure a <span> never crosses a newline.
+
+ $content = $matches[0];
+ $content = explode("\n", $content);
+ foreach ($content as $key => $line) {
+ $content[$key] =
+ '<span class="'.$this->replaceClass.'">'.
+ $line.
+ '</span>';
+ }
+ return implode("\n", $content);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php
@@ -0,0 +1,43 @@
+<?php
+
+final class PhutilInvisibleSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $keys = array_map('chr', range(0x0, 0x1F));
+ $vals = array_map(
+ array($this, 'decimalToHtmlEntityDecoded'), range(0x2400, 0x241F));
+
+ $invisible = array_combine($keys, $vals);
+
+ $result = array();
+ foreach (str_split($source) as $character) {
+ if (isset($invisible[$character])) {
+ $result[] = phutil_tag(
+ 'span',
+ array('class' => 'invisible'),
+ $invisible[$character]);
+
+ if ($character === "\n") {
+ $result[] = $character;
+ }
+ } else {
+ $result[] = $character;
+ }
+ }
+
+ $result = phutil_implode_html('', $result);
+ return new ImmediateFuture($result);
+ }
+
+ private function decimalToHtmlEntityDecoded($dec) {
+ return html_entity_decode("&#{$dec};");
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php
@@ -0,0 +1,72 @@
+<?php
+
+final class PhutilLexerSyntaxHighlighter extends PhutilSyntaxHighlighter {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $strip = false;
+ $state = 'start';
+ $lang = idx($this->config, 'language');
+
+ if ($lang == 'php') {
+ if (strpos($source, '<?') === false) {
+ $state = 'php';
+ }
+ }
+
+ $lexer = idx($this->config, 'lexer');
+ $tokens = $lexer->getTokens($source, $state);
+ $tokens = $lexer->mergeTokens($tokens);
+
+ $result = array();
+ foreach ($tokens as $token) {
+ list($type, $value, $context) = $token;
+
+ $data_name = null;
+ switch ($type) {
+ case 'nc':
+ case 'nf':
+ case 'na':
+ $data_name = $value;
+ break;
+ }
+
+ if (strpos($value, "\n") !== false) {
+ $value = explode("\n", $value);
+ } else {
+ $value = array($value);
+ }
+ foreach ($value as $part) {
+ if (strlen($part)) {
+ if ($type) {
+ $result[] = phutil_tag(
+ 'span',
+ array(
+ 'class' => $type,
+ 'data-symbol-context' => $context,
+ 'data-symbol-name' => $data_name,
+ ),
+ $part);
+ } else {
+ $result[] = $part;
+ }
+ }
+ $result[] = "\n";
+ }
+
+ // Throw away the last "\n".
+ array_pop($result);
+ }
+
+ $result = phutil_implode_html('', $result);
+
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php
@@ -0,0 +1,229 @@
+<?php
+
+final class PhutilPygmentsSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+ $language = idx($this->config, 'language');
+
+ if (preg_match('/\r(?!\n)/', $source)) {
+ // TODO: Pygments converts "\r" newlines into "\n" newlines, so we can't
+ // use it on files with "\r" newlines. If we have "\r" not followed by
+ // "\n" in the file, skip highlighting.
+ $language = null;
+ }
+
+ if ($language) {
+ $language = $this->getPygmentsLexerNameFromLanguageName($language);
+
+ // See T13224. Under Ubuntu, avoid leaving an intermedite "dash" shell
+ // process so we hit "pygmentize" directly if we have to SIGKILL this
+ // because it explodes.
+
+ $future = new ExecFuture(
+ 'exec pygmentize -O encoding=utf-8 -O stripnl=False -f html -l %s',
+ $language);
+
+ $scrub = false;
+ if ($language == 'php' && strpos($source, '<?') === false) {
+ $source = "<?php\n".$source;
+ $scrub = true;
+ }
+
+ // See T13224. In some cases, "pygmentize" has explosive runtime on small
+ // inputs. Put a hard cap on how long it is allowed to run for to limit
+ // the amount of damage it can do.
+ $future->setTimeout(15);
+
+ $future->write($source);
+
+ return new PhutilDefaultSyntaxHighlighterEnginePygmentsFuture(
+ $future,
+ $source,
+ $scrub);
+ }
+
+ return id(new PhutilDefaultSyntaxHighlighter())
+ ->getHighlightFuture($source);
+ }
+
+ private function getPygmentsLexerNameFromLanguageName($language) {
+ static $map = array(
+ 'adb' => 'ada',
+ 'ads' => 'ada',
+ 'ahkl' => 'ahk',
+ 'as' => 'as3',
+ 'asax' => 'aspx-vb',
+ 'ascx' => 'aspx-vb',
+ 'ashx' => 'aspx-vb',
+ 'ASM' => 'nasm',
+ 'asm' => 'nasm',
+ 'asmx' => 'aspx-vb',
+ 'aspx' => 'aspx-vb',
+ 'autodelegate' => 'myghty',
+ 'autohandler' => 'mason',
+ 'aux' => 'tex',
+ 'axd' => 'aspx-vb',
+ 'b' => 'brainfuck',
+ 'bas' => 'vb.net',
+ 'bf' => 'brainfuck',
+ 'bmx' => 'blitzmax',
+ 'c++' => 'cpp',
+ 'c++-objdump' => 'cpp-objdump',
+ 'cc' => 'cpp',
+ 'cfc' => 'cfm',
+ 'cfg' => 'ini',
+ 'cfml' => 'cfm',
+ 'cl' => 'common-lisp',
+ 'clj' => 'clojure',
+ 'cmd' => 'bat',
+ 'coffee' => 'coffee-script',
+ 'cs' => 'csharp',
+ 'csh' => 'tcsh',
+ 'cw' => 'redcode',
+ 'cxx' => 'cpp',
+ 'cxx-objdump' => 'cpp-objdump',
+ 'darcspatch' => 'dpatch',
+ 'def' => 'modula2',
+ 'dhandler' => 'mason',
+ 'di' => 'd',
+ 'duby' => 'rb',
+ 'dyl' => 'dylan',
+ 'ebuild' => 'bash',
+ 'eclass' => 'bash',
+ 'el' => 'common-lisp',
+ 'eps' => 'postscript',
+ 'erl' => 'erlang',
+ 'erl-sh' => 'erl',
+ 'f' => 'fortran',
+ 'f90' => 'fortran',
+ 'feature' => 'Cucumber',
+ 'fhtml' => 'velocity',
+ 'flx' => 'felix',
+ 'flxh' => 'felix',
+ 'frag' => 'glsl',
+ 'g' => 'antlr-ruby',
+ 'G' => 'antlr-ruby',
+ 'gdc' => 'gooddata-cl',
+ 'gemspec' => 'rb',
+ 'geo' => 'glsl',
+ 'GNUmakefile' => 'make',
+ 'h' => 'c',
+ 'h++' => 'cpp',
+ 'hh' => 'cpp',
+ 'hpp' => 'cpp',
+ 'hql' => 'sql',
+ 'hrl' => 'erlang',
+ 'hs' => 'haskell',
+ 'htaccess' => 'apacheconf',
+ 'htm' => 'html',
+ 'html' => 'html+evoque',
+ 'hxx' => 'cpp',
+ 'hy' => 'hybris',
+ 'hyb' => 'hybris',
+ 'ik' => 'ioke',
+ 'inc' => 'pov',
+ 'j' => 'objective-j',
+ 'jbst' => 'duel',
+ 'kid' => 'genshi',
+ 'ksh' => 'bash',
+ 'less' => 'css',
+ 'lgt' => 'logtalk',
+ 'lisp' => 'common-lisp',
+ 'll' => 'llvm',
+ 'm' => 'objective-c',
+ 'mak' => 'make',
+ 'Makefile' => 'make',
+ 'makefile' => 'make',
+ 'man' => 'groff',
+ 'mao' => 'mako',
+ 'mc' => 'mason',
+ 'md' => 'minid',
+ 'mhtml' => 'mason',
+ 'mi' => 'mason',
+ 'ml' => 'ocaml',
+ 'mli' => 'ocaml',
+ 'mll' => 'ocaml',
+ 'mly' => 'ocaml',
+ 'mm' => 'objective-c',
+ 'mo' => 'modelica',
+ 'mod' => 'modula2',
+ 'moo' => 'moocode',
+ 'mu' => 'mupad',
+ 'myt' => 'myghty',
+ 'ns2' => 'newspeak',
+ 'pas' => 'delphi',
+ 'patch' => 'diff',
+ 'phtml' => 'html+php',
+ 'pl' => 'prolog',
+ 'plot' => 'gnuplot',
+ 'plt' => 'gnuplot',
+ 'pm' => 'perl',
+ 'po' => 'pot',
+ 'pp' => 'puppet',
+ 'pro' => 'prolog',
+ 'proto' => 'protobuf',
+ 'ps' => 'postscript',
+ 'pxd' => 'cython',
+ 'pxi' => 'cython',
+ 'py' => 'python',
+ 'pyw' => 'python',
+ 'pyx' => 'cython',
+ 'R' => 'splus',
+ 'r' => 'rebol',
+ 'r3' => 'rebol',
+ 'rake' => 'rb',
+ 'Rakefile' => 'rb',
+ 'rbw' => 'rb',
+ 'rbx' => 'rb',
+ 'rest' => 'rst',
+ 'rl' => 'ragel-em',
+ 'robot' => 'robotframework',
+ 'Rout' => 'rconsole',
+ 'rss' => 'xml',
+ 's' => 'gas',
+ 'S' => 'splus',
+ 'sc' => 'python',
+ 'scm' => 'scheme',
+ 'SConscript' => 'python',
+ 'SConstruct' => 'python',
+ 'scss' => 'css',
+ 'sh' => 'bash',
+ 'sh-session' => 'console',
+ 'spt' => 'cheetah',
+ 'sqlite3-console' => 'sqlite3',
+ 'st' => 'smalltalk',
+ 'sv' => 'v',
+ 'tac' => 'python',
+ 'tmpl' => 'cheetah',
+ 'toc' => 'tex',
+ 'tpl' => 'smarty',
+ 'txt' => 'text',
+ 'vapi' => 'vala',
+ 'vb' => 'vb.net',
+ 'vert' => 'glsl',
+ 'vhd' => 'vhdl',
+ 'vimrc' => 'vim',
+ 'vm' => 'velocity',
+ 'weechatlog' => 'irc',
+ 'wlua' => 'lua',
+ 'wsdl' => 'xml',
+ 'xhtml' => 'html',
+ 'xml' => 'xml+evoque',
+ 'xqy' => 'xquery',
+ 'xsd' => 'xml',
+ 'xsl' => 'xslt',
+ 'xslt' => 'xml',
+ 'yml' => 'yaml',
+ );
+
+ return idx($map, $language, $language);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php
@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Highlights source code with a rainbow of colors, regardless of the language.
+ * This highlighter is useless, absurd, and extremely slow.
+ */
+final class PhutilRainbowSyntaxHighlighter extends Phobject {
+
+ private $config = array();
+
+ public function setConfig($key, $value) {
+ $this->config[$key] = $value;
+ return $this;
+ }
+
+ public function getHighlightFuture($source) {
+
+ $color = 0;
+ $colors = array(
+ 'rbw_r',
+ 'rbw_o',
+ 'rbw_y',
+ 'rbw_g',
+ 'rbw_b',
+ 'rbw_i',
+ 'rbw_v',
+ );
+
+ $result = array();
+ foreach (phutil_utf8v($source) as $character) {
+ if ($character == ' ' || $character == "\n") {
+ $result[] = $character;
+ continue;
+ }
+ $result[] = phutil_tag(
+ 'span',
+ array('class' => $colors[$color]),
+ $character);
+ $color = ($color + 1) % count($colors);
+ }
+
+ $result = phutil_implode_html('', $result);
+ return new ImmediateFuture($result);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighter.php
@@ -0,0 +1,6 @@
+<?php
+
+abstract class PhutilSyntaxHighlighter extends Phobject {
+ abstract public function setConfig($key, $value);
+ abstract public function getHighlightFuture($source);
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilSyntaxHighlighterException.php
@@ -0,0 +1,3 @@
+<?php
+
+final class PhutilSyntaxHighlighterException extends Exception {}
diff --git a/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php b/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php
@@ -0,0 +1,18 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighter extends Phobject {
+
+ public function getHighlightFuture($source) {
+ $scrub = false;
+ if (strpos($source, '<?') === false) {
+ $source = "<?php\n".$source;
+ $scrub = true;
+ }
+
+ return new PhutilXHPASTSyntaxHighlighterFuture(
+ PhutilXHPASTBinary::getParserFuture($source),
+ $source,
+ $scrub);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php
@@ -0,0 +1,24 @@
+<?php
+
+final class PhutilJSONFragmentLexerHighlighterTestCase extends PhutilTestCase {
+
+ public function testLexer() {
+ $highlighter = id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('language', 'json')
+ ->setConfig('lexer', new PhutilJSONFragmentLexer());
+
+ $path = dirname(__FILE__).'/data/jsonfragment/';
+ foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) {
+ if (preg_match('/.test$/', $f)) {
+ $expect = preg_replace('/.test$/', '.expect', $f);
+ $source = Filesystem::readFile($path.'/'.$f);
+
+ $this->assertEqual(
+ Filesystem::readFile($path.'/'.$expect),
+ (string)$highlighter->getHighlightFuture($source)->resolve(),
+ $f);
+ }
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php
@@ -0,0 +1,25 @@
+<?php
+
+final class PhutilPHPFragmentLexerHighlighterTestCase extends PhutilTestCase {
+
+ public function testLexer() {
+ $highlighter = new PhutilLexerSyntaxHighlighter();
+ $highlighter->setConfig('language', 'php');
+ $highlighter->setConfig('lexer', new PhutilPHPFragmentLexer());
+
+
+ $path = dirname(__FILE__).'/phpfragment/';
+ foreach (Filesystem::listDirectory($path, $include_hidden = false) as $f) {
+ if (preg_match('/.test$/', $f)) {
+ $expect = preg_replace('/.test$/', '.expect', $f);
+ $source = Filesystem::readFile($path.'/'.$f);
+
+ $this->assertEqual(
+ Filesystem::readFile($path.'/'.$expect),
+ (string)$highlighter->getHighlightFuture($source)->resolve(),
+ $f);
+ }
+ }
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php
@@ -0,0 +1,39 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighterTestCase extends PhutilTestCase {
+
+ private function highlight($source) {
+ $highlighter = new PhutilXHPASTSyntaxHighlighter();
+ $future = $highlighter->getHighlightFuture($source);
+ return $future->resolve();
+ }
+
+ private function read($file) {
+ $path = dirname(__FILE__).'/xhpast/'.$file;
+ return Filesystem::readFile($path);
+ }
+
+ public function testBuiltinClassnames() {
+ $this->assertEqual(
+ $this->read('builtin-classname.expect'),
+ (string)$this->highlight($this->read('builtin-classname.source')),
+ pht('Builtin classnames should not be marked as linkable symbols.'));
+ $this->assertEqual(
+ rtrim($this->read('trailing-comment.expect')),
+ (string)$this->highlight($this->read('trailing-comment.source')),
+ pht('Trailing comments should not be dropped.'));
+ $this->assertEqual(
+ $this->read('multiline-token.expect'),
+ (string)$this->highlight($this->read('multiline-token.source')),
+ pht('Multi-line tokens should be split across lines.'));
+ $this->assertEqual(
+ $this->read('leading-whitespace.expect'),
+ (string)$this->highlight($this->read('leading-whitespace.source')),
+ pht('Snippets with leading whitespace should be preserved.'));
+ $this->assertEqual(
+ $this->read('no-leading-whitespace.expect'),
+ (string)$this->highlight($this->read('no-leading-whitespace.source')),
+ pht('Snippets with no leading whitespace should be preserved.'));
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.expect
@@ -0,0 +1,12 @@
+<span class="o">{</span>
+ <span class="s">"key"</span><span class="o">:</span> <span class="mf">3.5</span><span class="o">,</span>
+ <span class="s">"true"</span><span class="o">:</span> <span class="k">true</span><span class="o">,</span>
+ <span class="s">"false"</span><span class="o">:</span> <span class="k">false</span><span class="o">,</span>
+ <span class="s">"null"</span><span class="o">:</span> <span class="k">null</span><span class="o">,</span>
+ <span class="s">"list"</span><span class="o">:</span> <span class="o">[</span><span class="mf">1</span><span class="o">,</span> <span class="mf">2</span><span class="o">,</span> <span class="mf">3</span><span class="o">],</span>
+ <span class="s">"object"</span><span class="o">:</span> <span class="o">{</span>
+ <span class="s">"k1"</span><span class="o">:</span> <span class="s">"v1"</span>
+ <span class="o">},</span>
+ <span class="s">"numbers"</span><span class="o">:</span> <span class="o">[</span><span class="mf">0</span>e<span class="mf">1</span><span class="o">,</span> <span class="mf">1</span>e<span class="mf">-1</span><span class="o">,</span> <span class="mf">-1</span>e<span class="mf">-1</span><span class="o">,</span> <span class="mf">-1</span>e+<span class="mf">1</span><span class="o">],</span>
+ <span class="s">"</span><span class="k">\"\u1234</span><span class="s">'abc[]{}..."</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/data/jsonfragment/basics.test
@@ -0,0 +1,12 @@
+{
+ "key": 3.5,
+ "true": true,
+ "false": false,
+ "null": null,
+ "list": [1, 2, 3],
+ "object": {
+ "k1": "v1"
+ },
+ "numbers": [0e1, 1e-1, -1e-1, -1e+1],
+ "\"\u1234'abc[]{}..."
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.expect
@@ -0,0 +1,16 @@
+<span class="cp"><?</span>
+
+<span class="c">// comment? comment! </span><span class="cp">?></span>
+
+data
+
+<span class="cp"><?php</span>
+
+<span class="cp">__halt_compiler</span> <span class="cm">/* ! */</span> <span class="o">(</span> <span class="c">// )</span>
+<span class="o">)</span> <span class="cm">/* ;;;; */</span>
+
+<span class="o">;</span>
+
+data data
+<?php
+data
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/abuse.test
@@ -0,0 +1,16 @@
+<?
+
+// comment? comment! ?>
+
+data
+
+<?php
+
+__halt_compiler /* ! */ ( // )
+) /* ;;;; */
+
+;
+
+data data
+<?php
+data
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.expect
@@ -0,0 +1,5 @@
+<span class="k">public</span> <span class="k">function</span> <span class="no">f</span><span class="o">()</span> <span class="o">{</span>
+ <span class="nc" data-symbol-name="ExampleClass">ExampleClass</span><span class="o">::</span><span class="na" data-symbol-context="ExampleClass" data-symbol-name="EXAMPLE_CONSTANT">EXAMPLE_CONSTANT</span><span class="o">;</span>
+ <span class="nc" data-symbol-name="ExampleClass">ExampleClass</span><span class="o">::</span><span class="nf" data-symbol-context="ExampleClass" data-symbol-name="exampleMethod">exampleMethod</span><span class="o">();</span>
+ <span class="nf" data-symbol-name="example_function">example_function</span><span class="o">();</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/basics.test
@@ -0,0 +1,5 @@
+public function f() {
+ ExampleClass::EXAMPLE_CONSTANT;
+ ExampleClass::exampleMethod();
+ example_function();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.expect
@@ -0,0 +1,3 @@
+ <span class="k">foreach</span> <span class="o">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="o">)</span> <span class="o">{</span>
+ <span class="nf" data-symbol-name="z">z</span><span class="o">();</span>
+ <span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/leading-whitespace.test
@@ -0,0 +1,3 @@
+ foreach ($x as $y) {
+ z();
+ }
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.expect
@@ -0,0 +1,3 @@
+<span class="k">foreach</span> <span class="o">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="o">)</span> <span class="o">{</span>
+ <span class="nf" data-symbol-name="z">z</span><span class="o">();</span>
+<span class="o">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/phpfragment/no-leading-whitespace.test
@@ -0,0 +1,3 @@
+foreach ($x as $y) {
+ z();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.expect
@@ -0,0 +1,10 @@
+<span class="o"><?php</span>
+
+<span class="k">class</span> <span data-symbol-name="C" class="nc">C</span> <span class="k">{</span>
+ <span class="k">public</span> <span class="k">function</span> <span class="nx">f</span><span class="k">(</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="D" class="nc">D</span><span class="k">::</span><span data-symbol-context="D" data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="nx">self</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="nx">parent</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="k">static</span><span class="k">::</span><span data-symbol-name="X" class="na">X</span><span class="k">;</span>
+ <span class="k">}</span>
+<span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/builtin-classname.source
@@ -0,0 +1,10 @@
+<?php
+
+class C {
+ public function f() {
+ D::X;
+ self::X;
+ parent::X;
+ static::X;
+ }
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.expect
@@ -0,0 +1,3 @@
+ <span class="k">foreach</span> <span class="k">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="z" class="nf">z</span><span class="k">(</span><span class="k">)</span><span class="k">;</span>
+ <span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/leading-whitespace.source
@@ -0,0 +1,3 @@
+ foreach ($x as $y) {
+ z();
+ }
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.expect
@@ -0,0 +1,5 @@
+<span class="o"><?php</span>
+
+<span class="c">/* this comment
+</span><span class="c">extends across
+</span><span class="c">multiple lines */</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/multiline-token.source
@@ -0,0 +1,5 @@
+<?php
+
+/* this comment
+extends across
+multiple lines */
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.expect
@@ -0,0 +1,3 @@
+<span class="k">foreach</span> <span class="k">(</span><span class="nv">$x</span> <span class="k">as</span> <span class="nv">$y</span><span class="k">)</span> <span class="k">{</span>
+ <span data-symbol-name="z" class="nf">z</span><span class="k">(</span><span class="k">)</span><span class="k">;</span>
+<span class="k">}</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/no-leading-whitespace.source
@@ -0,0 +1,3 @@
+foreach ($x as $y) {
+ z();
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.expect
@@ -0,0 +1,3 @@
+<span class="o"><?php</span>
+<span class="c">// xyz
+</span>
diff --git a/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/__tests__/xhpast/trailing-comment.source
@@ -0,0 +1,2 @@
+<?php
+// xyz
diff --git a/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php b/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php
@@ -0,0 +1,33 @@
+<?php
+
+final class PhutilDefaultSyntaxHighlighterEnginePygmentsFuture
+ extends FutureProxy {
+
+ private $source;
+ private $scrub;
+
+ public function __construct(Future $proxied, $source, $scrub = false) {
+ parent::__construct($proxied);
+ $this->source = $source;
+ $this->scrub = $scrub;
+ }
+
+ protected function didReceiveResult($result) {
+ list($err, $stdout, $stderr) = $result;
+
+ if (!$err && strlen($stdout)) {
+ // Strip off fluff Pygments adds.
+ $stdout = preg_replace(
+ '@^<div class="highlight"><pre>(.*)</pre></div>\s*$@s',
+ '\1',
+ $stdout);
+ if ($this->scrub) {
+ $stdout = preg_replace('/^.*\n/', '', $stdout);
+ }
+ return phutil_safe_html($stdout);
+ }
+
+ throw new PhutilSyntaxHighlighterException($stderr, $err);
+ }
+
+}
diff --git a/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php
@@ -0,0 +1,262 @@
+<?php
+
+final class PhutilXHPASTSyntaxHighlighterFuture extends FutureProxy {
+
+ private $source;
+ private $scrub;
+
+ public function __construct(Future $proxied, $source, $scrub = false) {
+ parent::__construct($proxied);
+ $this->source = $source;
+ $this->scrub = $scrub;
+ }
+
+ protected function didReceiveResult($result) {
+ try {
+ return $this->applyXHPHighlight($result);
+ } catch (Exception $ex) {
+ // XHP can't highlight source that isn't syntactically valid. Fall back
+ // to the fragment lexer.
+ $source = ($this->scrub
+ ? preg_replace('/^.*\n/', '', $this->source)
+ : $this->source);
+ return id(new PhutilLexerSyntaxHighlighter())
+ ->setConfig('lexer', new PhutilPHPFragmentLexer())
+ ->setConfig('language', 'php')
+ ->getHighlightFuture($source)
+ ->resolve();
+ }
+ }
+
+ private function applyXHPHighlight($result) {
+
+ // We perform two passes here: one using the AST to find symbols we care
+ // about -- particularly, class names and function names. These are used
+ // in the crossreference stuff to link into Diffusion. After we've done our
+ // AST pass, we do a followup pass on the token stream to catch all the
+ // simple stuff like strings and comments.
+
+ $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
+ $this->source,
+ $result);
+
+ $root = $tree->getRootNode();
+
+ $tokens = $root->getTokens();
+ $interesting_symbols = $this->findInterestingSymbols($root);
+
+
+ if ($this->scrub) {
+ // If we're scrubbing, we prepended "<?php\n" to the text to force the
+ // highlighter to treat it as PHP source. Now, we need to remove that.
+
+ $ok = false;
+ if (count($tokens) >= 2) {
+ if ($tokens[0]->getTypeName() === 'T_OPEN_TAG') {
+ if ($tokens[1]->getTypeName() === 'T_WHITESPACE') {
+ $ok = true;
+ }
+ }
+ }
+
+ if (!$ok) {
+ throw new Exception(
+ pht(
+ 'Expected T_OPEN_TAG, T_WHITESPACE tokens at head of results '.
+ 'for highlighting parse of PHP snippet.'));
+ }
+
+ // Remove the "<?php".
+ unset($tokens[0]);
+
+ $value = $tokens[1]->getValue();
+ if ((strlen($value) < 1) || ($value[0] != "\n")) {
+ throw new Exception(
+ pht(
+ 'Expected "\\n" at beginning of T_WHITESPACE token at head of '.
+ 'tokens for highlighting parse of PHP snippet.'));
+ }
+
+ $value = substr($value, 1);
+ $tokens[1]->overwriteValue($value);
+ }
+
+ $out = array();
+ foreach ($tokens as $key => $token) {
+ $value = $token->getValue();
+ $class = null;
+ $multi = false;
+ $attrs = array();
+ if (isset($interesting_symbols[$key])) {
+ $sym = $interesting_symbols[$key];
+ $class = $sym[0];
+ $attrs['data-symbol-context'] = idx($sym, 'context');
+ $attrs['data-symbol-name'] = idx($sym, 'symbol');
+ } else {
+ switch ($token->getTypeName()) {
+ case 'T_WHITESPACE':
+ break;
+ case 'T_DOC_COMMENT':
+ $class = 'dc';
+ $multi = true;
+ break;
+ case 'T_COMMENT':
+ $class = 'c';
+ $multi = true;
+ break;
+ case 'T_CONSTANT_ENCAPSED_STRING':
+ case 'T_ENCAPSED_AND_WHITESPACE':
+ case 'T_INLINE_HTML':
+ $class = 's';
+ $multi = true;
+ break;
+ case 'T_VARIABLE':
+ $class = 'nv';
+ break;
+ case 'T_OPEN_TAG':
+ case 'T_OPEN_TAG_WITH_ECHO':
+ case 'T_CLOSE_TAG':
+ $class = 'o';
+ break;
+ case 'T_LNUMBER':
+ case 'T_DNUMBER':
+ $class = 'm';
+ break;
+ case 'T_STRING':
+ static $magic = array(
+ 'true' => true,
+ 'false' => true,
+ 'null' => true,
+ );
+ if (isset($magic[strtolower($value)])) {
+ $class = 'k';
+ break;
+ }
+ $class = 'nx';
+ break;
+ default:
+ $class = 'k';
+ break;
+ }
+ }
+
+ if ($class) {
+ $attrs['class'] = $class;
+ if ($multi) {
+ // If the token may have multiple lines in it, make sure each
+ // <span> crosses no more than one line so the lines can be put
+ // in a table, etc., later.
+ $value = phutil_split_lines($value, $retain_endings = true);
+ } else {
+ $value = array($value);
+ }
+ foreach ($value as $val) {
+ $out[] = phutil_tag('span', $attrs, $val);
+ }
+ } else {
+ $out[] = $value;
+ }
+ }
+
+ return phutil_implode_html('', $out);
+ }
+
+ private function findInterestingSymbols(XHPASTNode $root) {
+ // Class name symbols appear in:
+ // class X extends X implements X, X { ... }
+ // new X();
+ // $x instanceof X
+ // catch (X $x)
+ // function f(X $x)
+ // X::f();
+ // X::$m;
+ // X::CONST;
+
+ // These are PHP builtin tokens which can appear in a classname context.
+ // Don't link them since they don't go anywhere useful.
+ static $builtin_class_tokens = array(
+ 'self' => true,
+ 'parent' => true,
+ 'static' => true,
+ );
+
+ // Fortunately XHPAST puts all of these in a special node type so it's
+ // easy to find them.
+ $result_map = array();
+ $class_names = $root->selectDescendantsOfType('n_CLASS_NAME');
+ foreach ($class_names as $class_name) {
+ foreach ($class_name->getTokens() as $key => $token) {
+ if (isset($builtin_class_tokens[$token->getValue()])) {
+ // This is something like "self::method()".
+ continue;
+ }
+ $result_map[$key] = array(
+ 'nc', // "Name, Class"
+ 'symbol' => $class_name->getConcreteString(),
+ );
+ }
+ }
+
+ // Function name symbols appear in:
+ // f()
+
+ $function_calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
+ foreach ($function_calls as $call) {
+ $call = $call->getChildByIndex(0);
+ if ($call->getTypeName() == 'n_SYMBOL_NAME') {
+ // This is a normal function call, not some $f() shenanigans.
+ foreach ($call->getTokens() as $key => $token) {
+ $result_map[$key] = array(
+ 'nf', // "Name, Function"
+ 'symbol' => $call->getConcreteString(),
+ );
+ }
+ }
+ }
+
+ // Upon encountering $x->y, link y without context, since $x is unknown.
+
+ $prop_access = $root->selectDescendantsOfType('n_OBJECT_PROPERTY_ACCESS');
+ foreach ($prop_access as $access) {
+ $right = $access->getChildByIndex(1);
+ if ($right->getTypeName() == 'n_INDEX_ACCESS') {
+ // otherwise $x->y[0] doesn't get highlighted
+ $right = $right->getChildByIndex(0);
+ }
+ if ($right->getTypeName() == 'n_STRING') {
+ foreach ($right->getTokens() as $key => $token) {
+ $result_map[$key] = array(
+ 'na', // "Name, Attribute"
+ 'symbol' => $right->getConcreteString(),
+ );
+ }
+ }
+ }
+
+ // Upon encountering x::y, try to link y with context x.
+
+ $static_access = $root->selectDescendantsOfType('n_CLASS_STATIC_ACCESS');
+ foreach ($static_access as $access) {
+ $class = $access->getChildByIndex(0);
+ $right = $access->getChildByIndex(1);
+ if ($class->getTypeName() == 'n_CLASS_NAME' &&
+ ($right->getTypeName() == 'n_STRING' ||
+ $right->getTypeName() == 'n_VARIABLE')) {
+ $classname = head($class->getTokens())->getValue();
+ $result = array(
+ 'na',
+ 'symbol' => ltrim($right->getConcreteString(), '$'),
+ );
+ if (!isset($builtin_class_tokens[$classname])) {
+ $result['context'] = $classname;
+ }
+ foreach ($right->getTokens() as $key => $token) {
+ $result_map[$key] = $result;
+ }
+ }
+ }
+
+ return $result_map;
+ }
+
+}
diff --git a/src/infrastructure/parser/PhutilPygmentizeParser.php b/src/infrastructure/parser/PhutilPygmentizeParser.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/parser/PhutilPygmentizeParser.php
@@ -0,0 +1,83 @@
+<?php
+
+/**
+ * Parser that converts `pygmetize` output or similar HTML blocks from "class"
+ * attributes to "style" attributes.
+ */
+final class PhutilPygmentizeParser extends Phobject {
+
+ private $map = array();
+
+ public function setMap(array $map) {
+ $this->map = $map;
+ return $this;
+ }
+
+ public function getMap() {
+ return $this->map;
+ }
+
+ public function parse($block) {
+ $class_look = 'class="';
+ $class_len = strlen($class_look);
+
+ $class_start = null;
+
+ $map = $this->map;
+
+ $len = strlen($block);
+ $out = '';
+ $mode = 'text';
+ for ($ii = 0; $ii < $len; $ii++) {
+ $c = $block[$ii];
+ switch ($mode) {
+ case 'text':
+ // We're in general text between tags, and just passing characers
+ // through unmodified.
+ if ($c == '<') {
+ $mode = 'tag';
+ }
+ $out .= $c;
+ break;
+ case 'tag':
+ // We're inside a tag, and looking for `class="` so we can rewrite
+ // it.
+ if ($c == '>') {
+ $mode = 'text';
+ }
+ if ($c == 'c') {
+ if (!substr_compare($block, $class_look, $ii, $class_len)) {
+ $mode = 'class';
+ $ii += $class_len;
+ $class_start = $ii;
+ }
+ }
+
+ if ($mode != 'class') {
+ $out .= $c;
+ }
+ break;
+ case 'class':
+ // We're inside a `class="..."` tag, and looking for the ending quote
+ // so we can replace it.
+ if ($c == '"') {
+ $class = substr($block, $class_start, $ii - $class_start);
+
+ // If this class is present in the map, rewrite it into an inline
+ // style attribute.
+ if (isset($map[$class])) {
+ $out .= 'style="'.phutil_escape_html($map[$class]).'"';
+ } else {
+ $out .= 'class="'.$class.'"';
+ }
+
+ $mode = 'tag';
+ }
+ break;
+ }
+ }
+
+ return $out;
+ }
+
+}
diff --git a/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php b/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/parser/__tests__/PhutilPygmentizeParserTestCase.php
@@ -0,0 +1,43 @@
+<?php
+
+final class PhutilPygmentizeParserTestCase extends PhutilTestCase {
+
+ public function testPygmentizeParser() {
+ $this->tryParser(
+ '',
+ '',
+ array(),
+ pht('Empty'));
+
+ $this->tryParser(
+ '<span class="mi">1</span>',
+ '<span style="color: #ff0000">1</span>',
+ array(
+ 'mi' => 'color: #ff0000',
+ ),
+ pht('Simple'));
+
+ $this->tryParser(
+ '<span class="mi">1</span>',
+ '<span class="mi">1</span>',
+ array(),
+ pht('Missing Class'));
+
+ $this->tryParser(
+ '<span data-symbol-name="X" class="nc">X</span>',
+ '<span data-symbol-name="X" style="color: #ff0000">X</span>',
+ array(
+ 'nc' => 'color: #ff0000',
+ ),
+ pht('Extra Attribute'));
+ }
+
+ private function tryParser($input, $expect, array $map, $label) {
+ $actual = id(new PhutilPygmentizeParser())
+ ->setMap($map)
+ ->parse($input);
+
+ $this->assertEqual($expect, $actual, pht('Pygmentize Parser: %s', $label));
+ }
+
+}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Tue, Sep 2, 7:19 PM (19 h, 8 m)
Storage Engine
amazon-s3
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
phabricator/secure/qy/x2/z7xi2ayzx5gzn5ku
Default Alt Text
D20977.diff (144 KB)
Attached To
Mode
D20977: Continue moving classes with no callers in libphutil or Arcanist to Phabricator
Attached
Detach File
Event Timeline
Log In to Comment