diff --git a/scripts/daemon/launch_daemon.php b/scripts/daemon/launch_daemon.php index 3c3996a..4bee9fb 100755 --- a/scripts/daemon/launch_daemon.php +++ b/scripts/daemon/launch_daemon.php @@ -1,7 +1,9 @@ #!/usr/bin/env php run(); diff --git a/scripts/utils/prosediff.php b/scripts/utils/prosediff.php new file mode 100755 index 0000000..e02c9c8 --- /dev/null +++ b/scripts/utils/prosediff.php @@ -0,0 +1,50 @@ +#!/usr/bin/env php +setTagline(pht('show prose differences')); +$args->setSynopsis(<<parseStandardArguments(); +$args->parse( + array( + array( + 'name' => 'files', + 'wildcard' => true, + ), + )); + +$files = $args->getArg('files'); +if (count($files) !== 2) { + $args->printHelpAndExit(); +} +$old_file = head($files); +$new_file = last($files); + +$old_data = Filesystem::readFile($old_file); +$new_data = Filesystem::readFile($new_file); + +$engine = new PhutilProseDifferenceEngine(); + +$prose_diff = $engine->getDiff($old_data, $new_data); + +foreach ($prose_diff->getParts() as $part) { + switch ($part['type']) { + case '-': + echo tsprintf('%B', $part['text']); + break; + case '+': + echo tsprintf('%B', $part['text']); + break; + case '=': + echo tsprintf('%B', $part['text']); + break; + } +} diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index a6b3644..232dd11 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1,981 +1,989 @@ 2, 'class' => array( 'AASTNode' => 'parser/aast/api/AASTNode.php', 'AASTNodeList' => 'parser/aast/api/AASTNodeList.php', 'AASTToken' => 'parser/aast/api/AASTToken.php', 'AASTTree' => 'parser/aast/api/AASTTree.php', 'AbstractDirectedGraph' => 'utils/AbstractDirectedGraph.php', 'AbstractDirectedGraphTestCase' => 'utils/__tests__/AbstractDirectedGraphTestCase.php', 'AphrontAccessDeniedQueryException' => 'aphront/storage/exception/AphrontAccessDeniedQueryException.php', 'AphrontBaseMySQLDatabaseConnection' => 'aphront/storage/connection/mysql/AphrontBaseMySQLDatabaseConnection.php', 'AphrontCharacterSetQueryException' => 'aphront/storage/exception/AphrontCharacterSetQueryException.php', 'AphrontConnectionLostQueryException' => 'aphront/storage/exception/AphrontConnectionLostQueryException.php', 'AphrontConnectionQueryException' => 'aphront/storage/exception/AphrontConnectionQueryException.php', 'AphrontCountQueryException' => 'aphront/storage/exception/AphrontCountQueryException.php', 'AphrontDatabaseConnection' => 'aphront/storage/connection/AphrontDatabaseConnection.php', 'AphrontDatabaseTransactionState' => 'aphront/storage/connection/AphrontDatabaseTransactionState.php', 'AphrontDeadlockQueryException' => 'aphront/storage/exception/AphrontDeadlockQueryException.php', 'AphrontDuplicateKeyQueryException' => 'aphront/storage/exception/AphrontDuplicateKeyQueryException.php', 'AphrontInvalidCredentialsQueryException' => 'aphront/storage/exception/AphrontInvalidCredentialsQueryException.php', 'AphrontIsolatedDatabaseConnection' => 'aphront/storage/connection/AphrontIsolatedDatabaseConnection.php', 'AphrontLockTimeoutQueryException' => 'aphront/storage/exception/AphrontLockTimeoutQueryException.php', 'AphrontMySQLDatabaseConnection' => 'aphront/storage/connection/mysql/AphrontMySQLDatabaseConnection.php', 'AphrontMySQLiDatabaseConnection' => 'aphront/storage/connection/mysql/AphrontMySQLiDatabaseConnection.php', 'AphrontNotSupportedQueryException' => 'aphront/storage/exception/AphrontNotSupportedQueryException.php', 'AphrontObjectMissingQueryException' => 'aphront/storage/exception/AphrontObjectMissingQueryException.php', 'AphrontParameterQueryException' => 'aphront/storage/exception/AphrontParameterQueryException.php', 'AphrontQueryException' => 'aphront/storage/exception/AphrontQueryException.php', 'AphrontQueryTimeoutQueryException' => 'aphront/storage/exception/AphrontQueryTimeoutQueryException.php', 'AphrontRecoverableQueryException' => 'aphront/storage/exception/AphrontRecoverableQueryException.php', 'AphrontRequestStream' => 'aphront/requeststream/AphrontRequestStream.php', 'AphrontSchemaQueryException' => 'aphront/storage/exception/AphrontSchemaQueryException.php', 'AphrontScopedUnguardedWriteCapability' => 'aphront/writeguard/AphrontScopedUnguardedWriteCapability.php', 'AphrontWriteGuard' => 'aphront/writeguard/AphrontWriteGuard.php', 'BaseHTTPFuture' => 'future/http/BaseHTTPFuture.php', 'CaseInsensitiveArray' => 'utils/CaseInsensitiveArray.php', 'CaseInsensitiveArrayTestCase' => 'utils/__tests__/CaseInsensitiveArrayTestCase.php', 'CommandException' => 'future/exec/CommandException.php', 'ConduitClient' => 'conduit/ConduitClient.php', 'ConduitClientException' => 'conduit/ConduitClientException.php', 'ConduitClientTestCase' => 'conduit/__tests__/ConduitClientTestCase.php', 'ConduitFuture' => 'conduit/ConduitFuture.php', 'ExecFuture' => 'future/exec/ExecFuture.php', 'ExecFutureTestCase' => 'future/exec/__tests__/ExecFutureTestCase.php', 'ExecPassthruTestCase' => 'future/exec/__tests__/ExecPassthruTestCase.php', 'FileFinder' => 'filesystem/FileFinder.php', 'FileFinderTestCase' => 'filesystem/__tests__/FileFinderTestCase.php', 'FileList' => 'filesystem/FileList.php', 'Filesystem' => 'filesystem/Filesystem.php', 'FilesystemException' => 'filesystem/FilesystemException.php', 'FilesystemTestCase' => 'filesystem/__tests__/FilesystemTestCase.php', 'Future' => 'future/Future.php', 'FutureIterator' => 'future/FutureIterator.php', 'FutureIteratorTestCase' => 'future/__tests__/FutureIteratorTestCase.php', 'FutureProxy' => 'future/FutureProxy.php', 'HTTPFuture' => 'future/http/HTTPFuture.php', 'HTTPFutureCURLResponseStatus' => 'future/http/status/HTTPFutureCURLResponseStatus.php', 'HTTPFutureCertificateResponseStatus' => 'future/http/status/HTTPFutureCertificateResponseStatus.php', 'HTTPFutureHTTPResponseStatus' => 'future/http/status/HTTPFutureHTTPResponseStatus.php', 'HTTPFutureParseResponseStatus' => 'future/http/status/HTTPFutureParseResponseStatus.php', 'HTTPFutureResponseStatus' => 'future/http/status/HTTPFutureResponseStatus.php', 'HTTPFutureTransportResponseStatus' => 'future/http/status/HTTPFutureTransportResponseStatus.php', 'HTTPSFuture' => 'future/http/HTTPSFuture.php', 'ImmediateFuture' => 'future/ImmediateFuture.php', 'LibphutilUSEnglishTranslation' => 'internationalization/translation/LibphutilUSEnglishTranslation.php', 'LinesOfALarge' => 'filesystem/linesofalarge/LinesOfALarge.php', 'LinesOfALargeExecFuture' => 'filesystem/linesofalarge/LinesOfALargeExecFuture.php', 'LinesOfALargeExecFutureTestCase' => 'filesystem/linesofalarge/__tests__/LinesOfALargeExecFutureTestCase.php', 'LinesOfALargeFile' => 'filesystem/linesofalarge/LinesOfALargeFile.php', 'LinesOfALargeFileTestCase' => 'filesystem/linesofalarge/__tests__/LinesOfALargeFileTestCase.php', 'MFilterTestHelper' => 'utils/__tests__/MFilterTestHelper.php', 'PHPASTParserTestCase' => 'parser/xhpast/__tests__/PHPASTParserTestCase.php', 'PhageAgentBootloader' => 'phage/bootloader/PhageAgentBootloader.php', 'PhageAgentTestCase' => 'phage/__tests__/PhageAgentTestCase.php', 'PhagePHPAgent' => 'phage/agent/PhagePHPAgent.php', 'PhagePHPAgentBootloader' => 'phage/bootloader/PhagePHPAgentBootloader.php', 'Phobject' => 'object/Phobject.php', 'PhobjectTestCase' => 'object/__tests__/PhobjectTestCase.php', 'PhutilAPCKeyValueCache' => 'cache/PhutilAPCKeyValueCache.php', 'PhutilAWSEC2Future' => 'future/aws/PhutilAWSEC2Future.php', 'PhutilAWSException' => 'future/aws/PhutilAWSException.php', 'PhutilAWSFuture' => 'future/aws/PhutilAWSFuture.php', 'PhutilAWSManagementWorkflow' => 'future/aws/management/PhutilAWSManagementWorkflow.php', 'PhutilAWSS3DeleteManagementWorkflow' => 'future/aws/management/PhutilAWSS3DeleteManagementWorkflow.php', 'PhutilAWSS3Future' => 'future/aws/PhutilAWSS3Future.php', 'PhutilAWSS3GetManagementWorkflow' => 'future/aws/management/PhutilAWSS3GetManagementWorkflow.php', 'PhutilAWSS3ManagementWorkflow' => 'future/aws/management/PhutilAWSS3ManagementWorkflow.php', 'PhutilAWSS3PutManagementWorkflow' => 'future/aws/management/PhutilAWSS3PutManagementWorkflow.php', 'PhutilAWSv4Signature' => 'future/aws/PhutilAWSv4Signature.php', 'PhutilAWSv4SignatureTestCase' => 'future/aws/__tests__/PhutilAWSv4SignatureTestCase.php', 'PhutilAggregateException' => 'error/PhutilAggregateException.php', 'PhutilAllCapsEnglishLocale' => 'internationalization/locales/PhutilAllCapsEnglishLocale.php', 'PhutilAmazonAuthAdapter' => 'auth/PhutilAmazonAuthAdapter.php', 'PhutilArgumentParser' => 'parser/argument/PhutilArgumentParser.php', 'PhutilArgumentParserException' => 'parser/argument/exception/PhutilArgumentParserException.php', 'PhutilArgumentParserTestCase' => 'parser/argument/__tests__/PhutilArgumentParserTestCase.php', 'PhutilArgumentSpecification' => 'parser/argument/PhutilArgumentSpecification.php', 'PhutilArgumentSpecificationException' => 'parser/argument/exception/PhutilArgumentSpecificationException.php', 'PhutilArgumentSpecificationTestCase' => 'parser/argument/__tests__/PhutilArgumentSpecificationTestCase.php', 'PhutilArgumentUsageException' => 'parser/argument/exception/PhutilArgumentUsageException.php', 'PhutilArgumentWorkflow' => 'parser/argument/workflow/PhutilArgumentWorkflow.php', 'PhutilArray' => 'utils/PhutilArray.php', 'PhutilArrayTestCase' => 'utils/__tests__/PhutilArrayTestCase.php', 'PhutilArrayWithDefaultValue' => 'utils/PhutilArrayWithDefaultValue.php', 'PhutilAsanaAuthAdapter' => 'auth/PhutilAsanaAuthAdapter.php', 'PhutilAsanaFuture' => 'future/asana/PhutilAsanaFuture.php', 'PhutilAuthAdapter' => 'auth/PhutilAuthAdapter.php', 'PhutilAuthConfigurationException' => 'auth/exception/PhutilAuthConfigurationException.php', 'PhutilAuthCredentialException' => 'auth/exception/PhutilAuthCredentialException.php', 'PhutilAuthException' => 'auth/exception/PhutilAuthException.php', 'PhutilAuthUserAbortedException' => 'auth/exception/PhutilAuthUserAbortedException.php', 'PhutilBallOfPHP' => 'phage/util/PhutilBallOfPHP.php', 'PhutilBitbucketAuthAdapter' => 'auth/PhutilBitbucketAuthAdapter.php', 'PhutilBootloader' => 'moduleutils/PhutilBootloader.php', 'PhutilBootloaderException' => 'moduleutils/PhutilBootloaderException.php', 'PhutilBritishEnglishLocale' => 'internationalization/locales/PhutilBritishEnglishLocale.php', 'PhutilBufferedIterator' => 'utils/PhutilBufferedIterator.php', 'PhutilBufferedIteratorTestCase' => 'utils/__tests__/PhutilBufferedIteratorTestCase.php', 'PhutilBugtraqParser' => 'parser/PhutilBugtraqParser.php', 'PhutilBugtraqParserTestCase' => 'parser/__tests__/PhutilBugtraqParserTestCase.php', 'PhutilCIDRBlock' => 'ip/PhutilCIDRBlock.php', 'PhutilCIDRList' => 'ip/PhutilCIDRList.php', 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'grammar/code/PhutilCLikeCodeSnippetContextFreeGrammar.php', 'PhutilCallbackFilterIterator' => 'utils/PhutilCallbackFilterIterator.php', 'PhutilChannel' => 'channel/PhutilChannel.php', 'PhutilChannelChannel' => 'channel/PhutilChannelChannel.php', 'PhutilChannelTestCase' => 'channel/__tests__/PhutilChannelTestCase.php', 'PhutilChunkedIterator' => 'utils/PhutilChunkedIterator.php', 'PhutilChunkedIteratorTestCase' => 'utils/__tests__/PhutilChunkedIteratorTestCase.php', 'PhutilClassMapQuery' => 'symbols/PhutilClassMapQuery.php', 'PhutilCodeSnippetContextFreeGrammar' => 'grammar/code/PhutilCodeSnippetContextFreeGrammar.php', 'PhutilCommandString' => 'xsprintf/PhutilCommandString.php', 'PhutilConsole' => 'console/PhutilConsole.php', 'PhutilConsoleBlock' => 'console/view/PhutilConsoleBlock.php', 'PhutilConsoleConcatenatedView' => 'console/view/PhutilConsoleConcatenatedView.php', 'PhutilConsoleFormatter' => 'console/PhutilConsoleFormatter.php', 'PhutilConsoleList' => 'console/view/PhutilConsoleList.php', 'PhutilConsoleMessage' => 'console/PhutilConsoleMessage.php', 'PhutilConsoleProgressBar' => 'console/PhutilConsoleProgressBar.php', 'PhutilConsoleServer' => 'console/PhutilConsoleServer.php', 'PhutilConsoleServerChannel' => 'console/PhutilConsoleServerChannel.php', 'PhutilConsoleStdinNotInteractiveException' => 'console/PhutilConsoleStdinNotInteractiveException.php', 'PhutilConsoleSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilConsoleSyntaxHighlighter.php', 'PhutilConsoleTable' => 'console/view/PhutilConsoleTable.php', 'PhutilConsoleView' => 'console/view/PhutilConsoleView.php', 'PhutilConsoleWrapTestCase' => 'console/__tests__/PhutilConsoleWrapTestCase.php', 'PhutilContextFreeGrammar' => 'grammar/PhutilContextFreeGrammar.php', 'PhutilCowsay' => 'utils/PhutilCowsay.php', 'PhutilCowsayTestCase' => 'utils/__tests__/PhutilCowsayTestCase.php', 'PhutilCsprintfTestCase' => 'xsprintf/__tests__/PhutilCsprintfTestCase.php', 'PhutilCzechLocale' => 'internationalization/locales/PhutilCzechLocale.php', 'PhutilDaemon' => 'daemon/PhutilDaemon.php', 'PhutilDaemonHandle' => 'daemon/PhutilDaemonHandle.php', 'PhutilDaemonOverseer' => 'daemon/PhutilDaemonOverseer.php', 'PhutilDaemonOverseerModule' => 'daemon/PhutilDaemonOverseerModule.php', 'PhutilDefaultSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilDefaultSyntaxHighlighter.php', 'PhutilDefaultSyntaxHighlighterEngine' => 'markup/syntax/engine/PhutilDefaultSyntaxHighlighterEngine.php', 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'markup/syntax/highlighter/pygments/PhutilDefaultSyntaxHighlighterEnginePygmentsFuture.php', 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'markup/syntax/engine/__tests__/PhutilDefaultSyntaxHighlighterEngineTestCase.php', 'PhutilDeferredLog' => 'filesystem/PhutilDeferredLog.php', 'PhutilDeferredLogTestCase' => 'filesystem/__tests__/PhutilDeferredLogTestCase.php', 'PhutilDirectedScalarGraph' => 'utils/PhutilDirectedScalarGraph.php', 'PhutilDirectoryFixture' => 'filesystem/PhutilDirectoryFixture.php', 'PhutilDirectoryKeyValueCache' => 'cache/PhutilDirectoryKeyValueCache.php', 'PhutilDisqusAuthAdapter' => 'auth/PhutilDisqusAuthAdapter.php', 'PhutilDivinerSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilDivinerSyntaxHighlighter.php', 'PhutilDocblockParser' => 'parser/PhutilDocblockParser.php', 'PhutilDocblockParserTestCase' => 'parser/__tests__/PhutilDocblockParserTestCase.php', 'PhutilEditDistanceMatrix' => 'utils/PhutilEditDistanceMatrix.php', 'PhutilEditDistanceMatrixTestCase' => 'utils/__tests__/PhutilEditDistanceMatrixTestCase.php', 'PhutilEditorConfig' => 'parser/PhutilEditorConfig.php', 'PhutilEditorConfigTestCase' => 'parser/__tests__/PhutilEditorConfigTestCase.php', 'PhutilEmailAddress' => 'parser/PhutilEmailAddress.php', 'PhutilEmailAddressTestCase' => 'parser/__tests__/PhutilEmailAddressTestCase.php', 'PhutilEmojiLocale' => 'internationalization/locales/PhutilEmojiLocale.php', 'PhutilEmptyAuthAdapter' => 'auth/PhutilEmptyAuthAdapter.php', 'PhutilEnglishCanadaLocale' => 'internationalization/locales/PhutilEnglishCanadaLocale.php', 'PhutilErrorHandler' => 'error/PhutilErrorHandler.php', 'PhutilErrorHandlerTestCase' => 'error/__tests__/PhutilErrorHandlerTestCase.php', 'PhutilErrorTrap' => 'error/PhutilErrorTrap.php', 'PhutilEvent' => 'events/PhutilEvent.php', 'PhutilEventConstants' => 'events/constant/PhutilEventConstants.php', 'PhutilEventEngine' => 'events/PhutilEventEngine.php', 'PhutilEventListener' => 'events/PhutilEventListener.php', 'PhutilEventType' => 'events/constant/PhutilEventType.php', 'PhutilExampleBufferedIterator' => 'utils/PhutilExampleBufferedIterator.php', 'PhutilExcessiveServiceCallsDaemon' => 'daemon/torture/PhutilExcessiveServiceCallsDaemon.php', 'PhutilExecChannel' => 'channel/PhutilExecChannel.php', 'PhutilExecPassthru' => 'future/exec/PhutilExecPassthru.php', 'PhutilExecutableFuture' => 'future/exec/PhutilExecutableFuture.php', 'PhutilExecutionEnvironment' => 'utils/PhutilExecutionEnvironment.php', 'PhutilExtensionsTestCase' => 'moduleutils/__tests__/PhutilExtensionsTestCase.php', 'PhutilFacebookAuthAdapter' => 'auth/PhutilFacebookAuthAdapter.php', 'PhutilFatalDaemon' => 'daemon/torture/PhutilFatalDaemon.php', 'PhutilFileLock' => 'filesystem/PhutilFileLock.php', 'PhutilFileLockTestCase' => 'filesystem/__tests__/PhutilFileLockTestCase.php', 'PhutilFileTree' => 'filesystem/PhutilFileTree.php', 'PhutilFrenchLocale' => 'internationalization/locales/PhutilFrenchLocale.php', 'PhutilGermanLocale' => 'internationalization/locales/PhutilGermanLocale.php', 'PhutilGitHubAuthAdapter' => 'auth/PhutilGitHubAuthAdapter.php', 'PhutilGitHubFuture' => 'future/github/PhutilGitHubFuture.php', 'PhutilGitHubResponse' => 'future/github/PhutilGitHubResponse.php', 'PhutilGitURI' => 'parser/PhutilGitURI.php', 'PhutilGitURITestCase' => 'parser/__tests__/PhutilGitURITestCase.php', 'PhutilGoogleAuthAdapter' => 'auth/PhutilGoogleAuthAdapter.php', + 'PhutilHTTPEngineExtension' => 'future/http/PhutilHTTPEngineExtension.php', 'PhutilHangForeverDaemon' => 'daemon/torture/PhutilHangForeverDaemon.php', 'PhutilHashingIterator' => 'utils/PhutilHashingIterator.php', 'PhutilHashingIteratorTestCase' => 'utils/__tests__/PhutilHashingIteratorTestCase.php', 'PhutilHelpArgumentWorkflow' => 'parser/argument/workflow/PhutilHelpArgumentWorkflow.php', 'PhutilHgsprintfTestCase' => 'xsprintf/__tests__/PhutilHgsprintfTestCase.php', 'PhutilHighIntensityIntervalDaemon' => 'daemon/torture/PhutilHighIntensityIntervalDaemon.php', 'PhutilINIParserException' => 'parser/exception/PhutilINIParserException.php', 'PhutilIPAddress' => 'ip/PhutilIPAddress.php', 'PhutilIPAddressTestCase' => 'ip/__tests__/PhutilIPAddressTestCase.php', 'PhutilInRequestKeyValueCache' => 'cache/PhutilInRequestKeyValueCache.php', 'PhutilInteractiveEditor' => 'console/PhutilInteractiveEditor.php', 'PhutilInvalidRuleParserGeneratorException' => 'parser/generator/exception/PhutilInvalidRuleParserGeneratorException.php', 'PhutilInvalidStateException' => 'exception/PhutilInvalidStateException.php', 'PhutilInvalidStateExceptionTestCase' => 'exception/__tests__/PhutilInvalidStateExceptionTestCase.php', 'PhutilInvisibleSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilInvisibleSyntaxHighlighter.php', 'PhutilIrreducibleRuleParserGeneratorException' => 'parser/generator/exception/PhutilIrreducibleRuleParserGeneratorException.php', 'PhutilJIRAAuthAdapter' => 'auth/PhutilJIRAAuthAdapter.php', 'PhutilJSON' => 'parser/PhutilJSON.php', 'PhutilJSONFragmentLexer' => 'lexer/PhutilJSONFragmentLexer.php', 'PhutilJSONFragmentLexerHighlighterTestCase' => 'markup/syntax/highlighter/__tests__/PhutilJSONFragmentLexerHighlighterTestCase.php', 'PhutilJSONParser' => 'parser/PhutilJSONParser.php', 'PhutilJSONParserException' => 'parser/exception/PhutilJSONParserException.php', 'PhutilJSONParserTestCase' => 'parser/__tests__/PhutilJSONParserTestCase.php', 'PhutilJSONProtocolChannel' => 'channel/PhutilJSONProtocolChannel.php', 'PhutilJSONProtocolChannelTestCase' => 'channel/__tests__/PhutilJSONProtocolChannelTestCase.php', 'PhutilJSONTestCase' => 'parser/__tests__/PhutilJSONTestCase.php', 'PhutilJavaCodeSnippetContextFreeGrammar' => 'grammar/code/PhutilJavaCodeSnippetContextFreeGrammar.php', 'PhutilKeyValueCache' => 'cache/PhutilKeyValueCache.php', 'PhutilKeyValueCacheNamespace' => 'cache/PhutilKeyValueCacheNamespace.php', 'PhutilKeyValueCacheProfiler' => 'cache/PhutilKeyValueCacheProfiler.php', 'PhutilKeyValueCacheProxy' => 'cache/PhutilKeyValueCacheProxy.php', 'PhutilKeyValueCacheStack' => 'cache/PhutilKeyValueCacheStack.php', 'PhutilKeyValueCacheTestCase' => 'cache/__tests__/PhutilKeyValueCacheTestCase.php', 'PhutilKoreanLocale' => 'internationalization/locales/PhutilKoreanLocale.php', 'PhutilLDAPAuthAdapter' => 'auth/PhutilLDAPAuthAdapter.php', 'PhutilLanguageGuesser' => 'parser/PhutilLanguageGuesser.php', 'PhutilLanguageGuesserTestCase' => 'parser/__tests__/PhutilLanguageGuesserTestCase.php', 'PhutilLexer' => 'lexer/PhutilLexer.php', 'PhutilLexerSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilLexerSyntaxHighlighter.php', 'PhutilLibraryConflictException' => 'moduleutils/PhutilLibraryConflictException.php', 'PhutilLibraryMapBuilder' => 'moduleutils/PhutilLibraryMapBuilder.php', 'PhutilLibraryTestCase' => '__tests__/PhutilLibraryTestCase.php', 'PhutilLipsumContextFreeGrammar' => 'grammar/PhutilLipsumContextFreeGrammar.php', 'PhutilLocale' => 'internationalization/PhutilLocale.php', 'PhutilLocaleTestCase' => 'internationalization/__tests__/PhutilLocaleTestCase.php', 'PhutilLock' => 'filesystem/PhutilLock.php', 'PhutilLockException' => 'filesystem/PhutilLockException.php', 'PhutilLogFileChannel' => 'channel/PhutilLogFileChannel.php', 'PhutilLunarPhase' => 'utils/PhutilLunarPhase.php', 'PhutilLunarPhaseTestCase' => 'utils/__tests__/PhutilLunarPhaseTestCase.php', 'PhutilMarkupEngine' => 'markup/PhutilMarkupEngine.php', 'PhutilMarkupTestCase' => 'markup/__tests__/PhutilMarkupTestCase.php', 'PhutilMemcacheKeyValueCache' => 'cache/PhutilMemcacheKeyValueCache.php', 'PhutilMethodNotImplementedException' => 'error/PhutilMethodNotImplementedException.php', 'PhutilMetricsChannel' => 'channel/PhutilMetricsChannel.php', 'PhutilMissingSymbolException' => 'symbols/exception/PhutilMissingSymbolException.php', 'PhutilModuleUtilsTestCase' => 'moduleutils/__tests__/PhutilModuleUtilsTestCase.php', 'PhutilNiceDaemon' => 'daemon/torture/PhutilNiceDaemon.php', 'PhutilNumber' => 'internationalization/PhutilNumber.php', 'PhutilOAuth1AuthAdapter' => 'auth/PhutilOAuth1AuthAdapter.php', 'PhutilOAuth1Future' => 'future/oauth/PhutilOAuth1Future.php', 'PhutilOAuth1FutureTestCase' => 'future/oauth/__tests__/PhutilOAuth1FutureTestCase.php', 'PhutilOAuthAuthAdapter' => 'auth/PhutilOAuthAuthAdapter.php', 'PhutilOnDiskKeyValueCache' => 'cache/PhutilOnDiskKeyValueCache.php', 'PhutilOpaqueEnvelope' => 'error/PhutilOpaqueEnvelope.php', 'PhutilOpaqueEnvelopeKey' => 'error/PhutilOpaqueEnvelopeKey.php', 'PhutilOpaqueEnvelopeTestCase' => 'error/__tests__/PhutilOpaqueEnvelopeTestCase.php', 'PhutilPHPCodeSnippetContextFreeGrammar' => 'grammar/code/PhutilPHPCodeSnippetContextFreeGrammar.php', 'PhutilPHPFragmentLexer' => 'lexer/PhutilPHPFragmentLexer.php', 'PhutilPHPFragmentLexerHighlighterTestCase' => 'markup/syntax/highlighter/__tests__/PhutilPHPFragmentLexerHighlighterTestCase.php', 'PhutilPHPFragmentLexerTestCase' => 'lexer/__tests__/PhutilPHPFragmentLexerTestCase.php', 'PhutilPHPObjectProtocolChannel' => 'channel/PhutilPHPObjectProtocolChannel.php', 'PhutilPHPObjectProtocolChannelTestCase' => 'channel/__tests__/PhutilPHPObjectProtocolChannelTestCase.php', 'PhutilParserGenerator' => 'parser/PhutilParserGenerator.php', 'PhutilParserGeneratorException' => 'parser/generator/exception/PhutilParserGeneratorException.php', 'PhutilParserGeneratorTestCase' => 'parser/__tests__/PhutilParserGeneratorTestCase.php', 'PhutilPayPalAPIFuture' => 'future/paypal/PhutilPayPalAPIFuture.php', 'PhutilPerson' => 'internationalization/PhutilPerson.php', 'PhutilPersonTest' => 'internationalization/__tests__/PhutilPersonTest.php', 'PhutilPersonaAuthAdapter' => 'auth/PhutilPersonaAuthAdapter.php', 'PhutilPhabricatorAuthAdapter' => 'auth/PhutilPhabricatorAuthAdapter.php', 'PhutilPhtTestCase' => 'internationalization/__tests__/PhutilPhtTestCase.php', 'PhutilPirateEnglishLocale' => 'internationalization/locales/PhutilPirateEnglishLocale.php', 'PhutilPortugueseBrazilLocale' => 'internationalization/locales/PhutilPortugueseBrazilLocale.php', 'PhutilPortuguesePortugalLocale' => 'internationalization/locales/PhutilPortuguesePortugalLocale.php', 'PhutilPregsprintfTestCase' => 'xsprintf/__tests__/PhutilPregsprintfTestCase.php', 'PhutilProcessGroupDaemon' => 'daemon/torture/PhutilProcessGroupDaemon.php', + 'PhutilProseDiff' => 'utils/PhutilProseDiff.php', + 'PhutilProseDiffTestCase' => 'utils/__tests__/PhutilProseDiffTestCase.php', + 'PhutilProseDifferenceEngine' => 'utils/PhutilProseDifferenceEngine.php', 'PhutilProtocolChannel' => 'channel/PhutilProtocolChannel.php', 'PhutilProxyException' => 'error/PhutilProxyException.php', 'PhutilProxyIterator' => 'utils/PhutilProxyIterator.php', 'PhutilPygmentizeParser' => 'parser/PhutilPygmentizeParser.php', 'PhutilPygmentizeParserTestCase' => 'parser/__tests__/PhutilPygmentizeParserTestCase.php', 'PhutilPygmentsSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilPygmentsSyntaxHighlighter.php', 'PhutilPythonFragmentLexer' => 'lexer/PhutilPythonFragmentLexer.php', 'PhutilQsprintfInterface' => 'xsprintf/PhutilQsprintfInterface.php', 'PhutilQueryStringParser' => 'parser/PhutilQueryStringParser.php', 'PhutilQueryStringParserTestCase' => 'parser/__tests__/PhutilQueryStringParserTestCase.php', 'PhutilRainbowSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilRainbowSyntaxHighlighter.php', 'PhutilRawEnglishLocale' => 'internationalization/locales/PhutilRawEnglishLocale.php', 'PhutilReadableSerializer' => 'readableserializer/PhutilReadableSerializer.php', 'PhutilReadableSerializerTestCase' => 'readableserializer/__tests__/PhutilReadableSerializerTestCase.php', 'PhutilRealNameContextFreeGrammar' => 'grammar/PhutilRealNameContextFreeGrammar.php', 'PhutilRemarkupBlockInterpreter' => 'markup/engine/remarkup/blockrule/PhutilRemarkupBlockInterpreter.php', 'PhutilRemarkupBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupBlockRule.php', 'PhutilRemarkupBlockStorage' => 'markup/engine/remarkup/PhutilRemarkupBlockStorage.php', 'PhutilRemarkupBoldRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupBoldRule.php', 'PhutilRemarkupCodeBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupCodeBlockRule.php', 'PhutilRemarkupDefaultBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupDefaultBlockRule.php', 'PhutilRemarkupDelRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupDelRule.php', 'PhutilRemarkupDocumentLinkRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupDocumentLinkRule.php', 'PhutilRemarkupEngine' => 'markup/engine/PhutilRemarkupEngine.php', 'PhutilRemarkupEngineTestCase' => 'markup/engine/__tests__/PhutilRemarkupEngineTestCase.php', 'PhutilRemarkupEscapeRemarkupRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupEscapeRemarkupRule.php', 'PhutilRemarkupHeaderBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupHeaderBlockRule.php', 'PhutilRemarkupHighlightRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupHighlightRule.php', 'PhutilRemarkupHorizontalRuleBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupHorizontalRuleBlockRule.php', 'PhutilRemarkupHyperlinkRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupHyperlinkRule.php', 'PhutilRemarkupInlineBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupInlineBlockRule.php', 'PhutilRemarkupInterpreterBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupInterpreterBlockRule.php', 'PhutilRemarkupItalicRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupItalicRule.php', 'PhutilRemarkupLinebreaksRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupLinebreaksRule.php', 'PhutilRemarkupListBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupListBlockRule.php', 'PhutilRemarkupLiteralBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupLiteralBlockRule.php', 'PhutilRemarkupMonospaceRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupMonospaceRule.php', 'PhutilRemarkupNoteBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupNoteBlockRule.php', 'PhutilRemarkupQuotesBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupQuotesBlockRule.php', 'PhutilRemarkupReplyBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupReplyBlockRule.php', 'PhutilRemarkupRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupRule.php', 'PhutilRemarkupSimpleTableBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupSimpleTableBlockRule.php', 'PhutilRemarkupTableBlockRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupTableBlockRule.php', 'PhutilRemarkupTestInterpreterRule' => 'markup/engine/remarkup/blockrule/PhutilRemarkupTestInterpreterRule.php', 'PhutilRemarkupUnderlineRule' => 'markup/engine/remarkup/markuprule/PhutilRemarkupUnderlineRule.php', 'PhutilRope' => 'utils/PhutilRope.php', 'PhutilRopeTestCase' => 'utils/__tests__/PhutilRopeTestCase.php', 'PhutilSafeHTML' => 'markup/PhutilSafeHTML.php', 'PhutilSafeHTMLProducerInterface' => 'markup/PhutilSafeHTMLProducerInterface.php', 'PhutilSafeHTMLTestCase' => 'markup/__tests__/PhutilSafeHTMLTestCase.php', 'PhutilSaturateStdoutDaemon' => 'daemon/torture/PhutilSaturateStdoutDaemon.php', 'PhutilServiceProfiler' => 'serviceprofiler/PhutilServiceProfiler.php', 'PhutilShellLexer' => 'lexer/PhutilShellLexer.php', 'PhutilShellLexerTestCase' => 'lexer/__tests__/PhutilShellLexerTestCase.php', 'PhutilSimpleOptions' => 'parser/PhutilSimpleOptions.php', 'PhutilSimpleOptionsLexer' => 'lexer/PhutilSimpleOptionsLexer.php', 'PhutilSimpleOptionsLexerTestCase' => 'lexer/__tests__/PhutilSimpleOptionsLexerTestCase.php', 'PhutilSimpleOptionsTestCase' => 'parser/__tests__/PhutilSimpleOptionsTestCase.php', 'PhutilSimplifiedChineseChinaLocale' => 'internationalization/locales/PhutilSimplifiedChineseChinaLocale.php', 'PhutilSocketChannel' => 'channel/PhutilSocketChannel.php', 'PhutilSortVector' => 'utils/PhutilSortVector.php', 'PhutilSpanishSpainLocale' => 'internationalization/locales/PhutilSpanishSpainLocale.php', 'PhutilSprite' => 'sprites/PhutilSprite.php', 'PhutilSpriteSheet' => 'sprites/PhutilSpriteSheet.php', 'PhutilStreamIterator' => 'utils/PhutilStreamIterator.php', 'PhutilSymbolLoader' => 'symbols/PhutilSymbolLoader.php', 'PhutilSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilSyntaxHighlighter.php', 'PhutilSyntaxHighlighterEngine' => 'markup/syntax/engine/PhutilSyntaxHighlighterEngine.php', 'PhutilSyntaxHighlighterException' => 'markup/syntax/highlighter/PhutilSyntaxHighlighterException.php', 'PhutilSystem' => 'utils/PhutilSystem.php', 'PhutilSystemTestCase' => 'utils/__tests__/PhutilSystemTestCase.php', 'PhutilTerminalString' => 'xsprintf/PhutilTerminalString.php', 'PhutilTestPhobject' => 'object/__tests__/PhutilTestPhobject.php', 'PhutilTortureTestDaemon' => 'daemon/torture/PhutilTortureTestDaemon.php', 'PhutilTranslation' => 'internationalization/PhutilTranslation.php', 'PhutilTranslationTestCase' => 'internationalization/__tests__/PhutilTranslationTestCase.php', 'PhutilTranslator' => 'internationalization/PhutilTranslator.php', 'PhutilTranslatorTestCase' => 'internationalization/__tests__/PhutilTranslatorTestCase.php', 'PhutilTsprintfTestCase' => 'xsprintf/__tests__/PhutilTsprintfTestCase.php', 'PhutilTwitchAuthAdapter' => 'auth/PhutilTwitchAuthAdapter.php', 'PhutilTwitchFuture' => 'future/twitch/PhutilTwitchFuture.php', 'PhutilTwitterAuthAdapter' => 'auth/PhutilTwitterAuthAdapter.php', 'PhutilTypeCheckException' => 'parser/exception/PhutilTypeCheckException.php', 'PhutilTypeExtraParametersException' => 'parser/exception/PhutilTypeExtraParametersException.php', 'PhutilTypeLexer' => 'lexer/PhutilTypeLexer.php', 'PhutilTypeMissingParametersException' => 'parser/exception/PhutilTypeMissingParametersException.php', 'PhutilTypeSpec' => 'parser/PhutilTypeSpec.php', 'PhutilTypeSpecTestCase' => 'parser/__tests__/PhutilTypeSpecTestCase.php', 'PhutilURI' => 'parser/PhutilURI.php', 'PhutilURITestCase' => 'parser/__tests__/PhutilURITestCase.php', 'PhutilUSEnglishLocale' => 'internationalization/locales/PhutilUSEnglishLocale.php', 'PhutilUTF8StringTruncator' => 'utils/PhutilUTF8StringTruncator.php', 'PhutilUTF8TestCase' => 'utils/__tests__/PhutilUTF8TestCase.php', 'PhutilUnknownSymbolParserGeneratorException' => 'parser/generator/exception/PhutilUnknownSymbolParserGeneratorException.php', 'PhutilUnreachableRuleParserGeneratorException' => 'parser/generator/exception/PhutilUnreachableRuleParserGeneratorException.php', 'PhutilUnreachableTerminalParserGeneratorException' => 'parser/generator/exception/PhutilUnreachableTerminalParserGeneratorException.php', 'PhutilUrisprintfTestCase' => 'xsprintf/__tests__/PhutilUrisprintfTestCase.php', 'PhutilUtilsTestCase' => 'utils/__tests__/PhutilUtilsTestCase.php', 'PhutilVeryWowEnglishLocale' => 'internationalization/locales/PhutilVeryWowEnglishLocale.php', 'PhutilWordPressAuthAdapter' => 'auth/PhutilWordPressAuthAdapter.php', 'PhutilWordPressFuture' => 'future/wordpress/PhutilWordPressFuture.php', 'PhutilXHPASTBinary' => 'parser/xhpast/bin/PhutilXHPASTBinary.php', 'PhutilXHPASTSyntaxHighlighter' => 'markup/syntax/highlighter/PhutilXHPASTSyntaxHighlighter.php', 'PhutilXHPASTSyntaxHighlighterFuture' => 'markup/syntax/highlighter/xhpast/PhutilXHPASTSyntaxHighlighterFuture.php', 'PhutilXHPASTSyntaxHighlighterTestCase' => 'markup/syntax/highlighter/__tests__/PhutilXHPASTSyntaxHighlighterTestCase.php', 'QueryFuture' => 'future/query/QueryFuture.php', 'TempFile' => 'filesystem/TempFile.php', 'TestAbstractDirectedGraph' => 'utils/__tests__/TestAbstractDirectedGraph.php', 'XHPASTNode' => 'parser/xhpast/api/XHPASTNode.php', 'XHPASTNodeTestCase' => 'parser/xhpast/api/__tests__/XHPASTNodeTestCase.php', 'XHPASTSyntaxErrorException' => 'parser/xhpast/api/XHPASTSyntaxErrorException.php', 'XHPASTToken' => 'parser/xhpast/api/XHPASTToken.php', 'XHPASTTree' => 'parser/xhpast/api/XHPASTTree.php', 'XHPASTTreeTestCase' => 'parser/xhpast/api/__tests__/XHPASTTreeTestCase.php', 'XsprintfUnknownConversionException' => 'xsprintf/exception/XsprintfUnknownConversionException.php', ), 'function' => array( 'array_fuse' => 'utils/utils.php', 'array_interleave' => 'utils/utils.php', 'array_mergev' => 'utils/utils.php', 'array_select_keys' => 'utils/utils.php', 'assert_instances_of' => 'utils/utils.php', 'assert_stringlike' => 'utils/utils.php', 'coalesce' => 'utils/utils.php', 'csprintf' => 'xsprintf/csprintf.php', 'exec_manual' => 'future/exec/execx.php', 'execx' => 'future/exec/execx.php', 'head' => 'utils/utils.php', 'head_key' => 'utils/utils.php', 'hgsprintf' => 'xsprintf/hgsprintf.php', 'hsprintf' => 'markup/render.php', 'id' => 'utils/utils.php', 'idx' => 'utils/utils.php', 'idxv' => 'utils/utils.php', 'ifilter' => 'utils/utils.php', 'igroup' => 'utils/utils.php', 'ipull' => 'utils/utils.php', 'isort' => 'utils/utils.php', 'jsprintf' => 'xsprintf/jsprintf.php', 'last' => 'utils/utils.php', 'last_key' => 'utils/utils.php', 'ldap_sprintf' => 'xsprintf/ldapsprintf.php', 'mfilter' => 'utils/utils.php', 'mgroup' => 'utils/utils.php', 'mpull' => 'utils/utils.php', 'msort' => 'utils/utils.php', 'msortv' => 'utils/utils.php', 'newv' => 'utils/utils.php', 'nonempty' => 'utils/utils.php', 'phlog' => 'error/phlog.php', 'pht' => 'internationalization/pht.php', 'phutil_censor_credentials' => 'utils/utils.php', 'phutil_console_confirm' => 'console/format.php', 'phutil_console_format' => 'console/format.php', 'phutil_console_get_terminal_width' => 'console/format.php', 'phutil_console_prompt' => 'console/format.php', 'phutil_console_require_tty' => 'console/format.php', 'phutil_console_wrap' => 'console/format.php', 'phutil_count' => 'internationalization/pht.php', 'phutil_date_format' => 'utils/viewutils.php', 'phutil_deprecated' => 'moduleutils/moduleutils.php', 'phutil_error_listener_example' => 'error/phlog.php', 'phutil_escape_html' => 'markup/render.php', 'phutil_escape_html_newlines' => 'markup/render.php', 'phutil_escape_uri' => 'markup/render.php', 'phutil_escape_uri_path_component' => 'markup/render.php', 'phutil_fnmatch' => 'utils/utils.php', 'phutil_format_bytes' => 'utils/viewutils.php', 'phutil_format_relative_time' => 'utils/viewutils.php', 'phutil_format_relative_time_detailed' => 'utils/viewutils.php', 'phutil_format_units_generic' => 'utils/viewutils.php', 'phutil_fwrite_nonblocking_stream' => 'utils/utils.php', 'phutil_get_current_library_name' => 'moduleutils/moduleutils.php', 'phutil_get_library_name_for_root' => 'moduleutils/moduleutils.php', 'phutil_get_library_root' => 'moduleutils/moduleutils.php', 'phutil_get_library_root_for_path' => 'moduleutils/moduleutils.php', 'phutil_get_signal_name' => 'future/exec/execx.php', 'phutil_hashes_are_identical' => 'utils/utils.php', 'phutil_implode_html' => 'markup/render.php', 'phutil_ini_decode' => 'utils/utils.php', 'phutil_is_hiphop_runtime' => 'utils/utils.php', 'phutil_is_utf8' => 'utils/utf8.php', 'phutil_is_utf8_slowly' => 'utils/utf8.php', 'phutil_is_utf8_with_only_bmp_characters' => 'utils/utf8.php', 'phutil_is_windows' => 'utils/utils.php', 'phutil_json_decode' => 'utils/utils.php', 'phutil_json_encode' => 'utils/utils.php', 'phutil_load_library' => 'moduleutils/core.php', 'phutil_loggable_string' => 'utils/utils.php', 'phutil_parse_bytes' => 'utils/viewutils.php', 'phutil_passthru' => 'future/exec/execx.php', 'phutil_register_library' => 'moduleutils/core.php', 'phutil_register_library_map' => 'moduleutils/core.php', 'phutil_safe_html' => 'markup/render.php', 'phutil_split_lines' => 'utils/utils.php', 'phutil_tag' => 'markup/render.php', 'phutil_tag_div' => 'markup/render.php', 'phutil_unescape_uri_path_component' => 'markup/render.php', 'phutil_units' => 'utils/utils.php', 'phutil_utf8_console_strlen' => 'utils/utf8.php', 'phutil_utf8_convert' => 'utils/utf8.php', 'phutil_utf8_hard_wrap' => 'utils/utf8.php', 'phutil_utf8_hard_wrap_html' => 'utils/utf8.php', 'phutil_utf8_is_combining_character' => 'utils/utf8.php', 'phutil_utf8_strlen' => 'utils/utf8.php', 'phutil_utf8_strtolower' => 'utils/utf8.php', 'phutil_utf8_strtoupper' => 'utils/utf8.php', 'phutil_utf8_strtr' => 'utils/utf8.php', 'phutil_utf8_ucwords' => 'utils/utf8.php', 'phutil_utf8ize' => 'utils/utf8.php', 'phutil_utf8v' => 'utils/utf8.php', 'phutil_utf8v_codepoints' => 'utils/utf8.php', 'phutil_utf8v_combine_characters' => 'utils/utf8.php', 'phutil_utf8v_combined' => 'utils/utf8.php', 'phutil_validate_json' => 'utils/utils.php', 'phutil_var_export' => 'utils/utils.php', 'ppull' => 'utils/utils.php', 'pregsprintf' => 'xsprintf/pregsprintf.php', 'qsprintf' => 'xsprintf/qsprintf.php', 'qsprintf_check_scalar_type' => 'xsprintf/qsprintf.php', 'qsprintf_check_type' => 'xsprintf/qsprintf.php', 'queryfx' => 'xsprintf/queryfx.php', 'queryfx_all' => 'xsprintf/queryfx.php', 'queryfx_one' => 'xsprintf/queryfx.php', 'tsprintf' => 'xsprintf/tsprintf.php', 'urisprintf' => 'xsprintf/urisprintf.php', 'vcsprintf' => 'xsprintf/csprintf.php', 'vjsprintf' => 'xsprintf/jsprintf.php', 'vqsprintf' => 'xsprintf/qsprintf.php', 'vurisprintf' => 'xsprintf/urisprintf.php', 'xhp_parser_node_constants' => 'parser/xhpast/parser_nodes.php', 'xhpast_parser_token_constants' => 'parser/xhpast/parser_tokens.php', 'xsprintf' => 'xsprintf/xsprintf.php', 'xsprintf_callback_example' => 'xsprintf/xsprintf.php', 'xsprintf_command' => 'xsprintf/csprintf.php', 'xsprintf_javascript' => 'xsprintf/jsprintf.php', 'xsprintf_ldap' => 'xsprintf/ldapsprintf.php', 'xsprintf_mercurial' => 'xsprintf/hgsprintf.php', 'xsprintf_query' => 'xsprintf/qsprintf.php', 'xsprintf_regex' => 'xsprintf/pregsprintf.php', 'xsprintf_terminal' => 'xsprintf/tsprintf.php', 'xsprintf_uri' => 'xsprintf/urisprintf.php', ), 'xmap' => array( 'AASTNode' => 'Phobject', 'AASTNodeList' => array( 'Phobject', 'Countable', 'Iterator', ), 'AASTToken' => 'Phobject', 'AASTTree' => 'Phobject', 'AbstractDirectedGraph' => 'Phobject', 'AbstractDirectedGraphTestCase' => 'PhutilTestCase', 'AphrontAccessDeniedQueryException' => 'AphrontQueryException', 'AphrontBaseMySQLDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontCharacterSetQueryException' => 'AphrontQueryException', 'AphrontConnectionLostQueryException' => 'AphrontRecoverableQueryException', 'AphrontConnectionQueryException' => 'AphrontQueryException', 'AphrontCountQueryException' => 'AphrontQueryException', 'AphrontDatabaseConnection' => array( 'Phobject', 'PhutilQsprintfInterface', ), 'AphrontDatabaseTransactionState' => 'Phobject', 'AphrontDeadlockQueryException' => 'AphrontRecoverableQueryException', 'AphrontDuplicateKeyQueryException' => 'AphrontQueryException', 'AphrontInvalidCredentialsQueryException' => 'AphrontQueryException', 'AphrontIsolatedDatabaseConnection' => 'AphrontDatabaseConnection', 'AphrontLockTimeoutQueryException' => 'AphrontRecoverableQueryException', 'AphrontMySQLDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', 'AphrontMySQLiDatabaseConnection' => 'AphrontBaseMySQLDatabaseConnection', 'AphrontNotSupportedQueryException' => 'AphrontQueryException', 'AphrontObjectMissingQueryException' => 'AphrontQueryException', 'AphrontParameterQueryException' => 'AphrontQueryException', 'AphrontQueryException' => 'Exception', 'AphrontQueryTimeoutQueryException' => 'AphrontRecoverableQueryException', 'AphrontRecoverableQueryException' => 'AphrontQueryException', 'AphrontRequestStream' => 'Phobject', 'AphrontSchemaQueryException' => 'AphrontQueryException', 'AphrontScopedUnguardedWriteCapability' => 'Phobject', 'AphrontWriteGuard' => 'Phobject', 'BaseHTTPFuture' => 'Future', 'CaseInsensitiveArray' => 'PhutilArray', 'CaseInsensitiveArrayTestCase' => 'PhutilTestCase', 'CommandException' => 'Exception', 'ConduitClient' => 'Phobject', 'ConduitClientException' => 'Exception', 'ConduitClientTestCase' => 'PhutilTestCase', 'ConduitFuture' => 'FutureProxy', 'ExecFuture' => 'PhutilExecutableFuture', 'ExecFutureTestCase' => 'PhutilTestCase', 'ExecPassthruTestCase' => 'PhutilTestCase', 'FileFinder' => 'Phobject', 'FileFinderTestCase' => 'PhutilTestCase', 'FileList' => 'Phobject', 'Filesystem' => 'Phobject', 'FilesystemException' => 'Exception', 'FilesystemTestCase' => 'PhutilTestCase', 'Future' => 'Phobject', 'FutureIterator' => array( 'Phobject', 'Iterator', ), 'FutureIteratorTestCase' => 'PhutilTestCase', 'FutureProxy' => 'Future', 'HTTPFuture' => 'BaseHTTPFuture', 'HTTPFutureCURLResponseStatus' => 'HTTPFutureResponseStatus', 'HTTPFutureCertificateResponseStatus' => 'HTTPFutureResponseStatus', 'HTTPFutureHTTPResponseStatus' => 'HTTPFutureResponseStatus', 'HTTPFutureParseResponseStatus' => 'HTTPFutureResponseStatus', 'HTTPFutureResponseStatus' => 'Exception', 'HTTPFutureTransportResponseStatus' => 'HTTPFutureResponseStatus', 'HTTPSFuture' => 'BaseHTTPFuture', 'ImmediateFuture' => 'Future', 'LibphutilUSEnglishTranslation' => 'PhutilTranslation', 'LinesOfALarge' => array( 'Phobject', 'Iterator', ), 'LinesOfALargeExecFuture' => 'LinesOfALarge', 'LinesOfALargeExecFutureTestCase' => 'PhutilTestCase', 'LinesOfALargeFile' => 'LinesOfALarge', 'LinesOfALargeFileTestCase' => 'PhutilTestCase', 'MFilterTestHelper' => 'Phobject', 'PHPASTParserTestCase' => 'PhutilTestCase', 'PhageAgentBootloader' => 'Phobject', 'PhageAgentTestCase' => 'PhutilTestCase', 'PhagePHPAgent' => 'Phobject', 'PhagePHPAgentBootloader' => 'PhageAgentBootloader', 'Phobject' => 'Iterator', 'PhobjectTestCase' => 'PhutilTestCase', 'PhutilAPCKeyValueCache' => 'PhutilKeyValueCache', 'PhutilAWSEC2Future' => 'PhutilAWSFuture', 'PhutilAWSException' => 'Exception', 'PhutilAWSFuture' => 'FutureProxy', 'PhutilAWSManagementWorkflow' => 'PhutilArgumentWorkflow', 'PhutilAWSS3DeleteManagementWorkflow' => 'PhutilAWSS3ManagementWorkflow', 'PhutilAWSS3Future' => 'PhutilAWSFuture', 'PhutilAWSS3GetManagementWorkflow' => 'PhutilAWSS3ManagementWorkflow', 'PhutilAWSS3ManagementWorkflow' => 'PhutilAWSManagementWorkflow', 'PhutilAWSS3PutManagementWorkflow' => 'PhutilAWSS3ManagementWorkflow', 'PhutilAWSv4Signature' => 'Phobject', 'PhutilAWSv4SignatureTestCase' => 'PhutilTestCase', 'PhutilAggregateException' => 'Exception', 'PhutilAllCapsEnglishLocale' => 'PhutilLocale', 'PhutilAmazonAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilArgumentParser' => 'Phobject', 'PhutilArgumentParserException' => 'Exception', 'PhutilArgumentParserTestCase' => 'PhutilTestCase', 'PhutilArgumentSpecification' => 'Phobject', 'PhutilArgumentSpecificationException' => 'PhutilArgumentParserException', 'PhutilArgumentSpecificationTestCase' => 'PhutilTestCase', 'PhutilArgumentUsageException' => 'PhutilArgumentParserException', 'PhutilArgumentWorkflow' => 'Phobject', 'PhutilArray' => array( 'Phobject', 'Countable', 'ArrayAccess', 'Iterator', ), 'PhutilArrayTestCase' => 'PhutilTestCase', 'PhutilArrayWithDefaultValue' => 'PhutilArray', 'PhutilAsanaAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilAsanaFuture' => 'FutureProxy', 'PhutilAuthAdapter' => 'Phobject', 'PhutilAuthConfigurationException' => 'PhutilAuthException', 'PhutilAuthCredentialException' => 'PhutilAuthException', 'PhutilAuthException' => 'Exception', 'PhutilAuthUserAbortedException' => 'PhutilAuthException', 'PhutilBallOfPHP' => 'Phobject', 'PhutilBitbucketAuthAdapter' => 'PhutilOAuth1AuthAdapter', 'PhutilBootloaderException' => 'Exception', 'PhutilBritishEnglishLocale' => 'PhutilLocale', 'PhutilBufferedIterator' => array( 'Phobject', 'Iterator', ), 'PhutilBufferedIteratorTestCase' => 'PhutilTestCase', 'PhutilBugtraqParser' => 'Phobject', 'PhutilBugtraqParserTestCase' => 'PhutilTestCase', 'PhutilCIDRBlock' => 'Phobject', 'PhutilCIDRList' => 'Phobject', 'PhutilCLikeCodeSnippetContextFreeGrammar' => 'PhutilCodeSnippetContextFreeGrammar', 'PhutilCallbackFilterIterator' => 'FilterIterator', 'PhutilChannel' => 'Phobject', 'PhutilChannelChannel' => 'PhutilChannel', 'PhutilChannelTestCase' => 'PhutilTestCase', 'PhutilChunkedIterator' => array( 'Phobject', 'Iterator', ), 'PhutilChunkedIteratorTestCase' => 'PhutilTestCase', 'PhutilClassMapQuery' => 'Phobject', 'PhutilCodeSnippetContextFreeGrammar' => 'PhutilContextFreeGrammar', 'PhutilCommandString' => 'Phobject', 'PhutilConsole' => 'Phobject', 'PhutilConsoleBlock' => 'PhutilConsoleView', 'PhutilConsoleConcatenatedView' => 'PhutilConsoleView', 'PhutilConsoleFormatter' => 'Phobject', 'PhutilConsoleList' => 'PhutilConsoleView', 'PhutilConsoleMessage' => 'Phobject', 'PhutilConsoleProgressBar' => 'Phobject', 'PhutilConsoleServer' => 'Phobject', 'PhutilConsoleServerChannel' => 'PhutilChannelChannel', 'PhutilConsoleStdinNotInteractiveException' => 'Exception', 'PhutilConsoleSyntaxHighlighter' => 'Phobject', 'PhutilConsoleTable' => 'PhutilConsoleView', 'PhutilConsoleView' => 'Phobject', 'PhutilConsoleWrapTestCase' => 'PhutilTestCase', 'PhutilContextFreeGrammar' => 'Phobject', 'PhutilCowsay' => 'Phobject', 'PhutilCowsayTestCase' => 'PhutilTestCase', 'PhutilCsprintfTestCase' => 'PhutilTestCase', 'PhutilCzechLocale' => 'PhutilLocale', 'PhutilDaemon' => 'Phobject', 'PhutilDaemonHandle' => 'Phobject', 'PhutilDaemonOverseer' => 'Phobject', 'PhutilDaemonOverseerModule' => 'Phobject', 'PhutilDefaultSyntaxHighlighter' => 'Phobject', 'PhutilDefaultSyntaxHighlighterEngine' => 'PhutilSyntaxHighlighterEngine', 'PhutilDefaultSyntaxHighlighterEnginePygmentsFuture' => 'FutureProxy', 'PhutilDefaultSyntaxHighlighterEngineTestCase' => 'PhutilTestCase', 'PhutilDeferredLog' => 'Phobject', 'PhutilDeferredLogTestCase' => 'PhutilTestCase', 'PhutilDirectedScalarGraph' => 'AbstractDirectedGraph', 'PhutilDirectoryFixture' => 'Phobject', 'PhutilDirectoryKeyValueCache' => 'PhutilKeyValueCache', 'PhutilDisqusAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilDivinerSyntaxHighlighter' => 'Phobject', 'PhutilDocblockParser' => 'Phobject', 'PhutilDocblockParserTestCase' => 'PhutilTestCase', 'PhutilEditDistanceMatrix' => 'Phobject', 'PhutilEditDistanceMatrixTestCase' => 'PhutilTestCase', 'PhutilEditorConfig' => 'Phobject', 'PhutilEditorConfigTestCase' => 'PhutilTestCase', 'PhutilEmailAddress' => 'Phobject', 'PhutilEmailAddressTestCase' => 'PhutilTestCase', 'PhutilEmojiLocale' => 'PhutilLocale', 'PhutilEmptyAuthAdapter' => 'PhutilAuthAdapter', 'PhutilEnglishCanadaLocale' => 'PhutilLocale', 'PhutilErrorHandler' => 'Phobject', 'PhutilErrorHandlerTestCase' => 'PhutilTestCase', 'PhutilErrorTrap' => 'Phobject', 'PhutilEvent' => 'Phobject', 'PhutilEventConstants' => 'Phobject', 'PhutilEventEngine' => 'Phobject', 'PhutilEventListener' => 'Phobject', 'PhutilEventType' => 'PhutilEventConstants', 'PhutilExampleBufferedIterator' => 'PhutilBufferedIterator', 'PhutilExcessiveServiceCallsDaemon' => 'PhutilTortureTestDaemon', 'PhutilExecChannel' => 'PhutilChannel', 'PhutilExecPassthru' => 'PhutilExecutableFuture', 'PhutilExecutableFuture' => 'Future', 'PhutilExecutionEnvironment' => 'Phobject', 'PhutilExtensionsTestCase' => 'PhutilTestCase', 'PhutilFacebookAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilFatalDaemon' => 'PhutilTortureTestDaemon', 'PhutilFileLock' => 'PhutilLock', 'PhutilFileLockTestCase' => 'PhutilTestCase', 'PhutilFileTree' => 'Phobject', 'PhutilFrenchLocale' => 'PhutilLocale', 'PhutilGermanLocale' => 'PhutilLocale', 'PhutilGitHubAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilGitHubFuture' => 'FutureProxy', 'PhutilGitHubResponse' => 'Phobject', 'PhutilGitURI' => 'Phobject', 'PhutilGitURITestCase' => 'PhutilTestCase', 'PhutilGoogleAuthAdapter' => 'PhutilOAuthAuthAdapter', + 'PhutilHTTPEngineExtension' => 'Phobject', 'PhutilHangForeverDaemon' => 'PhutilTortureTestDaemon', 'PhutilHashingIterator' => array( 'PhutilProxyIterator', 'Iterator', ), 'PhutilHashingIteratorTestCase' => 'PhutilTestCase', 'PhutilHelpArgumentWorkflow' => 'PhutilArgumentWorkflow', 'PhutilHgsprintfTestCase' => 'PhutilTestCase', 'PhutilHighIntensityIntervalDaemon' => 'PhutilTortureTestDaemon', 'PhutilINIParserException' => 'Exception', 'PhutilIPAddress' => 'Phobject', 'PhutilIPAddressTestCase' => 'PhutilTestCase', 'PhutilInRequestKeyValueCache' => 'PhutilKeyValueCache', 'PhutilInteractiveEditor' => 'Phobject', 'PhutilInvalidRuleParserGeneratorException' => 'PhutilParserGeneratorException', 'PhutilInvalidStateException' => 'Exception', 'PhutilInvalidStateExceptionTestCase' => 'PhutilTestCase', 'PhutilInvisibleSyntaxHighlighter' => 'Phobject', 'PhutilIrreducibleRuleParserGeneratorException' => 'PhutilParserGeneratorException', 'PhutilJIRAAuthAdapter' => 'PhutilOAuth1AuthAdapter', 'PhutilJSON' => 'Phobject', 'PhutilJSONFragmentLexer' => 'PhutilLexer', 'PhutilJSONFragmentLexerHighlighterTestCase' => 'PhutilTestCase', 'PhutilJSONParser' => 'Phobject', 'PhutilJSONParserException' => 'Exception', 'PhutilJSONParserTestCase' => 'PhutilTestCase', 'PhutilJSONProtocolChannel' => 'PhutilProtocolChannel', 'PhutilJSONProtocolChannelTestCase' => 'PhutilTestCase', 'PhutilJSONTestCase' => 'PhutilTestCase', 'PhutilJavaCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', 'PhutilKeyValueCache' => 'Phobject', 'PhutilKeyValueCacheNamespace' => 'PhutilKeyValueCacheProxy', 'PhutilKeyValueCacheProfiler' => 'PhutilKeyValueCacheProxy', 'PhutilKeyValueCacheProxy' => 'PhutilKeyValueCache', 'PhutilKeyValueCacheStack' => 'PhutilKeyValueCache', 'PhutilKeyValueCacheTestCase' => 'PhutilTestCase', 'PhutilKoreanLocale' => 'PhutilLocale', 'PhutilLDAPAuthAdapter' => 'PhutilAuthAdapter', 'PhutilLanguageGuesser' => 'Phobject', 'PhutilLanguageGuesserTestCase' => 'PhutilTestCase', 'PhutilLexer' => 'Phobject', 'PhutilLexerSyntaxHighlighter' => 'PhutilSyntaxHighlighter', 'PhutilLibraryConflictException' => 'Exception', 'PhutilLibraryMapBuilder' => 'Phobject', 'PhutilLibraryTestCase' => 'PhutilTestCase', 'PhutilLipsumContextFreeGrammar' => 'PhutilContextFreeGrammar', 'PhutilLocale' => 'Phobject', 'PhutilLocaleTestCase' => 'PhutilTestCase', 'PhutilLock' => 'Phobject', 'PhutilLockException' => 'Exception', 'PhutilLogFileChannel' => 'PhutilChannelChannel', 'PhutilLunarPhase' => 'Phobject', 'PhutilLunarPhaseTestCase' => 'PhutilTestCase', 'PhutilMarkupEngine' => 'Phobject', 'PhutilMarkupTestCase' => 'PhutilTestCase', 'PhutilMemcacheKeyValueCache' => 'PhutilKeyValueCache', 'PhutilMethodNotImplementedException' => 'Exception', 'PhutilMetricsChannel' => 'PhutilChannelChannel', 'PhutilMissingSymbolException' => 'Exception', 'PhutilModuleUtilsTestCase' => 'PhutilTestCase', 'PhutilNiceDaemon' => 'PhutilTortureTestDaemon', 'PhutilNumber' => 'Phobject', 'PhutilOAuth1AuthAdapter' => 'PhutilAuthAdapter', 'PhutilOAuth1Future' => 'FutureProxy', 'PhutilOAuth1FutureTestCase' => 'PhutilTestCase', 'PhutilOAuthAuthAdapter' => 'PhutilAuthAdapter', 'PhutilOnDiskKeyValueCache' => 'PhutilKeyValueCache', 'PhutilOpaqueEnvelope' => 'Phobject', 'PhutilOpaqueEnvelopeKey' => 'Phobject', 'PhutilOpaqueEnvelopeTestCase' => 'PhutilTestCase', 'PhutilPHPCodeSnippetContextFreeGrammar' => 'PhutilCLikeCodeSnippetContextFreeGrammar', 'PhutilPHPFragmentLexer' => 'PhutilLexer', 'PhutilPHPFragmentLexerHighlighterTestCase' => 'PhutilTestCase', 'PhutilPHPFragmentLexerTestCase' => 'PhutilTestCase', 'PhutilPHPObjectProtocolChannel' => 'PhutilProtocolChannel', 'PhutilPHPObjectProtocolChannelTestCase' => 'PhutilTestCase', 'PhutilParserGenerator' => 'Phobject', 'PhutilParserGeneratorException' => 'Exception', 'PhutilParserGeneratorTestCase' => 'PhutilTestCase', 'PhutilPayPalAPIFuture' => 'FutureProxy', 'PhutilPersonTest' => array( 'Phobject', 'PhutilPerson', ), 'PhutilPersonaAuthAdapter' => 'PhutilAuthAdapter', 'PhutilPhabricatorAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilPhtTestCase' => 'PhutilTestCase', 'PhutilPirateEnglishLocale' => 'PhutilLocale', 'PhutilPortugueseBrazilLocale' => 'PhutilLocale', 'PhutilPortuguesePortugalLocale' => 'PhutilLocale', 'PhutilPregsprintfTestCase' => 'PhutilTestCase', 'PhutilProcessGroupDaemon' => 'PhutilTortureTestDaemon', + 'PhutilProseDiff' => 'Phobject', + 'PhutilProseDiffTestCase' => 'PhutilTestCase', + 'PhutilProseDifferenceEngine' => 'Phobject', 'PhutilProtocolChannel' => 'PhutilChannelChannel', 'PhutilProxyException' => 'Exception', 'PhutilProxyIterator' => array( 'Phobject', 'Iterator', ), 'PhutilPygmentizeParser' => 'Phobject', 'PhutilPygmentizeParserTestCase' => 'PhutilTestCase', 'PhutilPygmentsSyntaxHighlighter' => 'Phobject', 'PhutilPythonFragmentLexer' => 'PhutilLexer', 'PhutilQueryStringParser' => 'Phobject', 'PhutilQueryStringParserTestCase' => 'PhutilTestCase', 'PhutilRainbowSyntaxHighlighter' => 'Phobject', 'PhutilRawEnglishLocale' => 'PhutilLocale', 'PhutilReadableSerializer' => 'Phobject', 'PhutilReadableSerializerTestCase' => 'PhutilTestCase', 'PhutilRealNameContextFreeGrammar' => 'PhutilContextFreeGrammar', 'PhutilRemarkupBlockInterpreter' => 'Phobject', 'PhutilRemarkupBlockRule' => 'Phobject', 'PhutilRemarkupBlockStorage' => 'Phobject', 'PhutilRemarkupBoldRule' => 'PhutilRemarkupRule', 'PhutilRemarkupCodeBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupDefaultBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupDelRule' => 'PhutilRemarkupRule', 'PhutilRemarkupDocumentLinkRule' => 'PhutilRemarkupRule', 'PhutilRemarkupEngine' => 'PhutilMarkupEngine', 'PhutilRemarkupEngineTestCase' => 'PhutilTestCase', 'PhutilRemarkupEscapeRemarkupRule' => 'PhutilRemarkupRule', 'PhutilRemarkupHeaderBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupHighlightRule' => 'PhutilRemarkupRule', 'PhutilRemarkupHorizontalRuleBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupHyperlinkRule' => 'PhutilRemarkupRule', 'PhutilRemarkupInlineBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupInterpreterBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupItalicRule' => 'PhutilRemarkupRule', 'PhutilRemarkupLinebreaksRule' => 'PhutilRemarkupRule', 'PhutilRemarkupListBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupLiteralBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupMonospaceRule' => 'PhutilRemarkupRule', 'PhutilRemarkupNoteBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupQuotesBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupReplyBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupRule' => 'Phobject', 'PhutilRemarkupSimpleTableBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupTableBlockRule' => 'PhutilRemarkupBlockRule', 'PhutilRemarkupTestInterpreterRule' => 'PhutilRemarkupBlockInterpreter', 'PhutilRemarkupUnderlineRule' => 'PhutilRemarkupRule', 'PhutilRope' => 'Phobject', 'PhutilRopeTestCase' => 'PhutilTestCase', 'PhutilSafeHTML' => 'Phobject', 'PhutilSafeHTMLTestCase' => 'PhutilTestCase', 'PhutilSaturateStdoutDaemon' => 'PhutilTortureTestDaemon', 'PhutilServiceProfiler' => 'Phobject', 'PhutilShellLexer' => 'PhutilLexer', 'PhutilShellLexerTestCase' => 'PhutilTestCase', 'PhutilSimpleOptions' => 'Phobject', 'PhutilSimpleOptionsLexer' => 'PhutilLexer', 'PhutilSimpleOptionsLexerTestCase' => 'PhutilTestCase', 'PhutilSimpleOptionsTestCase' => 'PhutilTestCase', 'PhutilSimplifiedChineseChinaLocale' => 'PhutilLocale', 'PhutilSocketChannel' => 'PhutilChannel', 'PhutilSortVector' => 'Phobject', 'PhutilSpanishSpainLocale' => 'PhutilLocale', 'PhutilSprite' => 'Phobject', 'PhutilSpriteSheet' => 'Phobject', 'PhutilStreamIterator' => array( 'Phobject', 'Iterator', ), 'PhutilSyntaxHighlighter' => 'Phobject', 'PhutilSyntaxHighlighterEngine' => 'Phobject', 'PhutilSyntaxHighlighterException' => 'Exception', 'PhutilSystem' => 'Phobject', 'PhutilSystemTestCase' => 'PhutilTestCase', 'PhutilTerminalString' => 'Phobject', 'PhutilTestPhobject' => 'Phobject', 'PhutilTortureTestDaemon' => 'PhutilDaemon', 'PhutilTranslation' => 'Phobject', 'PhutilTranslationTestCase' => 'PhutilTestCase', 'PhutilTranslator' => 'Phobject', 'PhutilTranslatorTestCase' => 'PhutilTestCase', 'PhutilTsprintfTestCase' => 'PhutilTestCase', 'PhutilTwitchAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilTwitchFuture' => 'FutureProxy', 'PhutilTwitterAuthAdapter' => 'PhutilOAuth1AuthAdapter', 'PhutilTypeCheckException' => 'Exception', 'PhutilTypeExtraParametersException' => 'Exception', 'PhutilTypeLexer' => 'PhutilLexer', 'PhutilTypeMissingParametersException' => 'Exception', 'PhutilTypeSpec' => 'Phobject', 'PhutilTypeSpecTestCase' => 'PhutilTestCase', 'PhutilURI' => 'Phobject', 'PhutilURITestCase' => 'PhutilTestCase', 'PhutilUSEnglishLocale' => 'PhutilLocale', 'PhutilUTF8StringTruncator' => 'Phobject', 'PhutilUTF8TestCase' => 'PhutilTestCase', 'PhutilUnknownSymbolParserGeneratorException' => 'PhutilParserGeneratorException', 'PhutilUnreachableRuleParserGeneratorException' => 'PhutilParserGeneratorException', 'PhutilUnreachableTerminalParserGeneratorException' => 'PhutilParserGeneratorException', 'PhutilUrisprintfTestCase' => 'PhutilTestCase', 'PhutilUtilsTestCase' => 'PhutilTestCase', 'PhutilVeryWowEnglishLocale' => 'PhutilLocale', 'PhutilWordPressAuthAdapter' => 'PhutilOAuthAuthAdapter', 'PhutilWordPressFuture' => 'FutureProxy', 'PhutilXHPASTBinary' => 'Phobject', 'PhutilXHPASTSyntaxHighlighter' => 'Phobject', 'PhutilXHPASTSyntaxHighlighterFuture' => 'FutureProxy', 'PhutilXHPASTSyntaxHighlighterTestCase' => 'PhutilTestCase', 'QueryFuture' => 'Future', 'TempFile' => 'Phobject', 'TestAbstractDirectedGraph' => 'AbstractDirectedGraph', 'XHPASTNode' => 'AASTNode', 'XHPASTNodeTestCase' => 'PhutilTestCase', 'XHPASTSyntaxErrorException' => 'Exception', 'XHPASTToken' => 'AASTToken', 'XHPASTTree' => 'AASTTree', 'XHPASTTreeTestCase' => 'PhutilTestCase', 'XsprintfUnknownConversionException' => 'InvalidArgumentException', ), )); diff --git a/src/daemon/PhutilDaemonHandle.php b/src/daemon/PhutilDaemonHandle.php index 3de873c..e6ff223 100644 --- a/src/daemon/PhutilDaemonHandle.php +++ b/src/daemon/PhutilDaemonHandle.php @@ -1,422 +1,409 @@ overseer = $overseer; $this->daemonClass = $daemon_class; $this->argv = $argv; $this->config = $config; $this->restartAt = time(); $this->daemonID = $this->generateDaemonID(); $this->dispatchEvent( self::EVENT_DID_LAUNCH, array( 'argv' => $this->argv, 'explicitArgv' => idx($this->config, 'argv'), )); } public function isRunning() { return (bool)$this->future; } public function isDone() { return (!$this->shouldRestart && !$this->isRunning()); } public function getFuture() { return $this->future; } - public function setSilent($silent) { - $this->silent = $silent; - return $this; - } - - public function getSilent() { - return $this->silent; - } - public function setTraceMemory($trace_memory) { $this->traceMemory = $trace_memory; return $this; } public function getTraceMemory() { return $this->traceMemory; } public function update() { $this->updateMemory(); if (!$this->isRunning()) { if (!$this->shouldRestart) { return; } if (!$this->restartAt || (time() < $this->restartAt)) { return; } if ($this->shouldShutdown) { return; } $this->startDaemonProcess(); } $future = $this->future; $result = null; if ($future->isReady()) { $result = $future->resolve(); } list($stdout, $stderr) = $future->read(); $future->discardBuffers(); if (strlen($stdout)) { $this->didReadStdout($stdout); } $stderr = trim($stderr); if (strlen($stderr)) { foreach (phutil_split_lines($stderr, false) as $line) { $this->logMessage('STDE', $line); } } if ($result !== null) { list($err) = $result; if ($err) { $this->logMessage('FAIL', pht('Process exited with error %s.', $err)); } else { $this->logMessage('DONE', pht('Process exited normally.')); } $this->future = null; if ($this->shouldShutdown) { $this->restartAt = null; } else { $this->scheduleRestart(); } } $this->updateHeartbeatEvent(); $this->updateHangDetection(); } private function updateHeartbeatEvent() { if ($this->heartbeat > time()) { return; } $this->heartbeat = time() + $this->getHeartbeatEventFrequency(); $this->dispatchEvent(self::EVENT_DID_HEARTBEAT); } private function updateHangDetection() { if (!$this->isRunning()) { return; } if (time() > $this->deadline) { $this->logMessage('HANG', pht('Hang detected. Restarting process.')); $this->annihilateProcessGroup(); $this->scheduleRestart(); } } private function scheduleRestart() { $this->logMessage('WAIT', pht('Waiting to restart process.')); $this->restartAt = time() + self::getWaitBeforeRestart(); } /** * Generate a unique ID for this daemon. * * @return string A unique daemon ID. */ private function generateDaemonID() { return substr(getmypid().':'.Filesystem::readRandomCharacters(12), 0, 12); } public function getDaemonID() { return $this->daemonID; } public function getPID() { return $this->pid; } private function getCaptureBufferSize() { return 65535; } private function getRequiredHeartbeatFrequency() { return 86400; } public static function getWaitBeforeRestart() { return 5; } public static function getHeartbeatEventFrequency() { return 120; } private function getKillDelay() { return 3; } private function getDaemonCWD() { $root = dirname(phutil_get_library_root('phutil')); return $root.'/scripts/daemon/exec/'; } private function newExecFuture() { $class = $this->daemonClass; $argv = $this->argv; $buffer_size = $this->getCaptureBufferSize(); // NOTE: PHP implements proc_open() by running 'sh -c'. On most systems this // is bash, but on Ubuntu it's dash. When you proc_open() using bash, you // get one new process (the command you ran). When you proc_open() using // dash, you get two new processes: the command you ran and a parent // "dash -c" (or "sh -c") process. This means that the child process's PID // is actually the 'dash' PID, not the command's PID. To avoid this, use // 'exec' to replace the shell process with the real process; without this, // the child will call posix_getppid(), be given the pid of the 'sh -c' // process, and send it SIGUSR1 to keepalive which will terminate it // immediately. We also won't be able to do process group management because // the shell process won't properly posix_setsid() so the pgid of the child // won't be meaningful. return id(new ExecFuture('exec ./exec_daemon.php %s %Ls', $class, $argv)) ->setCWD($this->getDaemonCWD()) ->setStdoutSizeLimit($buffer_size) ->setStderrSizeLimit($buffer_size) ->write(json_encode($this->config)); } /** * Dispatch an event to event listeners. * * @param string Event type. * @param dict Event parameters. * @return void */ private function dispatchEvent($type, array $params = array()) { $data = array( 'id' => $this->daemonID, 'daemonClass' => $this->daemonClass, 'childPID' => $this->pid, ) + $params; $event = new PhutilEvent($type, $data); try { PhutilEventEngine::dispatchEvent($event); } catch (Exception $ex) { phlog($ex); } } private function annihilateProcessGroup() { $pid = $this->pid; $pgid = posix_getpgid($pid); if ($pid && $pgid) { posix_kill(-$pgid, SIGTERM); sleep($this->getKillDelay()); posix_kill(-$pgid, SIGKILL); $this->pid = null; } } private function updateMemory() { if ($this->traceMemory) { $this->logMessage( 'RAMS', pht( 'Overseer Memory Usage: %s KB', new PhutilNumber(memory_get_usage() / 1024, 1))); } } private function startDaemonProcess() { $this->logMessage('INIT', pht('Starting process.')); $this->deadline = time() + $this->getRequiredHeartbeatFrequency(); $this->heartbeat = time() + self::getHeartbeatEventFrequency(); $this->stdoutBuffer = ''; $this->future = $this->newExecFuture(); $this->future->start(); $this->pid = $this->future->getPID(); } private function didReadStdout($data) { $this->stdoutBuffer .= $data; while (true) { $pos = strpos($this->stdoutBuffer, "\n"); if ($pos === false) { break; } $message = substr($this->stdoutBuffer, 0, $pos); $this->stdoutBuffer = substr($this->stdoutBuffer, $pos + 1); try { $structure = phutil_json_decode($message); } catch (PhutilJSONParserException $ex) { $structure = array(); } switch (idx($structure, 0)) { case PhutilDaemon::MESSAGETYPE_STDOUT: $this->logMessage('STDO', idx($structure, 1)); break; case PhutilDaemon::MESSAGETYPE_HEARTBEAT: $this->deadline = time() + $this->getRequiredHeartbeatFrequency(); break; case PhutilDaemon::MESSAGETYPE_BUSY: $this->overseer->didBeginWork($this); break; case PhutilDaemon::MESSAGETYPE_IDLE: $this->overseer->didBeginIdle($this); break; case PhutilDaemon::MESSAGETYPE_DOWN: // The daemon is exiting because it doesn't have enough work and it // is trying to scale the pool down. We should not restart it. $this->shouldRestart = false; $this->shouldShutdown = true; break; default: // If we can't parse this or it isn't a message we understand, just // emit the raw message. $this->logMessage('STDO', pht(' %s', $message)); break; } } } public function didReceiveNotifySignal($signo) { $pid = $this->pid; if ($pid) { posix_kill($pid, $signo); } } public function didReceiveReloadSignal($signo) { $signame = phutil_get_signal_name($signo); if ($signame) { $sigmsg = pht( 'Reloading in response to signal %d (%s).', $signo, $signame); } else { $sigmsg = pht( 'Reloading in response to signal %d.', $signo); } $this->logMessage('RELO', $sigmsg, $signo); // This signal means "stop the current process gracefully, then launch // a new identical process once it exits". This can be used to update // daemons after code changes (the new processes will run the new code) // without aborting any running tasks. // We SIGINT the daemon but don't set the shutdown flag, so it will // naturally be restarted after it exits, as though it had exited after an // unhandled exception. posix_kill($this->pid, SIGINT); } public function didReceiveGracefulSignal($signo) { $this->shouldShutdown = true; $this->shouldRestart = false; $signame = phutil_get_signal_name($signo); if ($signame) { $sigmsg = pht( 'Graceful shutdown in response to signal %d (%s).', $signo, $signame); } else { $sigmsg = pht( 'Graceful shutdown in response to signal %d.', $signo); } $this->logMessage('DONE', $sigmsg, $signo); posix_kill($this->pid, SIGINT); } public function didReceiveTerminalSignal($signo) { $this->shouldShutdown = true; $this->shouldRestart = false; $signame = phutil_get_signal_name($signo); if ($signame) { $sigmsg = pht( 'Shutting down in response to signal %s (%s).', $signo, $signame); } else { $sigmsg = pht('Shutting down in response to signal %s.', $signo); } $this->logMessage('EXIT', $sigmsg, $signo); $this->annihilateProcessGroup(); } private function logMessage($type, $message, $context = null) { - if (!$this->getSilent()) { - echo date('Y-m-d g:i:s A').' ['.$type.'] '.$message."\n"; - } - + $this->overseer->logMessage($type, $message, $context); $this->dispatchEvent( self::EVENT_DID_LOG, array( 'type' => $type, 'message' => $message, 'context' => $context, )); } public function didRemoveDaemon() { $this->dispatchEvent(self::EVENT_WILL_EXIT); } } diff --git a/src/daemon/PhutilDaemonOverseer.php b/src/daemon/PhutilDaemonOverseer.php index 9feb626..adf9928 100644 --- a/src/daemon/PhutilDaemonOverseer.php +++ b/src/daemon/PhutilDaemonOverseer.php @@ -1,506 +1,516 @@ enableDiscardMode(); $args = new PhutilArgumentParser($argv); $args->setTagline(pht('daemon overseer')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( array( 'name' => 'trace-memory', 'help' => pht('Enable debug memory tracing.'), ), array( 'name' => 'verbose', 'help' => pht('Enable verbose activity logging.'), ), array( 'name' => 'label', 'short' => 'l', 'param' => 'label', 'help' => pht( 'Optional process label. Makes "%s" nicer, no behavioral effects.', 'ps'), ), )); $argv = array(); if ($args->getArg('trace')) { $this->traceMode = true; $argv[] = '--trace'; } if ($args->getArg('trace-memory')) { $this->traceMode = true; $this->traceMemory = true; $argv[] = '--trace-memory'; } $verbose = $args->getArg('verbose'); if ($verbose) { $this->verbose = true; $argv[] = '--verbose'; } $label = $args->getArg('label'); if ($label) { $argv[] = '-l'; $argv[] = $label; } $this->argv = $argv; if (function_exists('posix_isatty') && posix_isatty(STDIN)) { fprintf(STDERR, pht('Reading daemon configuration from stdin...')."\n"); } $config = @file_get_contents('php://stdin'); $config = id(new PhutilJSONParser())->parse($config); $this->libraries = idx($config, 'load'); $this->log = idx($config, 'log'); $this->daemonize = idx($config, 'daemonize'); $this->piddir = idx($config, 'piddir'); $this->config = $config; if (self::$instance) { throw new Exception( pht('You may not instantiate more than one Overseer per process.')); } self::$instance = $this; $this->startEpoch = time(); // Check this before we daemonize, since if it's an issue the child will // exit immediately. if ($this->piddir) { $dir = $this->piddir; try { Filesystem::assertWritable($dir); } catch (Exception $ex) { throw new Exception( pht( "Specified daemon PID directory ('%s') does not exist or is ". "not writable by the daemon user!", $dir)); } } if (!idx($config, 'daemons')) { throw new PhutilArgumentUsageException( pht('You must specify at least one daemon to start!')); } if ($this->log) { // NOTE: Now that we're committed to daemonizing, redirect the error // log if we have a `--log` parameter. Do this at the last moment // so as many setup issues as possible are surfaced. ini_set('error_log', $this->log); } if ($this->daemonize) { // We need to get rid of these or the daemon will hang when we TERM it // waiting for something to read the buffers. TODO: Learn how unix works. fclose(STDOUT); fclose(STDERR); ob_start(); $pid = pcntl_fork(); if ($pid === -1) { throw new Exception(pht('Unable to fork!')); } else if ($pid) { exit(0); } } $this->modules = PhutilDaemonOverseerModule::getAllModules(); - declare(ticks = 1); pcntl_signal(SIGUSR2, array($this, 'didReceiveNotifySignal')); pcntl_signal(SIGHUP, array($this, 'didReceiveReloadSignal')); pcntl_signal(SIGINT, array($this, 'didReceiveGracefulSignal')); pcntl_signal(SIGTERM, array($this, 'didReceiveTerminalSignal')); } public function addLibrary($library) { $this->libraries[] = $library; return $this; } public function run() { $this->daemons = array(); foreach ($this->config['daemons'] as $config) { $config += array( 'argv' => array(), 'autoscale' => array(), ); $daemon = new PhutilDaemonHandle( $this, $config['class'], $this->argv, array( 'log' => $this->log, 'argv' => $config['argv'], 'load' => $this->libraries, 'autoscale' => $config['autoscale'], )); - $daemon->setSilent((!$this->traceMode && !$this->verbose)); $daemon->setTraceMemory($this->traceMemory); $this->addDaemon($daemon, $config); } $should_reload = false; while (true) { foreach ($this->modules as $module) { try { if ($module->shouldReloadDaemons()) { + $this->logMessage( + 'RELO', + pht( + 'Reloading daemons (triggered by overseer module "%s").', + get_class($module))); $should_reload = true; } } catch (Exception $ex) { phlog($ex); } } + if ($should_reload) { $this->didReceiveReloadSignal(SIGHUP); $should_reload = false; } $futures = array(); foreach ($this->getDaemonHandles() as $daemon) { $daemon->update(); if ($daemon->isRunning()) { $futures[] = $daemon->getFuture(); } if ($daemon->isDone()) { $this->removeDaemon($daemon); } } $this->updatePidfile(); $this->updateAutoscale(); if ($futures) { $iter = id(new FutureIterator($futures)) ->setUpdateInterval(1); foreach ($iter as $future) { break; } } else { if ($this->inGracefulShutdown) { break; } sleep(1); } } exit($this->err); } private function addDaemon(PhutilDaemonHandle $daemon, array $config) { $id = $daemon->getDaemonID(); $this->daemons[$id] = array( 'handle' => $daemon, 'config' => $config, ); $autoscale_group = $this->getAutoscaleGroup($daemon); if ($autoscale_group) { $this->autoscale[$autoscale_group][$id] = true; } return $this; } private function removeDaemon(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); $autoscale_group = $this->getAutoscaleGroup($daemon); if ($autoscale_group) { unset($this->autoscale[$autoscale_group][$id]); } unset($this->daemons[$id]); $daemon->didRemoveDaemon(); return $this; } private function getAutoscaleGroup(PhutilDaemonHandle $daemon) { return $this->getAutoscaleProperty($daemon, 'group'); } private function getAutoscaleProperty( PhutilDaemonHandle $daemon, $key, $default = null) { $id = $daemon->getDaemonID(); $autoscale = $this->daemons[$id]['config']['autoscale']; return idx($autoscale, $key, $default); } public function didBeginWork(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); $busy = idx($this->daemons[$daemon->getDaemonID()], 'busy'); if (!$busy) { $this->daemons[$id]['busy'] = time(); } } public function didBeginIdle(PhutilDaemonHandle $daemon) { $id = $daemon->getDaemonID(); unset($this->daemons[$id]['busy']); } public function updateAutoscale() { foreach ($this->autoscale as $group => $daemons) { $daemon = $this->daemons[head_key($daemons)]['handle']; $scaleup_duration = $this->getAutoscaleProperty($daemon, 'up', 2); $max_pool_size = $this->getAutoscaleProperty($daemon, 'pool', 8); $reserve = $this->getAutoscaleProperty($daemon, 'reserve', 0); // Don't scale a group if it is already at the maximum pool size. if (count($daemons) >= $max_pool_size) { continue; } $should_scale = true; foreach ($daemons as $daemon_id => $ignored) { $busy = idx($this->daemons[$daemon_id], 'busy'); if (!$busy) { // At least one daemon in the group hasn't reported that it has // started work. $should_scale = false; break; } if ((time() - $busy) < $scaleup_duration) { // At least one daemon in the group was idle recently, so we have // not fullly $should_scale = false; break; } } // If we have a configured memory reserve for this pool, it tells us that // we should not scale up unless there's at least that much memory left // on the system (for example, a reserve of 0.25 means that 25% of system // memory must be free to autoscale). if ($should_scale && $reserve) { // On some systems this may be slightly more expensive than other // checks, so only do it once we're prepared to scale up. $memory = PhutilSystem::getSystemMemoryInformation(); $free_ratio = ($memory['free'] / $memory['total']); // If we don't have enough free memory, don't scale. if ($free_ratio <= $reserve) { continue; } } if ($should_scale) { $config = $this->daemons[$daemon_id]['config']; $config['autoscale']['clone'] = true; $clone = new PhutilDaemonHandle( $this, $config['class'], $this->argv, array( 'log' => $this->log, 'argv' => $config['argv'], 'load' => $this->libraries, 'autoscale' => $config['autoscale'], )); $this->addDaemon($clone, $config); // Don't scale more than one pool up per iteration. Otherwise, we could // break the memory barrier if we have a lot of pools and scale them // all up at once. return; } } } public function didReceiveNotifySignal($signo) { foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveNotifySignal($signo); } } public function didReceiveReloadSignal($signo) { foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveReloadSignal($signo); } } public function didReceiveGracefulSignal($signo) { // If we receive SIGINT more than once, interpret it like SIGTERM. if ($this->inGracefulShutdown) { return $this->didReceiveTerminalSignal($signo); } $this->inGracefulShutdown = true; foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveGracefulSignal($signo); } } public function didReceiveTerminalSignal($signo) { $this->err = 128 + $signo; if ($this->inAbruptShutdown) { exit($this->err); } $this->inAbruptShutdown = true; foreach ($this->getDaemonHandles() as $daemon) { $daemon->didReceiveTerminalSignal($signo); } } private function getDaemonHandles() { return ipull($this->daemons, 'handle'); } /** * Identify running daemons by examining the process table. This isn't * completely reliable, but can be used as a fallback if the pid files fail * or we end up with stray daemons by other means. * * Example output (array keys are process IDs): * * array( * 12345 => array( * 'type' => 'overseer', * 'command' => 'php launch_daemon.php --daemonize ...', * 'pid' => 12345, * ), * 12346 => array( * 'type' => 'daemon', * 'command' => 'php exec_daemon.php ...', * 'pid' => 12346, * ), * ); * * @return dict Map of PIDs to process information, identifying running * daemon processes. */ public static function findRunningDaemons() { $results = array(); list($err, $processes) = exec_manual('ps -o pid,command -a -x -w -w -w'); if ($err) { return $results; } $processes = array_filter(explode("\n", trim($processes))); foreach ($processes as $process) { list($pid, $command) = preg_split('/\s+/', trim($process), 2); $pattern = '/((launch|exec)_daemon.php|phd-daemon)/'; $matches = null; if (!preg_match($pattern, $command, $matches)) { continue; } switch ($matches[1]) { case 'exec_daemon.php': $type = 'daemon'; break; case 'launch_daemon.php': case 'phd-daemon': default: $type = 'overseer'; break; } $results[(int)$pid] = array( 'type' => $type, 'command' => $command, 'pid' => (int)$pid, ); } return $results; } private function updatePidfile() { if (!$this->piddir) { return; } $daemons = array(); foreach ($this->daemons as $daemon) { $handle = $daemon['handle']; $config = $daemon['config']; if (!$handle->isRunning()) { continue; } $daemons[] = array( 'pid' => $handle->getPID(), 'id' => $handle->getDaemonID(), 'config' => $config, ); } $pidfile = array( 'pid' => getmypid(), 'start' => $this->startEpoch, 'config' => $this->config, 'daemons' => $daemons, ); if ($pidfile !== $this->lastPidfile) { $this->lastPidfile = $pidfile; $pidfile_path = $this->piddir.'/daemon.'.getmypid(); Filesystem::writeFile($pidfile_path, json_encode($pidfile)); } } + public function logMessage($type, $message, $context = null) { + if ($this->traceMode || $this->verbose) { + error_log(date('Y-m-d g:i:s A').' ['.$type.'] '.$message); + } + } + } diff --git a/src/future/http/HTTPSFuture.php b/src/future/http/HTTPSFuture.php index 289f412..255e7f6 100644 --- a/src/future/http/HTTPSFuture.php +++ b/src/future/http/HTTPSFuture.php @@ -1,686 +1,689 @@ cabundle = $temp; return $this; } /** * Set the SSL certificate to use for this session, given a path. * * @param string The path to a valid SSL certificate for this session * @return this */ public function setCABundleFromPath($path) { $this->cabundle = $path; return $this; } /** * Get the path to the SSL certificate for this session. * * @return string|null */ public function getCABundle() { return $this->cabundle; } /** * Set whether Location headers in the response will be respected. * The default is true. * * @param boolean true to follow any Location header present in the response, * false to return the request directly * @return this */ public function setFollowLocation($follow) { $this->followLocation = $follow; return $this; } /** * Get whether Location headers in the response will be respected. * * @return boolean */ public function getFollowLocation() { return $this->followLocation; } /** * Set the fallback CA certificate if one is not specified * for the session, given a path. * * @param string The path to a valid SSL certificate * @return void */ public static function setGlobalCABundleFromPath($path) { self::$globalCABundle = $path; } /** * Set the fallback CA certificate if one is not specified * for the session, given a string. * * @param string The certificate * @return void */ public static function setGlobalCABundleFromString($certificate) { $temp = new TempFile(); Filesystem::writeFile($temp, $certificate); self::$globalCABundle = $temp; } /** * Get the fallback global CA certificate * * @return string */ public static function getGlobalCABundle() { return self::$globalCABundle; } - /** - * Set a list of domains to blindly trust. Certificates for these domains - * will not be validated. - * - * @param list List of domain names to trust blindly. - * @return void - */ - public static function setBlindlyTrustDomains(array $domains) { - self::$blindTrustDomains = array_fuse($domains); - } - /** * Load contents of remote URI. Behaves pretty much like * `@file_get_contents($uri)` but doesn't require `allow_url_fopen`. * * @param string * @param float * @return string|false */ public static function loadContent($uri, $timeout = null) { $future = new HTTPSFuture($uri); if ($timeout !== null) { $future->setTimeout($timeout); } try { list($body) = $future->resolvex(); return $body; } catch (HTTPFutureResponseStatus $ex) { return false; } } /** * Attach a file to the request. * * @param string HTTP parameter name. * @param string File content. * @param string File name. * @param string File mime type. * @return this */ public function attachFileData($key, $data, $name, $mime_type) { if (isset($this->files[$key])) { throw new Exception( pht( '%s currently supports only one file attachment for each '. 'parameter name. You are trying to attach two different files with '. 'the same parameter, "%s".', __CLASS__, $key)); } $this->files[$key] = array( 'data' => $data, 'name' => $name, 'mime' => $mime_type, ); return $this; } public function isReady() { if (isset($this->result)) { return true; } $uri = $this->getURI(); $domain = id(new PhutilURI($uri))->getDomain(); if (!$this->handle) { + $uri_object = new PhutilURI($uri); + $proxy = PhutilHTTPEngineExtension::buildHTTPProxyURI($uri_object); + $profiler = PhutilServiceProfiler::getInstance(); $this->profilerCallID = $profiler->beginServiceCall( array( 'type' => 'http', 'uri' => $uri, + 'proxy' => (string)$proxy, )); if (!self::$multi) { self::$multi = curl_multi_init(); if (!self::$multi) { throw new Exception(pht('%s failed!', 'curl_multi_init()')); } } if (!empty(self::$pool[$domain])) { $curl = array_pop(self::$pool[$domain]); } else { $curl = curl_init(); if (!$curl) { throw new Exception(pht('%s failed!', 'curl_init()')); } } $this->handle = $curl; curl_multi_add_handle(self::$multi, $curl); curl_setopt($curl, CURLOPT_URL, $uri); if (defined('CURLOPT_PROTOCOLS')) { // cURL supports a lot of protocols, and by default it will honor // redirects across protocols (for instance, from HTTP to POP3). Beyond // being very silly, this also has security implications: // // http://blog.volema.com/curl-rce.html // // Disable all protocols other than HTTP and HTTPS. $allowed_protocols = CURLPROTO_HTTPS | CURLPROTO_HTTP; curl_setopt($curl, CURLOPT_PROTOCOLS, $allowed_protocols); curl_setopt($curl, CURLOPT_REDIR_PROTOCOLS, $allowed_protocols); } if (strlen($this->rawBody)) { if ($this->getData()) { throw new Exception( pht( 'You can not execute an HTTP future with both a raw request '. 'body and structured request data.')); } // We aren't actually going to use this file handle, since we are // just pushing data through the callback, but cURL gets upset if // we don't hand it a real file handle. $tmp = new TempFile(); $this->fileHandle = fopen($tmp, 'r'); // NOTE: We must set CURLOPT_PUT here to make cURL use CURLOPT_INFILE. // We'll possibly overwrite the method later on, unless this is really // a PUT request. curl_setopt($curl, CURLOPT_PUT, true); curl_setopt($curl, CURLOPT_INFILE, $this->fileHandle); curl_setopt($curl, CURLOPT_INFILESIZE, strlen($this->rawBody)); curl_setopt($curl, CURLOPT_READFUNCTION, array($this, 'willWriteBody')); } else { $data = $this->formatRequestDataForCURL(); curl_setopt($curl, CURLOPT_POSTFIELDS, $data); } $headers = $this->getHeaders(); $saw_expect = false; for ($ii = 0; $ii < count($headers); $ii++) { list($name, $value) = $headers[$ii]; $headers[$ii] = $name.': '.$value; if (!strncasecmp($name, 'Expect', strlen('Expect'))) { $saw_expect = true; } } if (!$saw_expect) { // cURL sends an "Expect" header by default for certain requests. While // there is some reasoning behind this, it causes a practical problem // in that lighttpd servers reject these requests with a 417. Both sides // are locked in an eternal struggle (lighttpd has introduced a // 'server.reject-expect-100-with-417' option to deal with this case). // // The ostensibly correct way to suppress this behavior on the cURL side // is to add an empty "Expect:" header. If we haven't seen some other // explicit "Expect:" header, do so. // // See here, for example, although this issue is fairly widespread: // http://curl.haxx.se/mail/archive-2009-07/0008.html $headers[] = 'Expect:'; } curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); // Set the requested HTTP method, e.g. GET / POST / PUT. curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->getMethod()); // Make sure we get the headers and data back. curl_setopt($curl, CURLOPT_HEADER, true); curl_setopt($curl, CURLOPT_WRITEFUNCTION, array($this, 'didReceiveDataCallback')); if ($this->followLocation) { curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); curl_setopt($curl, CURLOPT_MAXREDIRS, 20); } if (defined('CURLOPT_TIMEOUT_MS')) { // If CURLOPT_TIMEOUT_MS is available, use the higher-precision timeout. $timeout = max(1, ceil(1000 * $this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT_MS, $timeout); } else { // Otherwise, fall back to the lower-precision timeout. $timeout = max(1, ceil($this->getTimeout())); curl_setopt($curl, CURLOPT_TIMEOUT, $timeout); } // We're going to try to set CAINFO below. This doesn't work at all on // OSX around Yosemite (see T5913). On these systems, we'll use the // system CA and then try to tell the user that their settings were // ignored and how to fix things if we encounter a CA-related error. // Assume we have custom CA settings to start with; we'll clear this // flag if we read the default CA info below. // Try some decent fallbacks here: // - First, check if a bundle is set explicitly for this request, via // `setCABundle()` or similar. // - Then, check if a global bundle is set explicitly for all requests, // via `setGlobalCABundle()` or similar. // - Then, if a local custom.pem exists, use that, because it probably // means that the user wants to override everything (also because the // user might not have access to change the box's php.ini to add // curl.cainfo). // - Otherwise, try using curl.cainfo. If it's set explicitly, it's // probably reasonable to try using it before we fall back to what // libphutil ships with. // - Lastly, try the default that libphutil ships with. If it doesn't // work, give up and yell at the user. if (!$this->getCABundle()) { $caroot = dirname(phutil_get_library_root('phutil')).'/resources/ssl/'; $ini_val = ini_get('curl.cainfo'); if (self::getGlobalCABundle()) { $this->setCABundleFromPath(self::getGlobalCABundle()); } else if (Filesystem::pathExists($caroot.'custom.pem')) { $this->setCABundleFromPath($caroot.'custom.pem'); } else if ($ini_val) { // TODO: We can probably do a pathExists() here, even. $this->setCABundleFromPath($ini_val); } else { $this->setCABundleFromPath($caroot.'default.pem'); } } if ($this->canSetCAInfo()) { curl_setopt($curl, CURLOPT_CAINFO, $this->getCABundle()); } - $domain = id(new PhutilURI($uri))->getDomain(); - if (!empty(self::$blindTrustDomains[$domain])) { - // Disable peer verification for domains that we blindly trust. - curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); - } else { - curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); + $verify_peer = 1; + $verify_host = 2; + + $extensions = PhutilHTTPEngineExtension::getAllExtensions(); + foreach ($extensions as $extension) { + if ($extension->shouldTrustAnySSLAuthorityForURI($uri_object)) { + $verify_peer = 0; + } + if ($extension->shouldTrustAnySSLHostnameForURI($uri_object)) { + $verify_host = 0; + } } + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, $verify_peer); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, $verify_host); curl_setopt($curl, CURLOPT_SSLVERSION, 0); + + if ($proxy) { + curl_setopt($curl, CURLOPT_PROXY, (string)$proxy); + } } else { $curl = $this->handle; if (!self::$results) { // NOTE: In curl_multi_select(), PHP calls curl_multi_fdset() but does // not check the return value of &maxfd for -1 until recent versions // of PHP (5.4.8 and newer). cURL may return -1 as maxfd in some unusual // situations; if it does, PHP enters select() with nfds=0, which blocks // until the timeout is reached. // // We could try to guess whether this will happen or not by examining // the version identifier, but we can also just sleep for only a short // period of time. curl_multi_select(self::$multi, 0.01); } } do { $active = null; $result = curl_multi_exec(self::$multi, $active); } while ($result == CURLM_CALL_MULTI_PERFORM); while ($info = curl_multi_info_read(self::$multi)) { if ($info['msg'] == CURLMSG_DONE) { self::$results[(int)$info['handle']] = $info; } } if (!array_key_exists((int)$curl, self::$results)) { return false; } // The request is complete, so release any temporary files we wrote // earlier. $this->temporaryFiles = array(); $info = self::$results[(int)$curl]; $result = $this->responseBuffer; $err_code = $info['result']; if ($err_code) { if (($err_code == CURLE_SSL_CACERT) && !$this->canSetCAInfo()) { $status = new HTTPFutureCertificateResponseStatus( HTTPFutureCertificateResponseStatus::ERROR_IMMUTABLE_CERTIFICATES, $uri); } else { $status = new HTTPFutureCURLResponseStatus($err_code, $uri); } $body = null; $headers = array(); $this->result = array($status, $body, $headers); } else { // cURL returns headers of all redirects, we strip all but the final one. $redirects = curl_getinfo($curl, CURLINFO_REDIRECT_COUNT); $result = preg_replace('/^(.*\r\n\r\n){'.$redirects.'}/sU', '', $result); $this->result = $this->parseRawHTTPResponse($result); } curl_multi_remove_handle(self::$multi, $curl); unset(self::$results[(int)$curl]); // NOTE: We want to use keepalive if possible. Return the handle to a // pool for the domain; don't close it. if ($this->shouldReuseHandles()) { self::$pool[$domain][] = $curl; } $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($this->profilerCallID, array()); return true; } /** * Callback invoked by cURL as it reads HTTP data from the response. We save * the data to a buffer. */ public function didReceiveDataCallback($handle, $data) { $this->responseBuffer .= $data; return strlen($data); } /** * Read data from the response buffer. * * NOTE: Like @{class:ExecFuture}, this method advances a read cursor but * does not discard the data. The data will still be buffered, and it will * all be returned when the future resolves. To discard the data after * reading it, call @{method:discardBuffers}. * * @return string Response data, if available. */ public function read() { $result = substr($this->responseBuffer, $this->responseBufferPos); $this->responseBufferPos = strlen($this->responseBuffer); return $result; } /** * Discard any buffered data. Normally, you call this after reading the * data with @{method:read}. * * @return this */ public function discardBuffers() { $this->responseBuffer = ''; $this->responseBufferPos = 0; return $this; } /** * Produces a value safe to pass to `CURLOPT_POSTFIELDS`. * * @return wild Some value, suitable for use in `CURLOPT_POSTFIELDS`. */ private function formatRequestDataForCURL() { // We're generating a value to hand to cURL as CURLOPT_POSTFIELDS. The way // cURL handles this value has some tricky caveats. // First, we can return either an array or a query string. If we return // an array, we get a "multipart/form-data" request. If we return a // query string, we get an "application/x-www-form-urlencoded" request. // Second, if we return an array we can't duplicate keys. The user might // want to send the same parameter multiple times. // Third, if we return an array and any of the values start with "@", // cURL includes arbitrary files off disk and sends them to an untrusted // remote server. For example, an array like: // // array('name' => '@/usr/local/secret') // // ...will attempt to read that file off disk and transmit its contents with // the request. This behavior is pretty surprising, and it can easily // become a relatively severe security vulnerability which allows an // attacker to read any file the HTTP process has access to. Since this // feature is very dangerous and not particularly useful, we prevent its // use. Broadly, this means we must reject some requests because they // contain an "@" in an inconvenient place. // Generally, to avoid the "@" case and because most servers usually // expect "application/x-www-form-urlencoded" data, we try to return a // string unless there are files attached to this request. $data = $this->getData(); $files = $this->files; $any_data = ($data || (is_string($data) && strlen($data))); $any_files = (bool)$this->files; if (!$any_data && !$any_files) { // No files or data, so just bail. return null; } if (!$any_files) { // If we don't have any files, just encode the data as a query string, // make sure it's not including any files, and we're good to go. if (is_array($data)) { $data = http_build_query($data, '', '&'); } $this->checkForDangerousCURLMagic($data, $is_query_string = true); return $data; } // If we've made it this far, we have some files, so we need to return // an array. First, convert the other data into an array if it isn't one // already. if (is_string($data)) { // NOTE: We explicitly don't want fancy array parsing here, so just // do a basic parse and then convert it into a dictionary ourselves. $parser = new PhutilQueryStringParser(); $pairs = $parser->parseQueryStringToPairList($data); $map = array(); foreach ($pairs as $pair) { list($key, $value) = $pair; if (array_key_exists($key, $map)) { throw new Exception( pht( 'Request specifies two values for key "%s", but parameter '. 'names must be unique if you are posting file data due to '. 'limitations with cURL.', $key)); } $map[$key] = $value; } $data = $map; } foreach ($data as $key => $value) { $this->checkForDangerousCURLMagic($value, $is_query_string = false); } foreach ($this->files as $name => $info) { if (array_key_exists($name, $data)) { throw new Exception( pht( 'Request specifies a file with key "%s", but that key is also '. 'defined by normal request data. Due to limitations with cURL, '. 'requests that post file data must use unique keys.', $name)); } $tmp = new TempFile($info['name']); Filesystem::writeFile($tmp, $info['data']); $this->temporaryFiles[] = $tmp; // In 5.5.0 and later, we can use CURLFile. Prior to that, we have to // use this "@" stuff. if (class_exists('CURLFile', false)) { $file_value = new CURLFile((string)$tmp, $info['mime'], $info['name']); } else { $file_value = '@'.(string)$tmp; } $data[$name] = $file_value; } return $data; } /** * Detect strings which will cause cURL to do horrible, insecure things. * * @param string Possibly dangerous string. * @param bool True if this string is being used as part of a query string. * @return void */ private function checkForDangerousCURLMagic($string, $is_query_string) { if (empty($string[0]) || ($string[0] != '@')) { // This isn't an "@..." string, so it's fine. return; } if ($is_query_string) { if (version_compare(phpversion(), '5.2.0', '<')) { throw new Exception( pht( 'Attempting to make an HTTP request, but query string data begins '. 'with "%s". Prior to PHP 5.2.0 this reads files off disk, which '. 'creates a wide attack window for security vulnerabilities. '. 'Upgrade PHP or avoid making cURL requests which begin with "%s".', '@', '@')); } // This is safe if we're on PHP 5.2.0 or newer. return; } throw new Exception( pht( 'Attempting to make an HTTP request which includes file data, but the '. 'value of a query parameter begins with "%s". PHP interprets these '. 'values to mean that it should read arbitrary files off disk and '. 'transmit them to remote servers. Declining to make this request.', '@')); } /** * Determine whether CURLOPT_CAINFO is usable on this system. */ private function canSetCAInfo() { // We cannot set CAInfo on OSX after Yosemite. $osx_version = PhutilExecutionEnvironment::getOSXVersion(); if ($osx_version) { if (version_compare($osx_version, 14, '>=')) { return false; } } return true; } /** * Write a raw HTTP body into the request. * * You must write the entire body before starting the request. * * @param string Raw body. * @return this */ public function write($raw_body) { $this->rawBody = $raw_body; return $this; } /** * Callback to pass data to cURL. */ public function willWriteBody($handle, $infile, $len) { $bytes = substr($this->rawBody, $this->rawBodyPos, $len); $this->rawBodyPos += $len; return $bytes; } private function shouldReuseHandles() { $curl_version = curl_version(); $version = idx($curl_version, 'version'); // NOTE: cURL 7.43.0 has a bug where the POST body length is not recomputed // properly when a handle is reused. For this version of cURL, disable // handle reuse and accept a small performance penalty. See T8654. if ($version == '7.43.0') { return false; } return true; } } diff --git a/src/future/http/PhutilHTTPEngineExtension.php b/src/future/http/PhutilHTTPEngineExtension.php new file mode 100644 index 0000000..6c1beb9 --- /dev/null +++ b/src/future/http/PhutilHTTPEngineExtension.php @@ -0,0 +1,141 @@ +getPhobjectClassConstant('EXTENSIONKEY'); + } + + final public static function getAllExtensions() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExtensionKey') + ->execute(); + } + + final public static function getExtension($key) { + $extensions = self::getAllExtensions(); + return idx($extensions, $key); + } + + final public static function requireExtension($key) { + $extension = self::getExtension($key); + + if (!$extension) { + throw new Exception( + pht( + 'No HTTP engine extension exists with extension key "%s".', + $key)); + } + + return $extension; + } + + final public static function buildHTTPProxyURI(PhutilURI $uri) { + $proxy = null; + $via = null; + + $extensions = self::getAllExtensions(); + foreach ($extensions as $extension) { + $extension_proxy = $extension->getHTTPProxyURI($uri); + + if ($extension_proxy === null) { + continue; + } + + if (!($extension_proxy instanceof PhutilURI)) { + throw new Exception( + pht( + 'HTTP extension "%s" (of class "%s") returned an invalid '. + 'result from "%s": expected null, or an object of class "%s".', + $extension->getExtensionName(), + get_class($extension), + 'getHTTPProxyURI()', + 'PhutilURI')); + } + + if ($proxy) { + throw new Exception( + pht( + 'Two different HTTP extensions ("%s" of class "%s" and "%s" of '. + 'class "%s") both provided a proxy URI for URI "%s". No more '. + 'than one extension may provide a proxy for any URI.', + $extension->getExtensionName(), + get_class($extension), + $via->getExtensionName(), + get_class($via), + (string)$uri)); + } + + $proxy = $extension_proxy; + $via = $extension; + } + + return $proxy; + } + +} diff --git a/src/serviceprofiler/PhutilServiceProfiler.php b/src/serviceprofiler/PhutilServiceProfiler.php index 352dcd6..4b0ed94 100644 --- a/src/serviceprofiler/PhutilServiceProfiler.php +++ b/src/serviceprofiler/PhutilServiceProfiler.php @@ -1,165 +1,178 @@ discardMode = true; } public static function getInstance() { if (empty(self::$instance)) { self::$instance = new PhutilServiceProfiler(); } return self::$instance; } public function beginServiceCall(array $data) { $data['begin'] = microtime(true); $id = $this->logSize++; $this->events[$id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'begin', $id, $data); } return $id; } public function endServiceCall($call_id, array $data) { $data = ($this->events[$call_id] + $data); $data['end'] = microtime(true); $data['duration'] = ($data['end'] - $data['begin']); $this->events[$call_id] = $data; foreach ($this->listeners as $listener) { call_user_func($listener, 'end', $call_id, $data); } if ($this->discardMode) { unset($this->events[$call_id]); } } public function getServiceCallLog() { return $this->events; } public function addListener($callback) { $this->listeners[] = $callback; } public static function installEchoListener() { $instance = self::getInstance(); $instance->addListener(array(__CLASS__, 'echoListener')); } public static function echoListener($type, $id, $data) { $is_begin = false; $is_end = false; switch ($type) { case 'begin': $is_begin = true; $mark = '>>>'; break; case 'end': $is_end = true; $mark = '<<<'; break; default: $mark = null; break; } $type = idx($data, 'type', 'mystery'); $desc = null; if ($is_begin) { switch ($type) { case 'connect': $desc = $data['database']; break; case 'query': $desc = substr($data['query'], 0, 512); break; case 'multi-query': $desc = array(); foreach ($data['queries'] as $query) { $desc[] = substr($query, 0, 256); } $desc = implode('; ', $desc); break; case 'exec': $desc = '$ '.$data['command']; break; case 'conduit': if (isset($data['size'])) { $desc = $data['method'].'() '; $desc .= pht('', new PhutilNumber($data['size'])); } else { $desc = $data['method'].'()'; } break; case 'http': - $desc = phutil_censor_credentials($data['uri']); + if (isset($data['proxy'])) { + $proxy = phutil_censor_credentials($data['proxy']); + } else { + $proxy = null; + } + + $uri = phutil_censor_credentials($data['uri']); + + if (strlen($proxy)) { + $desc = "{$proxy} >> {$uri}"; + } else { + $desc = $uri; + } + break; case 'lint': $desc = $data['linter']; if (isset($data['paths'])) { $desc .= ' '.pht('', phutil_count($data['paths'])); } break; case 'lock': $desc = $data['name']; break; case 'event': $desc = $data['kind'].' '; $desc .= pht('', new PhutilNumber($data['count'])); break; case 'ldap': $call = idx($data, 'call', '?'); $params = array(); switch ($call) { case 'connect': $params[] = $data['host'].':'.$data['port']; break; case 'start-tls': break; case 'bind': $params[] = $data['user']; break; case 'search': $params[] = $data['dn']; $params[] = $data['query']; break; default: $params[] = '?'; break; } $desc = "{$call} (".implode(', ', $params).")"; break; } } else if ($is_end) { $desc = pht( '%s us', new PhutilNumber((int)(1000000 * $data['duration']))); } $console = PhutilConsole::getConsole(); $console->writeLog( "%s [%s] <%s> %s\n", $mark, $id, $type, $desc); } } diff --git a/src/utils/PhutilEditDistanceMatrix.php b/src/utils/PhutilEditDistanceMatrix.php index fadfc83..1dd21f7 100644 --- a/src/utils/PhutilEditDistanceMatrix.php +++ b/src/utils/PhutilEditDistanceMatrix.php @@ -1,491 +1,555 @@ setSequences(str_split('ran'), str_split('rat')); * * $cost = $matrix->getEditDistance(); * * Edit distance computation is slow and requires a large amount of memory; * both are roughly O(N * M) in the length of the strings. * * You can customize the cost of insertions, deletions and replacements. By * default, each has a cost of 1. * * $matrix->setReplaceCost(1); * * By default, transpositions are not evaluated (i.e., the algorithm is * Levenshtein). You can cause transpositions to be evaluated by setting a * transpose cost (which will change the algorithm to Damerau-Levenshtein): * * $matrix->setTransposeCost(1); * * You can also provide a cost to alter the type of operation being applied. * Many strings have several equivalently expensive edit sequences, but one * some sequences are more readable to humans than others. Providing a small * cost to alter operation type tends to smooth out the sequence and produce * long runs of a single operation, which are generally more readable. For * example, these strings: * * (x) * ((x)) * * ...have edit strings "issis" and "isssi", which describe edit operations with * the same cost, but the latter string is more comprehensible to human viewers. * * If you set an alter cost, you must call @{method:setComputeString} to enable * type computation. The alter cost should generally be smaller than `c / N`, * where `c` is the smallest operational cost and `N` is the length of the * longest string. For example, if you are using the default costs (insert = 1, * delete = 1, replace = 1) and computing edit distances for strings of fewer * than 1,000 characters, you might set the alter cost to 0.001. */ final class PhutilEditDistanceMatrix extends Phobject { private $insertCost = 1; private $deleteCost = 1; private $replaceCost = 1; private $transposeCost = null; private $alterCost = 0; private $maximumLength; private $computeString; + private $applySmoothing = self::SMOOTHING_NONE; private $x; private $y; private $prefix; private $suffix; private $distanceMatrix = null; private $typeMatrix = null; + const SMOOTHING_NONE = 'none'; + const SMOOTHING_INTERNAL = 'internal'; + const SMOOTHING_FULL = 'full'; + public function setMaximumLength($maximum_length) { $this->maximumLength = $maximum_length; return $this; } public function getMaximumLength() { return coalesce($this->maximumLength, $this->getInfinity()); } public function setComputeString($compute_string) { $this->computeString = $compute_string; return $this; } public function getComputeString() { return $this->computeString; } public function setTransposeCost($transpose_cost) { $this->transposeCost = $transpose_cost; return $this; } public function getTransposeCost() { return $this->transposeCost; } public function setReplaceCost($replace_cost) { $this->replaceCost = $replace_cost; return $this; } public function getReplaceCost() { return $this->replaceCost; } public function setDeleteCost($delete_cost) { $this->deleteCost = $delete_cost; return $this; } public function getDeleteCost() { return $this->deleteCost; } public function setInsertCost($insert_cost) { $this->insertCost = $insert_cost; return $this; } public function getInsertCost() { return $this->insertCost; } public function setAlterCost($alter_cost) { $this->alterCost = $alter_cost; return $this; } public function getAlterCost() { return $this->alterCost; } + public function setApplySmoothing($apply_smoothing) { + $this->applySmoothing = $apply_smoothing; + return $this; + } + + public function getApplySmoothing() { + return $this->applySmoothing; + } + public function setSequences(array $x, array $y) { // NOTE: We strip common prefixes and suffixes from the inputs because // the runtime of the edit distance algorithm is large and it is common // to diff similar strings. $xl = count($x); $yl = count($y); $l = min($xl, $yl); $prefix_l = 0; $suffix_l = 0; for ($ii = 0; $ii < $l; $ii++) { if ($x[$ii] !== $y[$ii]) { break; } $prefix_l++; } for ($ii = 1; $ii <= ($l - $prefix_l); $ii++) { if ($x[$xl - $ii] !== $y[$yl - $ii]) { break; } $suffix_l++; } $this->prefix = array_slice($x, 0, $prefix_l); $this->suffix = array_slice($x, $xl - $suffix_l); $this->x = array_slice($x, $prefix_l, $xl - ($suffix_l + $prefix_l)); $this->y = array_slice($y, $prefix_l, $yl - ($suffix_l + $prefix_l)); $this->distanceMatrix = null; return $this; } private function requireSequences() { if ($this->x === null) { throw new PhutilInvalidStateException('setSequences'); } } public function getEditDistance() { $this->requireSequences(); $x = $this->x; $y = $this->y; if (!$x) { return $this->insertCost * count($y); } if (!$y) { return $this->deleteCost * count($x); } $max = $this->getMaximumLength(); if (count($x) > $max || count($y) > $max) { return ($this->insertCost * count($y)) + ($this->deleteCost * count($x)); } if ($x === $y) { return 0; } $matrix = $this->getDistanceMatrix(); return $matrix[count($x)][count($y)]; } /** * Return a string representing the edits between the sequences. The string * has these characters: * * - s (same): Same character in both strings. * - i (insert): Character inserted. * - d (delete): Character deleted. * - x (replace): Character replaced. * - t (transpose): Character transposed. */ public function getEditString() { $this->requireSequences(); $x = $this->x; $y = $this->y; if (!$x && !$y) { return $this->padEditString(''); } if (!$x) { return $this->padEditString(str_repeat('i', count($y))); } if (!$y) { return $this->padEditString(str_repeat('d', count($x))); } if ($x === $y) { return $this->padEditString(str_repeat('s', count($x))); } $max = $this->getMaximumLength(); if (count($x) > $max || count($y) > $max) { return $this->padEditString( str_repeat('d', count($x)). str_repeat('i', count($y))); } $matrix = $this->getTypeMatrix(); $xx = count($x); $yy = count($y); $transposes = array(); $str = ''; while (true) { $type = $matrix[$xx][$yy]; if (is_array($type)) { $chr = 't'; $transposes[$type[0]][$type[1]] = true; $type = $type[2]; } else { $chr = $type; } if (isset($transposes[$xx][$yy])) { $chr = 't'; } if ($type == 's' || $type == 'x') { $xx -= 1; $yy -= 1; } else if ($type == 'i') { $yy -= 1; } else if ($type == 'd') { $xx -= 1; } else { throw new Exception(pht("Unknown type '%s' in type matrix.", $type)); } $str .= $chr; if ($xx <= 0 && $yy <= 0) { break; } } - return $this->padEditString(strrev($str)); + $str = strrev($str); + + // For full smoothing, we pad the edit string before smoothing it, so + // ranges of similar characters at the beginning or end of the string can + // also be smoothed. + + // For internal smoothing, we only smooth ranges within the change itself. + + $smoothing = $this->getApplySmoothing(); + switch ($smoothing) { + case self::SMOOTHING_FULL: + $str = $this->padEditString($str); + $str = $this->applySmoothing($str, true); + break; + case self::SMOOTHING_INTERNAL: + $str = $this->applySmoothing($str, false); + $str = $this->padEditString($str); + break; + case self::SMOOTHING_NONE: + $str = $this->padEditString($str); + break; + default: + throw new Exception( + pht( + 'Unknown smoothing type "%s".', + $smoothing)); + } + + return $str; } private function padEditString($str) { if ($this->prefix) { $str = str_repeat('s', count($this->prefix)).$str; } if ($this->suffix) { $str = $str.str_repeat('s', count($this->suffix)); } return $str; } private function getTypeMatrix() { if (!$this->computeString) { throw new PhutilInvalidStateException('setComputeString'); } if ($this->typeMatrix === null) { $this->computeMatrix($this->x, $this->y); } return $this->typeMatrix; } private function getDistanceMatrix() { if ($this->distanceMatrix === null) { $this->computeMatrix($this->x, $this->y); } return $this->distanceMatrix; } private function computeMatrix(array $x, array $y) { $xl = count($x); $yl = count($y); $m = array(); $infinity = $this->getInfinity(); $use_damerau = ($this->transposeCost !== null); if ($use_damerau) { // Initialize the default cost of a transpose. $m[-1][-1] = $infinity; // Initialize the character position dictionary for Damerau. $d = array(); foreach ($x as $c) { $d[$c] = -1; } foreach ($y as $c) { $d[$c] = -1; } // Initialize the transpose costs for Damerau. for ($xx = 0; $xx <= $xl; $xx++) { $m[$xx][-1] = $infinity; } for ($yy = 0; $yy <= $yl; $yy++) { $m[-1][$yy] = $infinity; } } // Initialize the top row of the matrix. for ($xx = 0; $xx <= $xl; $xx++) { $m[$xx][0] = $xx * $this->deleteCost; } // Initialize the left column of the matrix. $cost = 0; for ($yy = 0; $yy <= $yl; $yy++) { $m[0][$yy] = $yy * $this->insertCost; } $use_types = ($this->computeString); if ($use_types) { $t = array(); for ($xx = 0; $xx <= $xl; $xx++) { $t[$xx][0] = 'd'; } for ($yy = 0; $yy <= $yl; $yy++) { $t[0][$yy] = 'i'; } $t[0][0] = 's'; } $alt_cost = $this->getAlterCost(); if ($alt_cost && !$use_types) { throw new Exception( pht( 'If you provide an alter cost with %s, you must enable '. 'type computation with %s.', 'setAlterCost()', 'setComputeStrings()')); } // Build the edit distance matrix. for ($xx = 1; $xx <= $xl; $xx++) { $last_dy = -1; for ($yy = 1; $yy <= $yl; $yy++) { if ($use_damerau) { $dx = $d[$y[$yy - 1]]; $dy = $last_dy; } if ($x[$xx - 1] === $y[$yy - 1]) { $rep_cost = $m[$xx - 1][$yy - 1] + 0; $rep_type = 's'; } else { $rep_cost = $m[$xx - 1][$yy - 1] + $this->replaceCost; $rep_type = 'x'; } $del_cost = $m[$xx - 1][$yy ] + $this->deleteCost; $ins_cost = $m[$xx ][$yy - 1] + $this->insertCost; if ($alt_cost) { $del_char = $t[$xx - 1][$yy ]; $ins_char = $t[$xx ][$yy - 1]; $rep_char = $t[$xx - 1][$yy - 1]; if ($del_char !== 'd') { $del_cost += $alt_cost; } if ($ins_char !== 'i') { $ins_cost += $alt_cost; } if ($rep_char !== $rep_type) { $rep_cost += $alt_cost; } } if ($rep_cost <= $del_cost && $rep_cost <= $ins_cost) { $cost = $rep_cost; $type = $rep_type; if ($rep_type === 's') { $last_dy = $yy - 1; } } else if ($ins_cost <= $del_cost) { $cost = $ins_cost; $type = 'i'; } else { $cost = $del_cost; $type = 'd'; } if ($use_damerau) { $trn_count = (($xx - $dx - 2) + ($yy - $dy - 2) + 1); $trn_cost = $m[$dx][$dy] + ($trn_count * $this->transposeCost); if ($trn_cost < $cost) { $cost = $trn_cost; $type = array($dx + 1, $dy + 1, $type); } } $m[$xx][$yy] = $cost; if ($use_types) { $t[$xx][$yy] = $type; } } if ($use_damerau) { $d[$x[$xx - 1]] = ($xx - 1); } } $this->distanceMatrix = $m; if ($use_types) { $this->typeMatrix = $t; } } private function getInfinity() { return INF; } private function printMatrix(array $m) { $x = $this->x; $y = $this->y; $p = '% 8s '; printf($p, ' '); foreach (head($m) as $yk => $yv) { printf($p, idx($y, $yk - 1, '-')); } echo "\n"; foreach ($m as $xk => $xv) { printf($p, idx($x, $xk - 1, '-')); foreach ($xv as $yk => $yv) { printf($p, ($yv == $this->getInfinity() ? 'inf' : $yv)); } echo "\n"; } } private function printTypeMatrix(array $t) { $x = $this->x; $y = $this->y; $p = '% 8s '; printf($p, ' '); foreach (head($t) as $yk => $yv) { printf($p, idx($y, $yk - 1, '-')); } echo "\n"; foreach ($t as $xk => $xv) { printf($p, idx($x, $xk - 1, '-')); foreach ($xv as $yk => $yv) { printf($p, ($yv == $this->getInfinity() ? 'inf' : $yv)); } echo "\n"; } } + private function applySmoothing($str, $full) { + if ($full) { + $prefix = '(^|[xdi])'; + $suffix = '([xdi]|\z)'; + } else { + $prefix = '([xdi])'; + $suffix = '([xdi])'; + } + + // Smooth the string out, by replacing short runs of similar characters + // with 'x' operations. This makes the result more readable to humans, + // since there are fewer choppy runs of short added and removed substrings. + do { + $original = $str; + $str = preg_replace('/'.$prefix.'(s{3})'.$suffix.'/', '$1xxx$3', $str); + $str = preg_replace('/'.$prefix.'(s{2})'.$suffix.'/', '$1xx$3', $str); + $str = preg_replace('/'.$prefix.'(s{1})'.$suffix.'/', '$1x$3', $str); + } while ($str != $original); + + return $str; + } + } diff --git a/src/utils/PhutilProseDiff.php b/src/utils/PhutilProseDiff.php new file mode 100644 index 0000000..f1b5b2c --- /dev/null +++ b/src/utils/PhutilProseDiff.php @@ -0,0 +1,280 @@ +parts[] = array( + 'type' => $type, + 'text' => $text, + ); + return $this; + } + + public function getParts() { + return $this->parts; + } + + /** + * Get diff parts, but replace large blocks of unchanged text with "." + * parts representing missing context. + */ + public function getSummaryParts() { + $parts = $this->getParts(); + + $head_key = head_key($parts); + $last_key = last_key($parts); + + $results = array(); + foreach ($parts as $key => $part) { + $is_head = ($key == $head_key); + $is_last = ($key == $last_key); + + switch ($part['type']) { + case '=': + $pieces = $this->splitTextForSummary($part['text']); + + if ($is_head || $is_last) { + $need = 2; + } else { + $need = 3; + } + + // We don't have enough pieces to omit anything, so just continue. + if (count($pieces) < $need) { + $results[] = $part; + break; + } + + if (!$is_head) { + $results[] = array( + 'type' => '=', + 'text' => head($pieces), + ); + } + + $results[] = array( + 'type' => '.', + 'text' => null, + ); + + if (!$is_last) { + $results[] = array( + 'type' => '=', + 'text' => last($pieces), + ); + } + break; + default: + $results[] = $part; + break; + } + } + + return $results; + } + + + public function reorderParts() { + // Reorder sequences of removed and added sections to put all the "-" + // parts together first, then all the "+" parts together. This produces + // a more human-readable result than intermingling them. + + $o_run = array(); + $n_run = array(); + $result = array(); + foreach ($this->parts as $part) { + $type = $part['type']; + switch ($type) { + case '-': + $o_run[] = $part; + break; + case '+': + $n_run[] = $part; + break; + default: + if ($o_run || $n_run) { + foreach ($this->combineRuns($o_run, $n_run) as $merged_part) { + $result[] = $merged_part; + } + $o_run = array(); + $n_run = array(); + } + $result[] = $part; + break; + } + } + + if ($o_run || $n_run) { + foreach ($this->combineRuns($o_run, $n_run) as $part) { + $result[] = $part; + } + } + + // Now, combine consecuitive runs of the same type of change (like a + // series of "-" parts) into a single run. + $combined = array(); + + $last = null; + $last_text = null; + foreach ($result as $part) { + $type = $part['type']; + + if ($last !== $type) { + if ($last !== null) { + $combined[] = array( + 'type' => $last, + 'text' => $last_text, + ); + } + $last_text = null; + $last = $type; + } + + $last_text .= $part['text']; + } + + if ($last_text !== null) { + $combined[] = array( + 'type' => $last, + 'text' => $last_text, + ); + } + + $this->parts = $combined; + + return $this; + } + + private function combineRuns($o_run, $n_run) { + $o_merge = $this->mergeParts($o_run); + $n_merge = $this->mergeParts($n_run); + + // When removed and added blocks share a prefix or suffix, we sometimes + // want to count it as unchanged (for example, if it is whitespace) but + // sometimes want to count it as changed (for example, if it is a word + // suffix like "ing"). Find common prefixes and suffixes of these layout + // characters and emit them as "=" (unchanged) blocks. + + $layout_characters = array( + ' ' => true, + "\n" => true, + '.' => true, + '!' => true, + ',' => true, + '?' => true, + ); + + $o_text = $o_merge['text']; + $n_text = $n_merge['text']; + $o_len = strlen($o_text); + $n_len = strlen($n_text); + $min_len = min($o_len, $n_len); + + $prefix_len = 0; + for ($pos = 0; $pos < $min_len; $pos++) { + $o = $o_text[$pos]; + $n = $n_text[$pos]; + if ($o !== $n) { + break; + } + if (empty($layout_characters[$o])) { + break; + } + $prefix_len++; + } + + $suffix_len = 0; + for ($pos = 1; $pos <= $min_len; $pos++) { + $o = $o_text[$o_len - $pos]; + $n = $n_text[$n_len - $pos]; + if ($o !== $n) { + break; + } + if (empty($layout_characters[$o])) { + break; + } + $suffix_len++; + } + + $results = array(); + + if ($prefix_len) { + $results[] = array( + 'type' => '=', + 'text' => substr($o_text, 0, $prefix_len), + ); + } + + if ($prefix_len < $o_len) { + $results[] = array( + 'type' => '-', + 'text' => substr($o_text, $prefix_len, $o_len - $suffix_len), + ); + } + + if ($prefix_len < $n_len) { + $results[] = array( + 'type' => '+', + 'text' => substr($n_text, $prefix_len, $n_len - $suffix_len), + ); + } + + if ($suffix_len) { + $results[] = array( + 'type' => '=', + 'text' => substr($o_text, -$suffix_len), + ); + } + + return $results; + } + + private function mergeParts(array $parts) { + $text = ''; + $type = null; + foreach ($parts as $part) { + $part_type = $part['type']; + if ($type === null) { + $type = $part_type; + } + if ($type !== $part_type) { + throw new Exception(pht('Can not merge parts of dissimilar types!')); + } + $text .= $part['text']; + } + + return array( + 'type' => $type, + 'text' => $text, + ); + } + + private function splitTextForSummary($text) { + $matches = null; + + $ok = preg_match('/^(\n*[^\n]+)\n/', $text, $matches); + if (!$ok) { + return array($text); + } + + $head = $matches[1]; + $text = substr($text, strlen($head)); + + $ok = preg_match('/\n([^\n]+\n*)\z/', $text, $matches); + if (!$ok) { + return array($text); + } + + $last = $matches[1]; + $text = substr($text, 0, -strlen($last)); + + if (!strlen(trim($text))) { + return array($head, $last); + } else { + return array($head, $text, $last); + } + } + +} diff --git a/src/utils/PhutilProseDifferenceEngine.php b/src/utils/PhutilProseDifferenceEngine.php new file mode 100644 index 0000000..bc7e910 --- /dev/null +++ b/src/utils/PhutilProseDifferenceEngine.php @@ -0,0 +1,104 @@ +buildDiff($diff, $u, $v, 1); + $diff->reorderParts(); + + return $diff; + } + + private function buildDiff(PhutilProseDiff $diff, $u, $v, $level) { + if ($level == 4) { + $diff->addPart('-', $u); + $diff->addPart('+', $v); + return; + } + + $u_parts = $this->splitCorpus($u, $level); + $v_parts = $this->splitCorpus($v, $level); + + $matrix = id(new PhutilEditDistanceMatrix()) + ->setSequences($u_parts, $v_parts) + ->setComputeString(true); + + // For word-level and character-level changes, smooth the output string + // to reduce the choppiness of the diff. + if ($level > 1) { + $matrix->setApplySmoothing(PhutilEditDistanceMatrix::SMOOTHING_FULL); + } + + $u_pos = 0; + $v_pos = 0; + + $edits = $matrix->getEditString(); + $edits_length = strlen($edits); + + for ($ii = 0; $ii < $edits_length; $ii++) { + $c = $edits[$ii]; + if ($c == 's') { + $diff->addPart('=', $u_parts[$u_pos]); + $u_pos++; + $v_pos++; + } else if ($c == 'd') { + $diff->addPart('-', $u_parts[$u_pos]); + $u_pos++; + } else if ($c == 'i') { + $diff->addPart('+', $v_parts[$v_pos]); + $v_pos++; + } else if ($c == 'x') { + $this->buildDiff($diff, $u_parts[$u_pos], $v_parts[$v_pos], $level + 1); + $u_pos++; + $v_pos++; + } else { + throw new Exception( + pht( + 'Unexpected character ("%s") in edit string.', + $c)); + } + } + } + + private function splitCorpus($corpus, $level) { + switch ($level) { + case 1: + // Level 1: Split into sentences. + $expr = '/([\n,!;?\.]+)/'; + break; + case 2: + // Level 2: Split into words. + $expr = '/(\s+)/'; + break; + case 3: + // Level 3: Split into characters. + return phutil_utf8v_combined($corpus); + } + + $pieces = preg_split($expr, $corpus, -1, PREG_SPLIT_DELIM_CAPTURE); + return $this->stitchPieces($pieces); + } + + private function stitchPieces(array $pieces) { + $results = array(); + $count = count($pieces); + for ($ii = 0; $ii < $count; $ii += 2) { + $result = $pieces[$ii]; + if ($ii + 1 < $count) { + $result .= $pieces[$ii + 1]; + } + $results[] = $result; + } + + // If the input ended with a delimiter, we can get an empty final piece. + // Just discard it. + if (last($results) == '') { + array_pop($results); + } + + return $results; + } + +} diff --git a/src/utils/__tests__/PhutilProseDiffTestCase.php b/src/utils/__tests__/PhutilProseDiffTestCase.php new file mode 100644 index 0000000..fffd8b7 --- /dev/null +++ b/src/utils/__tests__/PhutilProseDiffTestCase.php @@ -0,0 +1,174 @@ +assertProseParts( + '', + '', + array(), + pht('Empty')); + + $this->assertProseParts( + "xxx\nyyy", + "xxx\nzzz\nyyy", + array( + "= xxx\n", + "+ zzz\n", + '= yyy', + ), + pht('Add Paragraph')); + + $this->assertProseParts( + "xxx\nzzz\nyyy", + "xxx\nyyy", + array( + "= xxx\n", + "- zzz\n", + '= yyy', + ), + pht('Remove Paragraph')); + + + // Without smoothing, the alogorithm identifies that "shark" and "cat" + // both contain the letter "a" and tries to express this as a very + // fine-grained edit which replaces "sh" with "c" and then "rk" with "t". + // This is technically correct, but it is much easier for human viewers to + // parse if we smooth this into a single removal and a single addition. + + $this->assertProseParts( + 'They say the shark has nine lives.', + 'They say the cat has nine lives.', + array( + '= They say the ', + '- shark', + '+ cat', + '= has nine lives.', + ), + pht('"Shark/cat" word edit smoothenss.')); + + $this->assertProseParts( + 'Rising quickly, she says', + 'Rising quickly, she remarks:', + array( + '= Rising quickly, she ', + '- says', + '+ remarks:', + ), + pht('"Says/remarks" word edit smoothenss.')); + + $this->assertProseParts( + 'See screenshots', + 'Viewed video files', + array( + '- See screenshots', + '+ Viewed video files', + ), + pht('Complete paragraph rewrite.')); + + $this->assertProseParts( + 'xaaax', + 'xbbbx', + array( + '- xaaax', + '+ xbbbx', + ), + pht('Whole word rewrite with common prefix and suffix.')); + + $this->assertProseParts( + ' aaa ', + ' bbb ', + array( + '= ', + '- aaa', + '+ bbb', + '= ', + ), + pht('Whole word rewrite with whitespace prefix and suffix.')); + + $this->assertSummaryProseParts( + "a\nb\nc\nd\ne\nf\ng\nh\n", + "a\nb\nc\nd\nX\nf\ng\nh\n", + array( + '.', + "= d\n", + '- e', + '+ X', + "= \nf", + '.', + ), + pht('Summary diff with middle change.')); + + $this->assertSummaryProseParts( + "a\nb\nc\nd\ne\nf\ng\nh\n", + "X\nb\nc\nd\ne\nf\ng\nh\n", + array( + '- a', + '+ X', + "= \nb", + '.', + ), + pht('Summary diff with head change.')); + + $this->assertSummaryProseParts( + "a\nb\nc\nd\ne\nf\ng\nh\n", + "a\nb\nc\nd\ne\nf\ng\nX\n", + array( + '.', + "= g\n", + '- h', + '+ X', + "= \n", + ), + pht('Summary diff with last change.')); + + } + + private function assertProseParts($old, $new, array $expect_parts, $label) { + $engine = new PhutilProseDifferenceEngine(); + $diff = $engine->getDiff($old, $new); + + $parts = $diff->getParts(); + + $this->assertParts($expect_parts, $parts, $label); + } + + private function assertSummaryProseParts( + $old, + $new, + array $expect_parts, + $label) { + + $engine = new PhutilProseDifferenceEngine(); + $diff = $engine->getDiff($old, $new); + + $parts = $diff->getSummaryParts(); + + $this->assertParts($expect_parts, $parts, $label); + } + + private function assertParts( + array $expect, + array $actual_parts, + $label) { + + $actual = array(); + foreach ($actual_parts as $actual_part) { + $type = $actual_part['type']; + $text = $actual_part['text']; + + switch ($type) { + case '.': + $actual[] = $type; + break; + default: + $actual[] = "{$type} {$text}"; + break; + } + } + + $this->assertEqual($expect, $actual, $label); + } + + +}