Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F15389146
D16227.id.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Referenced Files
None
Subscribers
None
D16227.id.diff
View Options
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
# Diviner
/docs/
/.divinercache/
+/src/.cache/
# libphutil
/src/.phutil_module_cache
diff --git a/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php b/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
--- a/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
+++ b/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
@@ -3,9 +3,13 @@
final class PhabricatorInternationalizationManagementExtractWorkflow
extends PhabricatorInternationalizationManagementWorkflow {
+ const CACHE_VERSION = 1;
+
protected function didConstruct() {
$this
->setName('extract')
+ ->setExamples(
+ '**extract** [__options__] __library__')
->setSynopsis(pht('Extract translatable strings.'))
->setArguments(
array(
@@ -13,44 +17,138 @@
'name' => 'paths',
'wildcard' => true,
),
+ array(
+ 'name' => 'clean',
+ 'help' => pht('Drop caches before extracting strings. Slow!'),
+ ),
));
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
+
$paths = $args->getArg('paths');
+ if (!$paths) {
+ $paths = array(getcwd());
+ }
- $futures = array();
+ $targets = array();
foreach ($paths as $path) {
$root = Filesystem::resolvePath($path);
- $path_files = id(new FileFinder($root))
- ->withType('f')
- ->withSuffix('php')
+
+ if (!Filesystem::pathExists($root) || !is_dir($root)) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Path "%s" does not exist, or is not a directory.',
+ $path));
+ }
+
+ $libraries = id(new FileFinder($path))
+ ->withPath('*/__phutil_library_init__.php')
->find();
+ if (!$libraries) {
+ throw new PhutilArgumentUsageException(
+ pht(
+ 'Path "%s" contains no libphutil libraries.',
+ $path));
+ }
- foreach ($path_files as $file) {
- $full_path = $root.DIRECTORY_SEPARATOR.$file;
- $data = Filesystem::readFile($full_path);
- $futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
+ foreach ($libraries as $library) {
+ $targets[] = Filesystem::resolvePath(dirname($library)).'/';
}
}
- $console->writeErr(
- "%s\n",
- pht('Found %s file(s)...', phutil_count($futures)));
+ $targets = array_unique($targets);
- $results = array();
+ foreach ($targets as $library) {
+ echo tsprintf(
+ "**<bg:blue> %s </bg>** %s\n",
+ pht('EXTRACT'),
+ pht(
+ 'Extracting "%s"...',
+ Filesystem::readablePath($library)));
+
+ $this->extractLibrary($library);
+ }
+
+ return 0;
+ }
+
+ private function extractLibrary($root) {
+ $files = $this->loadLibraryFiles($root);
+ $cache = $this->readCache($root);
+
+ $modified = $this->getModifiedFiles($files, $cache);
+ $cache['files'] = $files;
+
+ if ($modified) {
+ echo tsprintf(
+ "**<bg:blue> %s </bg>** %s\n",
+ pht('MODIFIED'),
+ pht(
+ 'Found %s modified file(s) (of %s total).',
+ phutil_count($modified),
+ phutil_count($files)));
+
+ $old_strings = idx($cache, 'strings');
+ $old_strings = array_select_keys($old_strings, $files);
+ $new_strings = $this->extractFiles($root, $modified);
+ $all_strings = $new_strings + $old_strings;
+ $cache['strings'] = $all_strings;
+
+ $this->writeStrings($root, $all_strings);
+ } else {
+ echo tsprintf(
+ "**<bg:blue> %s </bg>** %s\n",
+ pht('NOT MODIFIED'),
+ pht('Strings for this library are already up to date.'));
+ }
+
+ $cache = id(new PhutilJSON())->encodeFormatted($cache);
+ $this->writeCache($root, 'i18n_files.json', $cache);
+ }
+
+ private function getModifiedFiles(array $files, array $cache) {
+ $known = idx($cache, 'files', array());
+ $known = array_fuse($known);
+
+ $modified = array();
+ foreach ($files as $file => $hash) {
+
+ if (isset($known[$hash])) {
+ continue;
+ }
+ $modified[$file] = $hash;
+ }
+
+ return $modified;
+ }
+
+ private function extractFiles($root_path, array $files) {
+ $hashes = array();
+
+ $futures = array();
+ foreach ($files as $file => $hash) {
+ $full_path = $root_path.DIRECTORY_SEPARATOR.$file;
+ $data = Filesystem::readFile($full_path);
+ $futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
+
+ $hashes[$full_path] = $hash;
+ }
$bar = id(new PhutilConsoleProgressBar())
->setTotal(count($futures));
$messages = array();
+ $results = array();
$futures = id(new FutureIterator($futures))
->limit(8);
foreach ($futures as $full_path => $future) {
$bar->update(1);
+ $hash = $hashes[$full_path];
+
try {
$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
Filesystem::readFile($full_path),
@@ -67,24 +165,27 @@
$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
foreach ($calls as $call) {
$name = $call->getChildByIndex(0)->getConcreteString();
- if ($name == 'pht') {
- $params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
- $string_node = $params->getChildByIndex(0);
- $string_line = $string_node->getLineNumber();
- try {
- $string_value = $string_node->evalStatic();
-
- $results[$string_value][] = array(
- 'file' => Filesystem::readablePath($full_path),
- 'line' => $string_line,
- );
- } catch (Exception $ex) {
- $messages[] = pht(
- 'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
- $call->getLineNumber(),
- $full_path,
- $ex->getMessage());
- }
+ if ($name != 'pht') {
+ continue;
+ }
+
+ $params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
+ $string_node = $params->getChildByIndex(0);
+ $string_line = $string_node->getLineNumber();
+ try {
+ $string_value = $string_node->evalStatic();
+
+ $results[$hash][] = array(
+ 'string' => $string_value,
+ 'file' => Filesystem::readablePath($full_path, $root_path),
+ 'line' => $string_line,
+ );
+ } catch (Exception $ex) {
+ $messages[] = pht(
+ 'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
+ $call->getLineNumber(),
+ $full_path,
+ $ex->getMessage());
}
}
@@ -93,28 +194,109 @@
$bar->done();
foreach ($messages as $message) {
- $console->writeErr("%s\n", $message);
+ echo tsprintf(
+ "**<bg:yellow> %s </bg>** %s\n",
+ pht('WARNING'),
+ $message);
}
- ksort($results);
+ return $results;
+ }
- $out = array();
- $out[] = '<?php';
- $out[] = '// @no'.'lint';
- $out[] = 'return array(';
- foreach ($results as $string => $locations) {
- foreach ($locations as $location) {
- $out[] = ' // '.$location['file'].':'.$location['line'];
+ private function writeStrings($root, array $strings) {
+ $map = array();
+ foreach ($strings as $hash => $string_list) {
+ foreach ($string_list as $string_info) {
+ $map[$string_info['string']]['uses'][] = array(
+ 'file' => $string_info['file'],
+ 'line' => $string_info['line'],
+ );
}
- $out[] = " '".addcslashes($string, "\0..\37\\'\177..\377")."' => null,";
- $out[] = null;
}
- $out[] = ');';
- $out[] = null;
- echo implode("\n", $out);
+ ksort($map);
- return 0;
+ $json = id(new PhutilJSON())->encodeFormatted($map);
+ $this->writeCache($root, 'i18n_strings.json', $json);
+ }
+
+ private function loadLibraryFiles($root) {
+ $files = id(new FileFinder($root))
+ ->withType('f')
+ ->withSuffix('php')
+ ->excludePath('*/.*')
+ ->setGenerateChecksums(true)
+ ->find();
+
+ $map = array();
+ foreach ($files as $file => $hash) {
+ $file = Filesystem::readablePath($file, $root);
+ $file = ltrim($file, '/');
+
+ if (dirname($file) == '.') {
+ continue;
+ }
+
+ if (dirname($file) == 'extensions') {
+ continue;
+ }
+
+ $map[$file] = md5($hash.$file);
+ }
+
+ return $map;
+ }
+
+ private function readCache($root) {
+ $path = $this->getCachePath($root, 'i18n_files.json');
+
+ $default = array(
+ 'version' => self::CACHE_VERSION,
+ 'files' => array(),
+ 'strings' => array(),
+ );
+
+ if ($this->getArgv()->getArg('clean')) {
+ return $default;
+ }
+
+ if (!Filesystem::pathExists($path)) {
+ return $default;
+ }
+
+ try {
+ $data = Filesystem::readFile($path);
+ } catch (Exception $ex) {
+ return $default;
+ }
+
+ try {
+ $cache = phutil_json_decode($data);
+ } catch (PhutilJSONParserException $e) {
+ return $default;
+ }
+
+ $version = idx($cache, 'version');
+ if ($version !== self::CACHE_VERSION) {
+ return $default;
+ }
+
+ return $cache;
+ }
+
+ private function writeCache($root, $file, $data) {
+ $path = $this->getCachePath($root, $file);
+
+ $cache_dir = dirname($path);
+ if (!Filesystem::pathExists($cache_dir)) {
+ Filesystem::createDirectory($cache_dir, 0755, true);
+ }
+
+ Filesystem::writeFile($path, $data);
+ }
+
+ private function getCachePath($root, $to_file) {
+ return $root.'/.cache/'.$to_file;
}
}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Mar 16, 4:56 AM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7575624
Default Alt Text
D16227.id.diff (9 KB)
Attached To
Mode
D16227: Make i18n string extraction faster and more flexible
Attached
Detach File
Event Timeline
Log In to Comment