Differential D16227 Diff 39038 src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php

Changeset View

Standalone View

View Options

src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php

	<?php			<?php

	final class PhabricatorInternationalizationManagementExtractWorkflow			final class PhabricatorInternationalizationManagementExtractWorkflow
	extends PhabricatorInternationalizationManagementWorkflow {			extends PhabricatorInternationalizationManagementWorkflow {

				const CACHE_VERSION = 1;

	protected function didConstruct() {			protected function didConstruct() {
	$this			$this
	->setName('extract')			->setName('extract')
				->setExamples(
				'extract [__options__] __library__')
	->setSynopsis(pht('Extract translatable strings.'))			->setSynopsis(pht('Extract translatable strings.'))
	->setArguments(			->setArguments(
	array(			array(
	array(			array(
	'name' => 'paths',			'name' => 'paths',
	'wildcard' => true,			'wildcard' => true,
	),			),
				array(
				'name' => 'clean',
				'help' => pht('Drop caches before extracting strings. Slow!'),
				),
	));			));
	}			}

	public function execute(PhutilArgumentParser $args) {			public function execute(PhutilArgumentParser $args) {
	$console = PhutilConsole::getConsole();			$console = PhutilConsole::getConsole();

	$paths = $args->getArg('paths');			$paths = $args->getArg('paths');
				if (!$paths) {
				$paths = array(getcwd());
				}

	$futures = array();			$targets = array();
	foreach ($paths as $path) {			foreach ($paths as $path) {
	$root = Filesystem::resolvePath($path);			$root = Filesystem::resolvePath($path);
	$path_files = id(new FileFinder($root))
	->withType('f')			if (!Filesystem::pathExists($root) \|\| !is_dir($root)) {
	->withSuffix('php')			throw new PhutilArgumentUsageException(
				pht(
				'Path "%s" does not exist, or is not a directory.',
				$path));
				}

				$libraries = id(new FileFinder($path))
				->withPath('*/__phutil_library_init__.php')
	->find();			->find();
				if (!$libraries) {
				throw new PhutilArgumentUsageException(
				pht(
				'Path "%s" contains no libphutil libraries.',
				$path));
				}

	foreach ($path_files as $file) {			foreach ($libraries as $library) {
	$full_path = $root.DIRECTORY_SEPARATOR.$file;			$targets[] = Filesystem::resolvePath(dirname($library)).'/';
	$data = Filesystem::readFile($full_path);
	$futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
	}			}
	}			}

	$console->writeErr(			$targets = array_unique($targets);
	"%s\n",
	pht('Found %s file(s)...', phutil_count($futures)));

	$results = array();			foreach ($targets as $library) {
				echo tsprintf(
				"<bg:blue> %s </bg> %s\n",
				pht('EXTRACT'),
				pht(
				'Extracting "%s"...',
				Filesystem::readablePath($library)));

				$this->extractLibrary($library);
				}

				return 0;
				}

				private function extractLibrary($root) {
				$files = $this->loadLibraryFiles($root);
				$cache = $this->readCache($root);

				$modified = $this->getModifiedFiles($files, $cache);
				$cache['files'] = $files;

				if ($modified) {
				echo tsprintf(
				"<bg:blue> %s </bg> %s\n",
				pht('MODIFIED'),
				pht(
				'Found %s modified file(s) (of %s total).',
				phutil_count($modified),
				phutil_count($files)));

				$old_strings = idx($cache, 'strings');
				$old_strings = array_select_keys($old_strings, $files);
				$new_strings = $this->extractFiles($root, $modified);
				$all_strings = $new_strings + $old_strings;
				$cache['strings'] = $all_strings;

				$this->writeStrings($root, $all_strings);
				} else {
				echo tsprintf(
				"<bg:blue> %s </bg> %s\n",
				pht('NOT MODIFIED'),
				pht('Strings for this library are already up to date.'));
				}

				$cache = id(new PhutilJSON())->encodeFormatted($cache);
				$this->writeCache($root, 'i18n_files.json', $cache);
				}

				private function getModifiedFiles(array $files, array $cache) {
				$known = idx($cache, 'files', array());
				$known = array_fuse($known);

				$modified = array();
				foreach ($files as $file => $hash) {

				if (isset($known[$hash])) {
				continue;
				}
				$modified[$file] = $hash;
				}

				return $modified;
				}

				private function extractFiles($root_path, array $files) {
				$hashes = array();

				$futures = array();
				foreach ($files as $file => $hash) {
				$full_path = $root_path.DIRECTORY_SEPARATOR.$file;
				$data = Filesystem::readFile($full_path);
				$futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);

				$hashes[$full_path] = $hash;
				}

	$bar = id(new PhutilConsoleProgressBar())			$bar = id(new PhutilConsoleProgressBar())
	->setTotal(count($futures));			->setTotal(count($futures));

	$messages = array();			$messages = array();
				$results = array();

	$futures = id(new FutureIterator($futures))			$futures = id(new FutureIterator($futures))
	->limit(8);			->limit(8);
	foreach ($futures as $full_path => $future) {			foreach ($futures as $full_path => $future) {
	$bar->update(1);			$bar->update(1);

				$hash = $hashes[$full_path];

	try {			try {
	$tree = XHPASTTree::newFromDataAndResolvedExecFuture(			$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
	Filesystem::readFile($full_path),			Filesystem::readFile($full_path),
	$future->resolve());			$future->resolve());
	} catch (Exception $ex) {			} catch (Exception $ex) {
	$messages[] = pht(			$messages[] = pht(
	'WARNING: Failed to extract strings from file "%s": %s',			'WARNING: Failed to extract strings from file "%s": %s',
	$full_path,			$full_path,
	$ex->getMessage());			$ex->getMessage());
	continue;			continue;
	}			}

	$root = $tree->getRootNode();			$root = $tree->getRootNode();
	$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');			$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
	foreach ($calls as $call) {			foreach ($calls as $call) {
	$name = $call->getChildByIndex(0)->getConcreteString();			$name = $call->getChildByIndex(0)->getConcreteString();
	if ($name == 'pht') {			if ($name != 'pht') {
				continue;
				}

	$params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');			$params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
	$string_node = $params->getChildByIndex(0);			$string_node = $params->getChildByIndex(0);
	$string_line = $string_node->getLineNumber();			$string_line = $string_node->getLineNumber();
	try {			try {
	$string_value = $string_node->evalStatic();			$string_value = $string_node->evalStatic();

	$results[$string_value][] = array(			$results[$hash][] = array(
	'file' => Filesystem::readablePath($full_path),			'string' => $string_value,
				'file' => Filesystem::readablePath($full_path, $root_path),
	'line' => $string_line,			'line' => $string_line,
	);			);
	} catch (Exception $ex) {			} catch (Exception $ex) {
	$messages[] = pht(			$messages[] = pht(
	'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',			'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
	$call->getLineNumber(),			$call->getLineNumber(),
	$full_path,			$full_path,
	$ex->getMessage());			$ex->getMessage());
	}			}
	}			}
	}

	$tree->dispose();			$tree->dispose();
	}			}
	$bar->done();			$bar->done();

	foreach ($messages as $message) {			foreach ($messages as $message) {
	$console->writeErr("%s\n", $message);			echo tsprintf(
				"<bg:yellow> %s </bg> %s\n",
				pht('WARNING'),
				$message);
				}

				return $results;
				}

				private function writeStrings($root, array $strings) {
				$map = array();
				foreach ($strings as $hash => $string_list) {
				foreach ($string_list as $string_info) {
				$map[$string_info['string']]['uses'][] = array(
				'file' => $string_info['file'],
				'line' => $string_info['line'],
				);
				}
				}

				ksort($map);

				$json = id(new PhutilJSON())->encodeFormatted($map);
				$this->writeCache($root, 'i18n_strings.json', $json);
				}

				private function loadLibraryFiles($root) {
				$files = id(new FileFinder($root))
				->withType('f')
				->withSuffix('php')
				->excludePath('/.')
				->setGenerateChecksums(true)
				->find();

				$map = array();
				foreach ($files as $file => $hash) {
				$file = Filesystem::readablePath($file, $root);
				$file = ltrim($file, '/');

				if (dirname($file) == '.') {
				continue;
	}			}

	ksort($results);			if (dirname($file) == 'extensions') {
				continue;
				}

	$out = array();			$map[$file] = md5($hash.$file);
	$out[] = '<?php';
	$out[] = '// @no'.'lint';
	$out[] = 'return array(';
	foreach ($results as $string => $locations) {
	foreach ($locations as $location) {
	$out[] = ' // '.$location['file'].':'.$location['line'];
	}			}
	$out[] = " '".addcslashes($string, "\0..\37\\'\177..\377")."' => null,";
	$out[] = null;			return $map;
	}			}
	$out[] = ');';
	$out[] = null;

	echo implode("\n", $out);			private function readCache($root) {
				$path = $this->getCachePath($root, 'i18n_files.json');

	return 0;			$default = array(
				'version' => self::CACHE_VERSION,
				'files' => array(),
				'strings' => array(),
				);

				if ($this->getArgv()->getArg('clean')) {
				return $default;
				}

				if (!Filesystem::pathExists($path)) {
				return $default;
				}

				try {
				$data = Filesystem::readFile($path);
				} catch (Exception $ex) {
				return $default;
				}

				try {
				$cache = phutil_json_decode($data);
				} catch (PhutilJSONParserException $e) {
				return $default;
				}

				$version = idx($cache, 'version');
				if ($version !== self::CACHE_VERSION) {
				return $default;
				}

				return $cache;
				}

				private function writeCache($root, $file, $data) {
				$path = $this->getCachePath($root, $file);

				$cache_dir = dirname($path);
				if (!Filesystem::pathExists($cache_dir)) {
				Filesystem::createDirectory($cache_dir, 0755, true);
				}

				Filesystem::writeFile($path, $data);
				}

				private function getCachePath($root, $to_file) {
				return $root.'/.cache/'.$to_file;
	}			}

	}			}