diff --git a/src/moduleutils/PhutilLibraryMapBuilder.php b/src/moduleutils/PhutilLibraryMapBuilder.php index e590e092..f51fc464 100644 --- a/src/moduleutils/PhutilLibraryMapBuilder.php +++ b/src/moduleutils/PhutilLibraryMapBuilder.php @@ -1,534 +1,482 @@ root = $root; } - /** - * Control status output. Use `--quiet` to set this. - * - * @param bool If true, don't show status output. - * @return this - * - * @task map - */ - public function setQuiet($quiet) { - $this->quiet = $quiet; - return $this; - } - /** * Control subprocess parallelism limit. Use `--limit` to set this. * * @param int Maximum number of subprocesses to run in parallel. * @return this * * @task map */ public function setSubprocessLimit($limit) { $this->subprocessLimit = $limit; return $this; } /** * Get the map of symbols in this library, analyzing the library to build it * if necessary. * * @return map Information about symbols in this library. * * @task map */ public function buildMap() { if ($this->librarySymbolMap === null) { $this->analyzeLibrary(); } return $this->librarySymbolMap; } /** * Get the map of files in this library, analyzing the library to build it * if necessary. * * Returns a map of file paths to information about symbols used and defined * in the file. * * @return map Information about files in this library. * * @task map */ public function buildFileSymbolMap() { if ($this->fileSymbolMap === null) { $this->analyzeLibrary(); } return $this->fileSymbolMap; } /** * Build and update the library map. * * @return void * * @task map */ public function buildAndWriteMap() { $library_map = $this->buildMap(); - $this->log(pht('Writing map...')); $this->writeLibraryMap($library_map); } - /** - * Write a status message to the user, if not running in quiet mode. - * - * @param string Message to write. - * @return this - * - * @task map - */ - private function log($message) { - if (!$this->quiet) { - @fwrite(STDERR, "%s\n", $message); - } - return $this; - } - /* -( Path Management )---------------------------------------------------- */ /** * Get the path to some file in the library. * * @param string A library-relative path. If omitted, returns the library * root path. * @return string An absolute path. * * @task path */ private function getPath($path = '') { return $this->root.'/'.$path; } /** * Get the path to the symbol cache file. * * @return string Absolute path to symbol cache. * * @task path */ private function getPathForSymbolCache() { return $this->getPath('.phutil_module_cache'); } /** * Get the path to the map file. * * @return string Absolute path to the library map. * * @task path */ private function getPathForLibraryMap() { return $this->getPath('__phutil_library_map__.php'); } /** * Get the path to the library init file. * * @return string Absolute path to the library init file * * @task path */ private function getPathForLibraryInit() { return $this->getPath('__phutil_library_init__.php'); } /* -( Symbol Analysis and Caching )---------------------------------------- */ /** * Load the library symbol cache, if it exists and is readable and valid. * * @return dict Map of content hashes to cache of output from * `extract-symbols.php`. * * @task symbol */ private function loadSymbolCache() { $cache_file = $this->getPathForSymbolCache(); try { $cache = Filesystem::readFile($cache_file); } catch (Exception $ex) { $cache = null; } $symbol_cache = array(); if ($cache) { try { $symbol_cache = phutil_json_decode($cache); } catch (PhutilJSONParserException $ex) { $symbol_cache = array(); } } $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY); if ($version != self::SYMBOL_CACHE_VERSION) { // Throw away caches from a different version of the library. $symbol_cache = array(); } unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]); return $symbol_cache; } /** * Write a symbol map to disk cache. * * @param dict Symbol map of relative paths to symbols. * @param dict Source map (like @{method:loadSourceFileMap}). * @return void * * @task symbol */ private function writeSymbolCache(array $symbol_map, array $source_map) { $cache_file = $this->getPathForSymbolCache(); $cache = array( self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION, ); foreach ($symbol_map as $file => $symbols) { $cache[$source_map[$file]] = $symbols; } $json = json_encode($cache); - try { - Filesystem::writeFile($cache_file, $json); - } catch (FilesystemException $ex) { - $this->log(pht('Unable to save the cache!')); - } + Filesystem::writeFile($cache_file, $json); } /** * Drop the symbol cache, forcing a clean rebuild. * * @return this * * @task symbol */ public function dropSymbolCache() { - $this->log(pht('Dropping symbol cache...')); Filesystem::remove($this->getPathForSymbolCache()); } /** * Build a future which returns a `extract-symbols.php` analysis of a source * file. * * @param string Relative path to the source file to analyze. * @return Future Analysis future. * * @task symbol */ private function buildSymbolAnalysisFuture($file) { $absolute_file = $this->getPath($file); return self::newExtractSymbolsFuture( array(), array($absolute_file)); } private static function newExtractSymbolsFuture(array $flags, array $paths) { $bin = dirname(__FILE__).'/../../support/lib/extract-symbols.php'; return new ExecFuture( 'php -f %R -- --ugly %Ls -- %Ls', $bin, $flags, $paths); } public static function newBuiltinMap() { $future = self::newExtractSymbolsFuture( array('--builtins'), array()); list($json) = $future->resolvex(); return phutil_json_decode($json); } /* -( Source Management )-------------------------------------------------- */ /** * Build a map of all source files in a library to hashes of their content. * Returns an array like this: * * array( * 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3', * // ... * ); * * @return dict Map of library-relative paths to content hashes. * @task source */ private function loadSourceFileMap() { $root = $this->getPath(); $init = $this->getPathForLibraryInit(); if (!Filesystem::pathExists($init)) { throw new Exception( pht( "Provided path '%s' is not a %s library.", $root, 'phutil')); } $files = id(new FileFinder($root)) ->withType('f') ->withSuffix('php') ->excludePath('*/.*') ->setGenerateChecksums(true) ->find(); $extensions_dir = 'extensions/'; $extensions_len = strlen($extensions_dir); $map = array(); foreach ($files as $file => $hash) { $file = Filesystem::readablePath($file, $root); $file = ltrim($file, '/'); if (dirname($file) == '.') { // We don't permit normal source files at the root level, so just ignore // them; they're special library files. continue; } // Ignore files in the extensions/ directory. if (!strncmp($file, $extensions_dir, $extensions_len)) { continue; } // We include also filename in the hash to handle cases when the file is // moved without modifying its content. $map[$file] = md5($hash.$file); } return $map; } /** * Convert the symbol analysis of all the source files in the library into * a library map. * * @param dict Symbol analysis of all source files. * @return dict Library map. * @task source */ private function buildLibraryMap(array $symbol_map) { $library_map = array( 'class' => array(), 'function' => array(), 'xmap' => array(), ); // Detect duplicate symbols within the library. foreach ($symbol_map as $file => $info) { foreach ($info['have'] as $type => $symbols) { foreach ($symbols as $symbol => $declaration) { $lib_type = ($type == 'interface') ? 'class' : $type; if (!empty($library_map[$lib_type][$symbol])) { $prior = $library_map[$lib_type][$symbol]; throw new Exception( pht( "Definition of %s '%s' in file '%s' duplicates prior ". "definition in file '%s'. You can not declare the ". "same symbol twice.", $type, $symbol, $file, $prior)); } $library_map[$lib_type][$symbol] = $file; } } $library_map['xmap'] += $info['xmap']; } // Simplify the common case (one parent) to make the file a little easier // to deal with. foreach ($library_map['xmap'] as $class => $extends) { if (count($extends) == 1) { $library_map['xmap'][$class] = reset($extends); } } // Sort the map so it is relatively stable across changes. foreach ($library_map as $key => $symbols) { ksort($symbols); $library_map[$key] = $symbols; } ksort($library_map); return $library_map; } /** * Write a finalized library map. * * @param dict Library map structure to write. * @return void * * @task source */ private function writeLibraryMap(array $library_map) { $map_file = $this->getPathForLibraryMap(); $version = self::LIBRARY_MAP_VERSION; $library_map = array( self::LIBRARY_MAP_VERSION_KEY => $version, ) + $library_map; $library_map = phutil_var_export($library_map); $at = '@'; $source_file = <<log(pht('Finding source files...')); $source_map = $this->loadSourceFileMap(); - $this->log( - pht('Found %s files.', new PhutilNumber(count($source_map)))); // Load the symbol cache with existing parsed symbols. This allows us // to remap libraries quickly by analyzing only changed files. - $this->log(pht('Loading symbol cache...')); $symbol_cache = $this->loadSymbolCache(); // If the XHPAST binary is not up-to-date, build it now. Otherwise, // `extract-symbols.php` will attempt to build the binary and will fail // miserably because it will be trying to build the same file multiple // times in parallel. if (!PhutilXHPASTBinary::isAvailable()) { PhutilXHPASTBinary::build(); } // Build out the symbol analysis for all the files in the library. For // each file, check if it's in cache. If we miss in the cache, do a fresh // analysis. $symbol_map = array(); $futures = array(); foreach ($source_map as $file => $hash) { if (!empty($symbol_cache[$hash])) { $symbol_map[$file] = $symbol_cache[$hash]; continue; } $futures[$file] = $this->buildSymbolAnalysisFuture($file); } - $this->log( - pht('Found %s files in cache.', new PhutilNumber(count($symbol_map)))); // Run the analyzer on any files which need analysis. if ($futures) { $limit = $this->subprocessLimit; - $this->log( - pht( - 'Analyzing %s file(s) with %s subprocess(es)...', - phutil_count($futures), - new PhutilNumber($limit))); - $progress = new PhutilConsoleProgressBar(); - if ($this->quiet) { - $progress->setQuiet(true); - } $progress->setTotal(count($futures)); $futures = id(new FutureIterator($futures)) ->limit($limit); foreach ($futures as $file => $future) { $result = $future->resolveJSON(); if (empty($result['error'])) { $symbol_map[$file] = $result; } else { $progress->done(false); throw new XHPASTSyntaxErrorException( $result['line'], $file.': '.$result['error']); } $progress->update(1); } $progress->done(); } $this->fileSymbolMap = $symbol_map; // We're done building the cache, so write it out immediately. Note that // we've only retained entries for files we found, so this implicitly cleans // out old cache entries. $this->writeSymbolCache($symbol_map, $source_map); // Our map is up to date, so either show it on stdout or write it to disk. - $this->log(pht('Building library map...')); - $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); } } diff --git a/src/workflow/ArcanistLiberateWorkflow.php b/src/workflow/ArcanistLiberateWorkflow.php index 1a85cbbc..9d6b476d 100644 --- a/src/workflow/ArcanistLiberateWorkflow.php +++ b/src/workflow/ArcanistLiberateWorkflow.php @@ -1,225 +1,243 @@ newWorkflowInformation() ->setSynopsis( pht('Create or update an Arcanist library.')) ->addExample(pht('**liberate**')) ->addExample(pht('**liberate** [__path__]')) ->setHelp($help); } public function getWorkflowArguments() { return array( $this->newWorkflowArgument('clean') ->setHelp( pht('Perform a clean rebuild, ignoring caches. Thorough, but slow.')), $this->newWorkflowArgument('argv') ->setWildcard(true) ->setIsPathArgument(true), ); } protected function newPrompts() { return array( $this->newPrompt('arc.liberate.create') ->setDescription( pht( 'Confirms creation of a new library.')), ); } public function runWorkflow() { $log = $this->getLogEngine(); $argv = $this->getArgument('argv'); if (count($argv) > 1) { throw new ArcanistUsageException( pht( 'Provide only one path to "arc liberate". The path should identify '. 'a directory where you want to create or update a library.')); } else if (!$argv) { $log->writeStatus( pht('SCAN'), pht('Searching for libraries in the current working directory...')); $init_files = id(new FileFinder(getcwd())) ->withPath('*/__phutil_library_init__.php') ->find(); if (!$init_files) { throw new ArcanistUsageException( pht( 'Unable to find any libraries under the current working '. 'directory. To create a library, provide a path.')); } $paths = array(); foreach ($init_files as $init) { $paths[] = Filesystem::resolvePath(dirname($init)); } } else { $paths = array( Filesystem::resolvePath(head($argv)), ); } + $any_errors = false; foreach ($paths as $path) { $log->writeStatus( pht('WORK'), pht( 'Updating library: %s', Filesystem::readablePath($path).DIRECTORY_SEPARATOR)); - $this->liberatePath($path); + $exit_code = $this->liberatePath($path); + if ($exit_code !== 0) { + $any_errors = true; + $log->writeError( + pht('ERROR'), + pht('Failed to update library: %s', $path)); + } } - $log->writeSuccess( - pht('DONE'), - pht('Updated %s librarie(s).', phutil_count($paths))); + if (!$any_errors) { + $log->writeSuccess( + pht('DONE'), + pht('Updated %s librarie(s).', phutil_count($paths))); + } return 0; } + /** + * @return int The exit code of running the rebuild-map.php script, which + * will be 0 to indicate success or non-zero for failure. + */ private function liberatePath($path) { if (!Filesystem::pathExists($path.'/__phutil_library_init__.php')) { echo tsprintf( "%s\n", pht( 'No library currently exists at the path "%s"...', $path)); $this->liberateCreateDirectory($path); - $this->liberateCreateLibrary($path); - return; + return $this->liberateCreateLibrary($path); } $version = $this->getLibraryFormatVersion($path); switch ($version) { case 1: throw new ArcanistUsageException( pht( 'This very old library is no longer supported.')); case 2: return $this->liberateVersion2($path); default: throw new ArcanistUsageException( pht("Unknown library version '%s'!", $version)); } - - echo tsprintf("%s\n", pht('Done.')); } private function getLibraryFormatVersion($path) { $map_file = $path.'/__phutil_library_map__.php'; if (!Filesystem::pathExists($map_file)) { // Default to library v1. return 1; } $map = Filesystem::readFile($map_file); $matches = null; if (preg_match('/@phutil-library-version (\d+)/', $map, $matches)) { return (int)$matches[1]; } return 1; } + /** + * @return int The exit code of running the rebuild-map.php script, which + * will be 0 to indicate success or non-zero for failure. + */ private function liberateVersion2($path) { $bin = $this->getScriptPath('support/lib/rebuild-map.php'); $argv = array(); if ($this->getArgument('clean')) { $argv[] = '--drop-cache'; } return phutil_passthru( 'php -f %R -- %Ls %R', $bin, $argv, $path); } private function liberateCreateDirectory($path) { if (Filesystem::pathExists($path)) { if (!is_dir($path)) { throw new ArcanistUsageException( pht( 'Provide a directory to create or update a libphutil library in.')); } return; } echo tsprintf( "%!\n%W\n", pht('NEW LIBRARY'), pht( 'The directory "%s" does not exist. Do you want to create it?', $path)); $query = pht('Create new library?'); $this->getPrompt('arc.liberate.create') ->setQuery($query) ->execute(); execx('mkdir -p %R', $path); } + /** + * @return int The exit code of running the rebuild-map.php script, which + * will be 0 to indicate success or non-zero for failure. + */ private function liberateCreateLibrary($path) { $init_path = $path.'/__phutil_library_init__.php'; if (Filesystem::pathExists($init_path)) { - return; + return 0; } echo pht("Creating new libphutil library in '%s'.", $path)."\n"; do { echo pht('Choose a name for the new library.')."\n"; $name = phutil_console_prompt( pht('What do you want to name this library?')); if (preg_match('/^[a-z-]+$/', $name)) { break; } else { echo phutil_console_format( "%s\n", pht( 'Library name should contain only lowercase letters and hyphens.')); } } while (true); $template = "liberateVersion2($path); + return $this->liberateVersion2($path); } private function getScriptPath($script) { $root = dirname(phutil_get_library_root('arcanist')); return $root.'/'.$script; } } diff --git a/support/lib/rebuild-map.php b/support/lib/rebuild-map.php index 66c1c8be..a8baeef6 100755 --- a/support/lib/rebuild-map.php +++ b/support/lib/rebuild-map.php @@ -1,78 +1,73 @@ #!/usr/bin/env php setTagline(pht('rebuild the library map file')); $args->setSynopsis(<<parseStandardArguments(); $args->parse( array( - array( - 'name' => 'quiet', - 'help' => pht('Do not write status messages to stderr.'), - ), array( 'name' => 'drop-cache', 'help' => pht( 'Drop the symbol cache and rebuild the entire map from scratch.'), ), array( 'name' => 'limit', 'param' => 'N', 'default' => 8, 'help' => pht( 'Controls the number of symbol mapper subprocesses run at once. '. 'Defaults to 8.'), ), array( 'name' => 'show', 'help' => pht( 'Print symbol map to stdout instead of writing it to the map file.'), ), array( 'name' => 'ugly', 'help' => pht( 'Use faster but less readable serialization for "--show".'), ), array( 'name' => 'root', 'wildcard' => true, ), )); $root = $args->getArg('root'); if (count($root) !== 1) { throw new Exception(pht('Provide exactly one library root!')); } $root = Filesystem::resolvePath(head($root)); $builder = new PhutilLibraryMapBuilder($root); -$builder->setQuiet($args->getArg('quiet')); $builder->setSubprocessLimit($args->getArg('limit')); if ($args->getArg('drop-cache')) { $builder->dropSymbolCache(); } if ($args->getArg('show')) { $library_map = $builder->buildMap(); if ($args->getArg('ugly')) { echo json_encode($library_map); } else { echo id(new PhutilJSON())->encodeFormatted($library_map); } } else { $builder->buildAndWriteMap(); } exit(0);