diff --git a/src/lint/engine/ArcanistLintEngine.php b/src/lint/engine/ArcanistLintEngine.php index b88d29b8..b98a4514 100644 --- a/src/lint/engine/ArcanistLintEngine.php +++ b/src/lint/engine/ArcanistLintEngine.php @@ -1,582 +1,654 @@ configurationManager = $configuration_manager; return $this; } final public function getConfigurationManager() { return $this->configurationManager; } final public function setWorkingCopy( ArcanistWorkingCopyIdentity $working_copy) { $this->workingCopy = $working_copy; return $this; } final public function getWorkingCopy() { return $this->workingCopy; } final public function setPaths($paths) { $this->paths = $paths; return $this; } public function getPaths() { return $this->paths; } final public function setPathChangedLines($path, $changed) { if ($changed === null) { $this->changedLines[$path] = null; } else { $this->changedLines[$path] = array_fill_keys($changed, true); } return $this; } final public function getPathChangedLines($path) { return idx($this->changedLines, $path); } final public function setFileData($data) { $this->fileData = $data + $this->fileData; return $this; } final public function setCommitHookMode($mode) { $this->commitHookMode = $mode; return $this; } final public function setHookAPI(ArcanistHookAPI $hook_api) { $this->hookAPI = $hook_api; return $this; } final public function getHookAPI() { return $this->hookAPI; } final public function setEnableAsyncLint($enable_async_lint) { $this->enableAsyncLint = $enable_async_lint; return $this; } final public function getEnableAsyncLint() { return $this->enableAsyncLint; } final public function loadData($path) { if (!isset($this->fileData[$path])) { if ($this->getCommitHookMode()) { $this->fileData[$path] = $this->getHookAPI() ->getCurrentFileData($path); } else { $disk_path = $this->getFilePathOnDisk($path); $this->fileData[$path] = Filesystem::readFile($disk_path); } } return $this->fileData[$path]; } public function pathExists($path) { if ($this->getCommitHookMode()) { $file_data = $this->loadData($path); return ($file_data !== null); } else { $disk_path = $this->getFilePathOnDisk($path); return Filesystem::pathExists($disk_path); } } final public function isDirectory($path) { if ($this->getCommitHookMode()) { // TODO: This won't get the right result in every case (we need more // metadata) but should almost always be correct. try { $this->loadData($path); return false; } catch (Exception $ex) { return true; } } else { $disk_path = $this->getFilePathOnDisk($path); return is_dir($disk_path); } } final public function isBinaryFile($path) { try { $data = $this->loadData($path); } catch (Exception $ex) { return false; } return ArcanistDiffUtils::isHeuristicBinaryFile($data); } final public function isSymbolicLink($path) { return is_link($this->getFilePathOnDisk($path)); } final public function getFilePathOnDisk($path) { return Filesystem::resolvePath( $path, $this->getWorkingCopy()->getProjectRoot()); } final public function setMinimumSeverity($severity) { $this->minimumSeverity = $severity; return $this; } final public function getCommitHookMode() { return $this->commitHookMode; } final public function run() { $linters = $this->buildLinters(); if (!$linters) { throw new ArcanistNoEffectException('No linters to run.'); } + foreach ($linters as $key => $linter) { + $linter->setLinterID($key); + } + $linters = msort($linters, 'getLinterPriority'); foreach ($linters as $linter) { $linter->setEngine($this); } $have_paths = false; foreach ($linters as $linter) { if ($linter->getPaths()) { $have_paths = true; break; } } if (!$have_paths) { throw new ArcanistNoEffectException('No paths are lintable.'); } $versions = array($this->getCacheVersion()); foreach ($linters as $linter) { $version = get_class($linter).':'.$linter->getCacheVersion(); $symbols = id(new PhutilSymbolLoader()) ->setType('class') ->setName(get_class($linter)) ->selectSymbolsWithoutLoading(); $symbol = idx($symbols, 'class$'.get_class($linter)); if ($symbol) { $version .= ':'.md5_file( phutil_get_library_root($symbol['library']).'/'.$symbol['where']); } $versions[] = $version; } $this->cacheVersion = crc32(implode("\n", $versions)); - $this->stopped = array(); - $exceptions = array(); - foreach ($linters as $linter_name => $linter) { - if (!is_string($linter_name)) { - $linter_name = get_class($linter); - } - try { - if (!$linter->canRun()) { - continue; - } - $paths = $linter->getPaths(); - - foreach ($paths as $key => $path) { - // Make sure each path has a result generated, even if it is empty - // (i.e., the file has no lint messages). - $result = $this->getResultForPath($path); - if (isset($this->stopped[$path])) { - unset($paths[$key]); - } - if (isset($this->cachedResults[$path][$this->cacheVersion])) { - $cached_result = $this->cachedResults[$path][$this->cacheVersion]; - - $use_cache = $this->shouldUseCache( - $linter->getCacheGranularity(), - idx($cached_result, 'repository_version')); + $runnable = $this->getRunnableLinters($linters); - if ($use_cache) { - unset($paths[$key]); - - if (idx($cached_result, 'stopped') == $linter_name) { - $this->stopped[$path] = $linter_name; - } - } - } - } - $paths = array_values($paths); - - if ($paths) { - $profiler = PhutilServiceProfiler::getInstance(); - $call_id = $profiler->beginServiceCall(array( - 'type' => 'lint', - 'linter' => $linter_name, - 'paths' => $paths, - )); - - try { - $linter->willLintPaths($paths); - foreach ($paths as $path) { - $linter->willLintPath($path); - $linter->lintPath($path); - if ($linter->didStopAllLinters()) { - $this->stopped[$path] = $linter_name; - } - } - } catch (Exception $ex) { - $profiler->endServiceCall($call_id, array()); - throw $ex; - } - $profiler->endServiceCall($call_id, array()); - } - - } catch (Exception $ex) { - $exceptions[$linter_name] = $ex; - } - } + $this->stopped = array(); - $exceptions += $this->didRunLinters($linters); + $exceptions = $this->executeLinters($runnable); - foreach ($linters as $linter) { + foreach ($runnable as $linter) { foreach ($linter->getLintMessages() as $message) { if (!$this->isSeverityEnabled($message->getSeverity())) { continue; } if (!$this->isRelevantMessage($message)) { continue; } $message->setGranularity($linter->getCacheGranularity()); $result = $this->getResultForPath($message->getPath()); $result->addMessage($message); } } if ($this->cachedResults) { foreach ($this->cachedResults as $path => $messages) { $messages = idx($messages, $this->cacheVersion, array()); $repository_version = idx($messages, 'repository_version'); unset($messages['stopped']); unset($messages['repository_version']); foreach ($messages as $message) { $use_cache = $this->shouldUseCache( idx($message, 'granularity'), $repository_version); if ($use_cache) { $this->getResultForPath($path)->addMessage( ArcanistLintMessage::newFromDictionary($message)); } } } } foreach ($this->results as $path => $result) { $disk_path = $this->getFilePathOnDisk($path); $result->setFilePathOnDisk($disk_path); if (isset($this->fileData[$path])) { $result->setData($this->fileData[$path]); } else if ($disk_path && Filesystem::pathExists($disk_path)) { // TODO: this may cause us to, e.g., load a large binary when we only // raised an error about its filename. We could refine this by looking // through the lint messages and doing this load only if any of them // have original/replacement text or something like that. try { $this->fileData[$path] = Filesystem::readFile($disk_path); $result->setData($this->fileData[$path]); } catch (FilesystemException $ex) { // Ignore this, it's noncritical that we access this data and it // might be unreadable or a directory or whatever else for plenty // of legitimate reasons. } } } if ($exceptions) { throw new PhutilAggregateException('Some linters failed:', $exceptions); } return $this->results; } final public function isSeverityEnabled($severity) { $minimum = $this->minimumSeverity; return ArcanistLintSeverity::isAtLeastAsSevere($severity, $minimum); } final private function shouldUseCache( $cache_granularity, $repository_version) { if ($this->commitHookMode) { return false; } switch ($cache_granularity) { case ArcanistLinter::GRANULARITY_FILE: return true; case ArcanistLinter::GRANULARITY_DIRECTORY: case ArcanistLinter::GRANULARITY_REPOSITORY: return ($this->repositoryVersion == $repository_version); default: return false; } } /** * @param dict>> * @return this */ final public function setCachedResults(array $results) { $this->cachedResults = $results; return $this; } final public function getResults() { return $this->results; } final public function getStoppedPaths() { return $this->stopped; } abstract public function buildLinters(); - final protected function didRunLinters(array $linters) { - assert_instances_of($linters, 'ArcanistLinter'); - - $exceptions = array(); - $profiler = PhutilServiceProfiler::getInstance(); - - foreach ($linters as $linter_name => $linter) { - if (!is_string($linter_name)) { - $linter_name = get_class($linter); - } - - $call_id = $profiler->beginServiceCall(array( - 'type' => 'lint', - 'linter' => $linter_name, - )); - - try { - $linter->didRunLinters(); - } catch (Exception $ex) { - $exceptions[$linter_name] = $ex; - } - $profiler->endServiceCall($call_id, array()); - } - - return $exceptions; - } - final public function setRepositoryVersion($version) { $this->repositoryVersion = $version; return $this; } final private function isRelevantMessage(ArcanistLintMessage $message) { // When a user runs "arc lint", we default to raising only warnings on // lines they have changed (errors are still raised anywhere in the // file). The list of $changed lines may be null, to indicate that the // path is a directory or a binary file so we should not exclude // warnings. if (!$this->changedLines || $message->isError() || $message->shouldBypassChangedLineFiltering()) { return true; } $locations = $message->getOtherLocations(); $locations[] = $message->toDictionary(); foreach ($locations as $location) { $path = idx($location, 'path', $message->getPath()); if (!array_key_exists($path, $this->changedLines)) { continue; } $changed = $this->getPathChangedLines($path); if ($changed === null || !$location['line']) { return true; } $last_line = $location['line']; if (isset($location['original'])) { $last_line += substr_count($location['original'], "\n"); } for ($l = $location['line']; $l <= $last_line; $l++) { if (!empty($changed[$l])) { return true; } } } return false; } final protected function getResultForPath($path) { if (empty($this->results[$path])) { $result = new ArcanistLintResult(); $result->setPath($path); $result->setCacheVersion($this->cacheVersion); $this->results[$path] = $result; } return $this->results[$path]; } final public function getLineAndCharFromOffset($path, $offset) { if (!isset($this->charToLine[$path])) { $char_to_line = array(); $line_to_first_char = array(); $lines = explode("\n", $this->loadData($path)); $line_number = 0; $line_start = 0; foreach ($lines as $line) { $len = strlen($line) + 1; // Account for "\n". $line_to_first_char[] = $line_start; $line_start += $len; for ($ii = 0; $ii < $len; $ii++) { $char_to_line[] = $line_number; } $line_number++; } $this->charToLine[$path] = $char_to_line; $this->lineToFirstChar[$path] = $line_to_first_char; } $line = $this->charToLine[$path][$offset]; $char = $offset - $this->lineToFirstChar[$path][$line]; return array($line, $char); } final public function getPostponedLinters() { return $this->postponedLinters; } final public function setPostponedLinters(array $linters) { $this->postponedLinters = $linters; return $this; } protected function getCacheVersion() { return 1; } /** * Get a named linter resource shared by another linter. * * This mechanism allows linters to share arbitrary resources, like the * results of computation. If several linters need to perform the same * expensive computation step, they can use a named resource to synchronize * construction of the result so it doesn't need to be built multiple * times. * * @param string Resource identifier. * @param wild Optionally, default value to return if resource does not * exist. * @return wild Resource, or default value if not present. */ public function getLinterResource($key, $default = null) { return idx($this->linterResources, $key, $default); } /** * Set a linter resource that other linters can access. * * See @{method:getLinterResource} for a description of this mechanism. * * @param string Resource identifier. * @param wild Resource. * @return this */ public function setLinterResource($key, $value) { $this->linterResources[$key] = $value; return $this; } + + private function getRunnableLinters(array $linters) { + assert_instances_of($linters, 'ArcanistLinter'); + + // TODO: The canRun() mechanism is only used by one linter, and just + // silently disables the linter. Almost every other linter handles this + // by throwing `ArcanistMissingLinterException`. Both mechanisms are not + // ideal; linters which can not run should emit a message, get marked as + // "skipped", and allow execution to continue. See T7045. + + $runnable = array(); + foreach ($linters as $key => $linter) { + if ($linter->canRun()) { + $runnable[$key] = $linter; + } + } + + return $runnable; + } + + private function executeLinters(array $runnable) { + $all_paths = $this->getPaths(); + $path_chunks = array_chunk($all_paths, 32, $preserve_keys = true); + + $exception_lists = array(); + foreach ($path_chunks as $chunk) { + $exception_lists[] = $this->executeLintersOnChunk($runnable, $chunk); + } + + return array_mergev($exception_lists); + } + + + private function executeLintersOnChunk(array $runnable, array $path_list) { + assert_instances_of($runnable, 'ArcanistLinter'); + + $path_map = array_fuse($path_list); + + $exceptions = array(); + $did_lint = array(); + foreach ($runnable as $linter) { + $linter_id = $linter->getLinterID(); + $paths = $linter->getPaths(); + + foreach ($paths as $key => $path) { + // If we aren't running this path in the current chunk of paths, + // skip it completely. + if (empty($path_map[$path])) { + unset($paths[$key]); + continue; + } + + // Make sure each path has a result generated, even if it is empty + // (i.e., the file has no lint messages). + $result = $this->getResultForPath($path); + + // If a linter has stopped all other linters for this path, don't + // actually run the linter. + if (isset($this->stopped[$path])) { + unset($paths[$key]); + continue; + } + + // If we have a cached result for this path, don't actually run the + // linter. + if (isset($this->cachedResults[$path][$this->cacheVersion])) { + $cached_result = $this->cachedResults[$path][$this->cacheVersion]; + + $use_cache = $this->shouldUseCache( + $linter->getCacheGranularity(), + idx($cached_result, 'repository_version')); + + if ($use_cache) { + unset($paths[$key]); + if (idx($cached_result, 'stopped') == $linter_id) { + $this->stopped[$path] = $linter_id; + } + } + } + } + + $paths = array_values($paths); + + if (!$paths) { + continue; + } + + try { + $this->executeLinterOnPaths($linter, $paths); + $did_lint[] = array($linter, $paths); + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + + foreach ($did_lint as $info) { + list($linter, $paths) = $info; + try { + $this->executeDidLintOnPaths($linter, $paths); + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + + return $exceptions; + } + + private function beginLintServiceCall(ArcanistLinter $linter, array $paths) { + $profiler = PhutilServiceProfiler::getInstance(); + + return $profiler->beginServiceCall( + array( + 'type' => 'lint', + 'linter' => $linter->getInfoName(), + 'paths' => $paths, + )); + } + + private function endLintServiceCall($call_id) { + $profiler = PhutilServiceProfiler::getInstance(); + $profiler->endServiceCall($call_id, array()); + } + + private function executeLinterOnPaths(ArcanistLinter $linter, array $paths) { + $call_id = $this->beginLintServiceCall($linter, $paths); + + try { + $linter->willLintPaths($paths); + foreach ($paths as $path) { + $linter->setActivePath($path); + $linter->lintPath($path); + if ($linter->didStopAllLinters()) { + $this->stopped[$path] = $linter->getLinterID(); + } + } + } catch (Exception $ex) { + $this->endLintServiceCall($call_id); + throw $ex; + } + + $this->endLintServiceCall($call_id); + } + + private function executeDidLintOnPaths(ArcanistLinter $linter, array $paths) { + $call_id = $this->beginLintServiceCall($linter, $paths); + + try { + $linter->didLintPaths($paths); + } catch (Exception $ex) { + $this->endLintServiceCall($call_id); + throw $ex; + } + + $this->endLintServiceCall($call_id); + } + + } diff --git a/src/lint/linter/ArcanistBaseXHPASTLinter.php b/src/lint/linter/ArcanistBaseXHPASTLinter.php index 23296673..ffd80aff 100644 --- a/src/lint/linter/ArcanistBaseXHPASTLinter.php +++ b/src/lint/linter/ArcanistBaseXHPASTLinter.php @@ -1,160 +1,198 @@ getVersion(); $version = PhutilXHPASTBinary::getVersion(); if ($version) { $parts[] = $version; } return implode('-', $parts); } final protected function raiseLintAtToken( XHPASTToken $token, $code, $desc, $replace = null) { return $this->raiseLintAtOffset( $token->getOffset(), $code, $desc, $token->getValue(), $replace); } final protected function raiseLintAtNode( XHPASTNode $node, $code, $desc, $replace = null) { return $this->raiseLintAtOffset( $node->getOffset(), $code, $desc, $node->getConcreteString(), $replace); } final protected function buildFutures(array $paths) { return $this->getXHPASTLinter()->buildSharedFutures($paths); } + protected function didResolveLinterFutures(array $futures) { + $this->getXHPASTLinter()->releaseSharedFutures(array_keys($futures)); + } + /* -( Sharing Parse Trees )------------------------------------------------ */ /** * Get the linter object which is responsible for building parse trees. * * When the engine specifies that several XHPAST linters should execute, * we designate one of them as the one which will actually build parse trees. * The other linters share trees, so they don't have to recompute them. * * Roughly, the first linter to execute elects itself as the builder. * Subsequent linters request builds and retrieve results from it. * * @return ArcanistBaseXHPASTLinter Responsible linter. * @task sharing */ final protected function getXHPASTLinter() { $resource_key = 'xhpast.linter'; // If we're the first linter to run, share ourselves. Otherwise, grab the // previously shared linter. $engine = $this->getEngine(); $linter = $engine->getLinterResource($resource_key); if (!$linter) { $linter = $this; $engine->setLinterResource($resource_key, $linter); } $base_class = __CLASS__; if (!($linter instanceof $base_class)) { throw new Exception( pht( 'Expected resource "%s" to be an instance of "%s"!', $resource_key, $base_class)); } return $linter; } /** * Build futures on this linter, for use and to share with other linters. * * @param list Paths to build futures for. * @return list Futures. * @task sharing */ final protected function buildSharedFutures(array $paths) { foreach ($paths as $path) { if (!isset($this->futures[$path])) { $this->futures[$path] = PhutilXHPASTBinary::getParserFuture( $this->getData($path)); + $this->refcount[$path] = 1; + } else { + $this->refcount[$path]++; } } return array_select_keys($this->futures, $paths); } + + /** + * Release futures on this linter which are no longer in use elsewhere. + * + * @param list Paths to release futures for. + * @return void + * @task sharing + */ + final protected function releaseSharedFutures(array $paths) { + foreach ($paths as $path) { + if (empty($this->refcount[$path])) { + throw new Exception( + pht( + 'Imbalanced calls to shared futures: each call to '. + 'buildSharedFutures() for a path must be paired with a call to '. + 'releaseSharedFutures().')); + } + + $this->refcount[$path]--; + + if (!$this->refcount[$path]) { + unset($this->refcount[$path]); + unset($this->futures[$path]); + unset($this->trees[$path]); + unset($this->exceptions[$path]); + } + } + } + + /** * Get a path's tree from the responsible linter. * * @param string Path to retrieve tree for. * @return XHPASTTree|null Tree, or null if unparseable. * @task sharing */ final protected function getXHPASTTreeForPath($path) { // If we aren't the linter responsible for actually building the parse // trees, go get the tree from that linter. if ($this->getXHPASTLinter() !== $this) { return $this->getXHPASTLinter()->getXHPASTTreeForPath($path); } if (!array_key_exists($path, $this->trees)) { $this->trees[$path] = null; try { $this->trees[$path] = XHPASTTree::newFromDataAndResolvedExecFuture( $this->getData($path), $this->futures[$path]->resolve()); $root = $this->trees[$path]->getRootNode(); $root->buildSelectCache(); $root->buildTokenCache(); } catch (Exception $ex) { $this->exceptions[$path] = $ex; } } return $this->trees[$path]; } /** * Get a path's parse exception from the responsible linter. * * @param string Path to retrieve exception for. * @return Exeption|null Parse exception, if available. * @task sharing */ final protected function getXHPASTExceptionForPath($path) { if ($this->getXHPASTLinter() !== $this) { return $this->getXHPASTLinter()->getXHPASTExceptionForPath($path); } return idx($this->exceptions, $path); } } diff --git a/src/lint/linter/ArcanistCSharpLinter.php b/src/lint/linter/ArcanistCSharpLinter.php index 3d466568..89c1f632 100644 --- a/src/lint/linter/ArcanistCSharpLinter.php +++ b/src/lint/linter/ArcanistCSharpLinter.php @@ -1,248 +1,249 @@ 'map>', 'help' => pht('Provide a discovery map.'), ); // TODO: This should probably be replaced with "bin" when this moves // to extend ExternalLinter. $options['binary'] = array( 'type' => 'string', 'help' => pht('Override default binary.'), ); return $options; } public function setLinterConfigurationValue($key, $value) { switch ($key) { case 'discovery': $this->discoveryMap = $value; return; case 'binary': $this->cslintHintPath = $value; return; } parent::setLinterConfigurationValue($key, $value); } protected function getLintCodeFromLinterConfigurationKey($code) { return $code; } public function setCustomSeverityMap(array $map) { foreach ($map as $code => $severity) { if (substr($code, 0, 2) === 'SA' && $severity == 'disabled') { throw new Exception( "In order to keep StyleCop integration with IDEs and other tools ". "consistent with Arcanist results, you aren't permitted to ". "disable StyleCop rules within '.arclint'. ". "Instead configure the severity using the StyleCop settings dialog ". "(usually accessible from within your IDE). StyleCop settings ". "for your project will be used when linting for Arcanist."); } } return parent::setCustomSeverityMap($map); } /** * Determines what executables and lint paths to use. Between platforms * this also changes whether the lint engine is run under .NET or Mono. It * also ensures that all of the required binaries are available for the lint * to run successfully. * * @return void */ private function loadEnvironment() { if ($this->loaded) { return; } // Determine runtime engine (.NET or Mono). if (phutil_is_windows()) { $this->runtimeEngine = ''; } else if (Filesystem::binaryExists('mono')) { $this->runtimeEngine = 'mono '; } else { throw new Exception('Unable to find Mono and you are not on Windows!'); } // Determine cslint path. $cslint = $this->cslintHintPath; if ($cslint !== null && file_exists($cslint)) { $this->cslintEngine = Filesystem::resolvePath($cslint); } else if (Filesystem::binaryExists('cslint.exe')) { $this->cslintEngine = 'cslint.exe'; } else { throw new Exception('Unable to locate cslint.'); } // Determine cslint version. $ver_future = new ExecFuture( '%C -v', $this->runtimeEngine.$this->cslintEngine); list($err, $stdout, $stderr) = $ver_future->resolve(); if ($err !== 0) { throw new Exception( 'You are running an old version of cslint. Please '. 'upgrade to version '.self::SUPPORTED_VERSION.'.'); } $ver = (int)$stdout; if ($ver < self::SUPPORTED_VERSION) { throw new Exception( 'You are running an old version of cslint. Please '. 'upgrade to version '.self::SUPPORTED_VERSION.'.'); } else if ($ver > self::SUPPORTED_VERSION) { throw new Exception( 'Arcanist does not support this version of cslint (it is '. 'newer). You can try upgrading Arcanist with `arc upgrade`.'); } $this->loaded = true; } public function lintPath($path) {} public function willLintPaths(array $paths) { $this->loadEnvironment(); $futures = array(); // Bulk linting up into futures, where the number of files // is based on how long the command is. $current_paths = array(); foreach ($paths as $path) { // If the current paths for the command, plus the next path // is greater than 6000 characters (less than the Windows // command line limit), then finalize this future and add it. $total = 0; foreach ($current_paths as $current_path) { $total += strlen($current_path) + 3; // Quotes and space. } if ($total + strlen($path) > 6000) { // %s won't pass through the JSON correctly // under Windows. This is probably because not only // does the JSON have quotation marks in the content, // but because there'll be a lot of escaping and // double escaping because the JSON also contains // regular expressions. cslint supports passing the // settings JSON through base64-encoded to mitigate // this issue. $futures[] = new ExecFuture( '%C --settings-base64=%s -r=. %Ls', $this->runtimeEngine.$this->cslintEngine, base64_encode(json_encode($this->discoveryMap)), $current_paths); $current_paths = array(); } // Append the path to the current paths array. $current_paths[] = $this->getEngine()->getFilePathOnDisk($path); } // If we still have paths left in current paths, then we need to create // a future for those too. if (count($current_paths) > 0) { $futures[] = new ExecFuture( '%C --settings-base64=%s -r=. %Ls', $this->runtimeEngine.$this->cslintEngine, base64_encode(json_encode($this->discoveryMap)), $current_paths); $current_paths = array(); } $this->futures = $futures; } - public function didRunLinters() { + public function didLintPaths(array $paths) { if ($this->futures) { $futures = id(new FutureIterator($this->futures)) ->limit(8); foreach ($futures as $future) { $this->resolveFuture($future); } + $this->futures = array(); } } protected function resolveFuture(Future $future) { list($stdout) = $future->resolvex(); $all_results = json_decode($stdout); foreach ($all_results as $results) { if ($results === null || $results->Issues === null) { return; } foreach ($results->Issues as $issue) { $message = new ArcanistLintMessage(); $message->setPath($results->FileName); $message->setLine($issue->LineNumber); $message->setCode($issue->Index->Code); $message->setName($issue->Index->Name); $message->setChar($issue->Column); $message->setOriginalText($issue->OriginalText); $message->setReplacementText($issue->ReplacementText); $desc = @vsprintf($issue->Index->Message, $issue->Parameters); if ($desc === false) { $desc = $issue->Index->Message; } $message->setDescription($desc); $severity = ArcanistLintSeverity::SEVERITY_ADVICE; switch ($issue->Index->Severity) { case 0: $severity = ArcanistLintSeverity::SEVERITY_ADVICE; break; case 1: $severity = ArcanistLintSeverity::SEVERITY_AUTOFIX; break; case 2: $severity = ArcanistLintSeverity::SEVERITY_WARNING; break; case 3: $severity = ArcanistLintSeverity::SEVERITY_ERROR; break; case 4: $severity = ArcanistLintSeverity::SEVERITY_DISABLED; break; } $severity_override = $this->getLintMessageSeverity($issue->Index->Code); if ($severity_override !== null) { $severity = $severity_override; } $message->setSeverity($severity); $this->addLintMessage($message); } } } protected function getDefaultMessageSeverity($code) { return null; } } diff --git a/src/lint/linter/ArcanistFutureLinter.php b/src/lint/linter/ArcanistFutureLinter.php index 1adeb43f..d14fe49a 100644 --- a/src/lint/linter/ArcanistFutureLinter.php +++ b/src/lint/linter/ArcanistFutureLinter.php @@ -1,33 +1,56 @@ getFuturesLimit(); $this->futures = id(new FutureIterator(array()))->limit($limit); foreach ($this->buildFutures($paths) as $path => $future) { $this->futures->addFuture($future, $path); } } - final public function lintPath($path) {} - - final public function didRunLinters() { - if ($this->futures) { - foreach ($this->futures as $path => $future) { - $this->willLintPath($path); - $this->resolveFuture($path, $future); - } + final public function lintPath($path) { + return; + } + + final public function didLintPaths(array $paths) { + if (!$this->futures) { + return; + } + + $map = array(); + foreach ($this->futures as $path => $future) { + $this->setActivePath($path); + $this->resolveFuture($path, $future); + $map[$path] = $future; } + $this->futures = array(); + + $this->didResolveLinterFutures($map); + } + + + /** + * Hook for cleaning up resources. + * + * This is invoked after a block of futures resolve, and allows linters to + * discard or clean up any shared resources they no longer need. + * + * @param map Map of paths to resolved futures. + * @return void + */ + protected function didResolveLinterFutures(array $futures) { + return; } } diff --git a/src/lint/linter/ArcanistLinter.php b/src/lint/linter/ArcanistLinter.php index c26bd226..d778e161 100644 --- a/src/lint/linter/ArcanistLinter.php +++ b/src/lint/linter/ArcanistLinter.php @@ -1,505 +1,640 @@ getLinterName(), $this->getLinterConfigurationName(), get_class($this)); } + +/* -( Runtime State )------------------------------------------------------ */ + + + /** + * @task state + */ + final public function getActivePath() { + return $this->activePath; + } + + + /** + * @task state + */ + final public function setActivePath($path) { + $this->stopAllLinters = false; + $this->activePath = $path; + return $this; + } + + + /** + * @task state + */ + final public function setEngine(ArcanistLintEngine $engine) { + $this->engine = $engine; + return $this; + } + + + /** + * @task state + */ + final protected function getEngine() { + return $this->engine; + } + + + /** + * Set the internal ID for this linter. + * + * This ID is assigned automatically by the @{class:ArcanistLintEngine}. + * + * @param string Unique linter ID. + * @return this + * @task state + */ + final public function setLinterID($id) { + $this->id = $id; + return $this; + } + + + /** + * Get the internal ID for this linter. + * + * Retrieves an internal linter ID managed by the @{class:ArcanistLintEngine}. + * This ID is a unique scalar which distinguishes linters in a list. + * + * @return string Unique linter ID. + * @task state + */ + final public function getLinterID() { + return $this->id; + } + + +/* -( Executing Linters )-------------------------------------------------- */ + + + /** + * Hook called before a list of paths are linted. + * + * Parallelizable linters can start multiple requests in parallel here, + * to improve performance. They can implement @{method:didLintPaths} to + * collect results. + * + * Linters which are not parallelizable should normally ignore this callback + * and implement @{method:lintPath} instead. + * + * @param list A list of paths to be linted + * @return void + * @task exec + */ + public function willLintPaths(array $paths) { + return; + } + + + /** + * Hook called for each path to be linted. + * + * Linters which are not parallelizable can do work here. + * + * Linters which are parallelizable may want to ignore this callback and + * implement @{method:willLintPaths} and @{method:didLintPaths} instead. + * + * @param string Path to lint. + * @return void + * @task exec + */ + public function lintPath($path) { + return; + } + + + /** + * Hook called after a list of paths are linted. + * + * Parallelizable linters can collect results here. + * + * Linters which are not paralleizable should normally ignore this callback + * and implement @{method:lintPath} instead. + * + * @param list A list of paths which were linted. + * @return void + * @task exec + */ + public function didLintPaths(array $paths) { + return; + } + + + /** + * Obsolete hook which was invoked before a path was linted. + * + * WARNING: This is an obsolete hook which is not called. If you maintain + * a linter which relies on it, update to use @{method:lintPath} instead. + * + * @task exec + */ + final public function willLintPath($path) { + // TODO: Remove this method after some time. In the meantime, the "final" + // will fatal subclasses which implement this hook and point at the API + // change so maintainers get fewer surprises. + throw new PhutilMethodNotImplementedException(); + } + + + /** + * Obsolete hook which was invoked after linters ran. + * + * WARNING: This is an obsolete hook which is not called. If you maintain + * a linter which relies on it, update to use @{method:didLintPaths} instead. + * + * @return void + * @task exec + */ + final public function didRunLinters() { + // TODO: Remove this method after some time. In the meantime, the "final" + // will fatal subclasses which implement this hook and point at the API + // change so maintainers get fewer surprises. + throw new PhutilMethodNotImplementedException(); + } + + public function getLinterPriority() { return 1.0; } /** * TODO: This should be `final`. */ public function setCustomSeverityMap(array $map) { $this->customSeverityMap = $map; return $this; } final public function setCustomSeverityRules(array $rules) { $this->customSeverityRules = $rules; return $this; } - final public function getActivePath() { - return $this->activePath; - } - final public function getOtherLocation($offset, $path = null) { if ($path === null) { $path = $this->getActivePath(); } list($line, $char) = $this->getEngine()->getLineAndCharFromOffset( $path, $offset); return array( 'path' => $path, 'line' => $line + 1, 'char' => $char, ); } final public function stopAllLinters() { $this->stopAllLinters = true; return $this; } final public function didStopAllLinters() { return $this->stopAllLinters; } final public function addPath($path) { $this->paths[$path] = $path; return $this; } final public function setPaths(array $paths) { $this->paths = $paths; return $this; } /** * Filter out paths which this linter doesn't act on (for example, because * they are binaries and the linter doesn't apply to binaries). */ final private function filterPaths($paths) { $engine = $this->getEngine(); $keep = array(); foreach ($paths as $path) { if (!$this->shouldLintDeletedFiles() && !$engine->pathExists($path)) { continue; } if (!$this->shouldLintDirectories() && $engine->isDirectory($path)) { continue; } if (!$this->shouldLintBinaryFiles() && $engine->isBinaryFile($path)) { continue; } if (!$this->shouldLintSymbolicLinks() && $engine->isSymbolicLink($path)) { continue; } $keep[] = $path; } return $keep; } final public function getPaths() { return $this->filterPaths(array_values($this->paths)); } final public function addData($path, $data) { $this->data[$path] = $data; return $this; } final protected function getData($path) { if (!array_key_exists($path, $this->data)) { $this->data[$path] = $this->getEngine()->loadData($path); } return $this->data[$path]; } - final public function setEngine(ArcanistLintEngine $engine) { - $this->engine = $engine; - return $this; - } - - final protected function getEngine() { - return $this->engine; - } - public function getCacheVersion() { return 0; } + final public function getLintMessageFullCode($short_code) { return $this->getLinterName().$short_code; } final public function getLintMessageSeverity($code) { $map = $this->customSeverityMap; if (isset($map[$code])) { return $map[$code]; } foreach ($this->customSeverityRules as $rule => $severity) { if (preg_match($rule, $code)) { return $severity; } } $map = $this->getLintSeverityMap(); if (isset($map[$code])) { return $map[$code]; } return $this->getDefaultMessageSeverity($code); } protected function getDefaultMessageSeverity($code) { return ArcanistLintSeverity::SEVERITY_ERROR; } final public function isMessageEnabled($code) { return ($this->getLintMessageSeverity($code) !== ArcanistLintSeverity::SEVERITY_DISABLED); } final public function getLintMessageName($code) { $map = $this->getLintNameMap(); if (isset($map[$code])) { return $map[$code]; } return 'Unknown lint message!'; } final protected function addLintMessage(ArcanistLintMessage $message) { if (!$this->getEngine()->getCommitHookMode()) { $root = $this->getEngine()->getWorkingCopy()->getProjectRoot(); $path = Filesystem::resolvePath($message->getPath(), $root); $message->setPath(Filesystem::readablePath($path, $root)); } $this->messages[] = $message; return $message; } final public function getLintMessages() { return $this->messages; } final protected function raiseLintAtLine( $line, $char, $code, $desc, $original = null, $replacement = null) { $message = id(new ArcanistLintMessage()) ->setPath($this->getActivePath()) ->setLine($line) ->setChar($char) ->setCode($this->getLintMessageFullCode($code)) ->setSeverity($this->getLintMessageSeverity($code)) ->setName($this->getLintMessageName($code)) ->setDescription($desc) ->setOriginalText($original) ->setReplacementText($replacement); return $this->addLintMessage($message); } final protected function raiseLintAtPath($code, $desc) { return $this->raiseLintAtLine(null, null, $code, $desc, null, null); } final protected function raiseLintAtOffset( $offset, $code, $desc, $original = null, $replacement = null) { $path = $this->getActivePath(); $engine = $this->getEngine(); if ($offset === null) { $line = null; $char = null; } else { list($line, $char) = $engine->getLineAndCharFromOffset($path, $offset); } return $this->raiseLintAtLine( $line + 1, $char + 1, $code, $desc, $original, $replacement); } - public function willLintPath($path) { - $this->stopAllLinters = false; - $this->activePath = $path; - } - public function canRun() { return true; } - public function willLintPaths(array $paths) { - return; - } - - abstract public function lintPath($path); abstract public function getLinterName(); public function getVersion() { return null; } - public function didRunLinters() { - // This is a hook. - } - final protected function isCodeEnabled($code) { $severity = $this->getLintMessageSeverity($code); return $this->getEngine()->isSeverityEnabled($severity); } public function getLintSeverityMap() { return array(); } public function getLintNameMap() { return array(); } public function getCacheGranularity() { return self::GRANULARITY_FILE; } /** * If this linter is selectable via `.arclint` configuration files, return * a short, human-readable name to identify it. For example, `"jshint"` or * `"pep8"`. * * If you do not implement this method, the linter will not be selectable * through `.arclint` files. */ public function getLinterConfigurationName() { return null; } public function getLinterConfigurationOptions() { if (!$this->canCustomizeLintSeverities()) { return array(); } return array( 'severity' => array( 'type' => 'optional map', 'help' => pht( 'Provide a map from lint codes to adjusted severity levels: error, '. 'warning, advice, autofix or disabled.'), ), 'severity.rules' => array( 'type' => 'optional map', 'help' => pht( 'Provide a map of regular expressions to severity levels. All '. 'matching codes have their severity adjusted.'), ), ); } public function setLinterConfigurationValue($key, $value) { $sev_map = array( 'error' => ArcanistLintSeverity::SEVERITY_ERROR, 'warning' => ArcanistLintSeverity::SEVERITY_WARNING, 'autofix' => ArcanistLintSeverity::SEVERITY_AUTOFIX, 'advice' => ArcanistLintSeverity::SEVERITY_ADVICE, 'disabled' => ArcanistLintSeverity::SEVERITY_DISABLED, ); switch ($key) { case 'severity': if (!$this->canCustomizeLintSeverities()) { break; } $custom = array(); foreach ($value as $code => $severity) { if (empty($sev_map[$severity])) { $valid = implode(', ', array_keys($sev_map)); throw new Exception( pht( 'Unknown lint severity "%s". Valid severities are: %s.', $severity, $valid)); } $code = $this->getLintCodeFromLinterConfigurationKey($code); $custom[$code] = $severity; } $this->setCustomSeverityMap($custom); return; case 'severity.rules': if (!$this->canCustomizeLintSeverities()) { break; } foreach ($value as $rule => $severity) { if (@preg_match($rule, '') === false) { throw new Exception( pht( 'Severity rule "%s" is not a valid regular expression.', $rule)); } if (empty($sev_map[$severity])) { $valid = implode(', ', array_keys($sev_map)); throw new Exception( pht( 'Unknown lint severity "%s". Valid severities are: %s.', $severity, $valid)); } } $this->setCustomSeverityRules($value); return; } throw new Exception("Incomplete implementation: {$key}!"); } protected function canCustomizeLintSeverities() { return true; } protected function shouldLintBinaryFiles() { return false; } protected function shouldLintDeletedFiles() { return false; } protected function shouldLintDirectories() { return false; } protected function shouldLintSymbolicLinks() { return false; } /** * Map a configuration lint code to an `arc` lint code. Primarily, this is * intended for validation, but can also be used to normalize case or * otherwise be more permissive in accepted inputs. * * If the code is not recognized, you should throw an exception. * * @param string Code specified in configuration. * @return string Normalized code to use in severity map. */ protected function getLintCodeFromLinterConfigurationKey($code) { return $code; } /** * Retrieve an old lint configuration value from `.arcconfig` or a similar * source. * * Modern linters should use @{method:getConfig} to read configuration from * `.arclint`. * * @param string Configuration key to retrieve. * @param wild Default value to return if key is not present in config. * @return wild Configured value, or default if no configuration exists. */ final protected function getDeprecatedConfiguration($key, $default = null) { // If we're being called in a context without an engine (probably from // `arc linters`), just return the default value. if (!$this->engine) { return $default; } $config = $this->getEngine()->getConfigurationManager(); // Construct a sentinel object so we can tell if we're reading config // or not. $sentinel = (object)array(); $result = $config->getConfigFromAnySource($key, $sentinel); // If we read config, warn the user that this mechanism is deprecated and // discouraged. if ($result !== $sentinel) { $console = PhutilConsole::getConsole(); $console->writeErr( "**%s**: %s\n", pht('Deprecation Warning'), pht( 'Configuration option "%s" is deprecated. Generally, linters should '. 'now be configured using an `.arclint` file. See "Arcanist User '. 'Guide: Lint" in the documentation for more information.', $key)); return $result; } return $default; } } diff --git a/src/lint/linter/__tests__/ArcanistLinterTestCase.php b/src/lint/linter/__tests__/ArcanistLinterTestCase.php index 4785d9e7..61a50b14 100644 --- a/src/lint/linter/__tests__/ArcanistLinterTestCase.php +++ b/src/lint/linter/__tests__/ArcanistLinterTestCase.php @@ -1,266 +1,266 @@ getLinter(); } $files = id(new FileFinder($root)) ->withType('f') ->withSuffix('lint-test') ->find(); $test_count = 0; foreach ($files as $file) { $this->lintFile($root.$file, $linter); $test_count++; } $this->assertTrue( ($test_count > 0), pht('Expected to find some .lint-test tests in directory %s!', $root)); } private function lintFile($file, ArcanistLinter $linter) { $linter = clone $linter; $contents = Filesystem::readFile($file); $contents = preg_split('/^~{4,}\n/m', $contents); if (count($contents) < 2) { throw new Exception( pht( "Expected '%s' separating test case and results.", '~~~~~~~~~~')); } list ($data, $expect, $xform, $config) = array_merge( $contents, array(null, null)); $basename = basename($file); if ($config) { $config = phutil_json_decode($config); } else { $config = array(); } PhutilTypeSpec::checkMap( $config, array( 'hook' => 'optional bool', 'config' => 'optional map', 'path' => 'optional string', 'mode' => 'optional string', 'stopped' => 'optional bool', )); $exception = null; $after_lint = null; $messages = null; $exception_message = false; $caught_exception = false; try { $tmp = new TempFile($basename); Filesystem::writeFile($tmp, $data); $full_path = (string)$tmp; $mode = idx($config, 'mode'); if ($mode) { Filesystem::changePermissions($tmp, octdec($mode)); } $dir = dirname($full_path); $path = basename($full_path); $working_copy = ArcanistWorkingCopyIdentity::newFromRootAndConfigFile( $dir, null, 'Unit Test'); $configuration_manager = new ArcanistConfigurationManager(); $configuration_manager->setWorkingCopyIdentity($working_copy); $engine = new ArcanistUnitTestableLintEngine(); $engine->setWorkingCopy($working_copy); $engine->setConfigurationManager($configuration_manager); - $engine->setPaths(array($path)); $engine->setCommitHookMode(idx($config, 'hook', false)); $path_name = idx($config, 'path', $path); + $engine->setPaths(array($path_name)); + $linter->addPath($path_name); $linter->addData($path_name, $data); foreach (idx($config, 'config', array()) as $key => $value) { $linter->setLinterConfigurationValue($key, $value); } $engine->addLinter($linter); $engine->addFileData($path_name, $data); $results = $engine->run(); - $this->assertEqual( 1, count($results), pht('Expect one result returned by linter.')); $assert_stopped = idx($config, 'stopped'); if ($assert_stopped !== null) { $this->assertEqual( $assert_stopped, $linter->didStopAllLinters(), $assert_stopped ? pht('Expect linter to be stopped.') : pht('Expect linter to not be stopped.')); } $result = reset($results); $patcher = ArcanistLintPatcher::newFromArcanistLintResult($result); $after_lint = $patcher->getModifiedFileContent(); } catch (ArcanistPhutilTestTerminatedException $ex) { throw $ex; } catch (Exception $exception) { $caught_exception = true; if ($exception instanceof PhutilAggregateException) { $caught_exception = false; foreach ($exception->getExceptions() as $ex) { if ($ex instanceof ArcanistUsageException || $ex instanceof ArcanistMissingLinterException) { $this->assertSkipped($ex->getMessage()); } else { $caught_exception = true; } } } else if ($exception instanceof ArcanistUsageException || $exception instanceof ArcanistMissingLinterException) { $this->assertSkipped($exception->getMessage()); } $exception_message = $exception->getMessage()."\n\n". $exception->getTraceAsString(); } $this->assertEqual(false, $caught_exception, $exception_message); $this->compareLint($basename, $expect, $result); $this->compareTransform($xform, $after_lint); } private function compareLint($file, $expect, ArcanistLintResult $result) { $seen = array(); $raised = array(); $message_map = array(); foreach ($result->getMessages() as $message) { $sev = $message->getSeverity(); $line = $message->getLine(); $char = $message->getChar(); $code = $message->getCode(); $name = $message->getName(); $message_key = $sev.':'.$line.':'.$char; $message_map[$message_key] = $message; $seen[] = $message_key; $raised[] = sprintf( ' %s: %s %s', pht('%s at line %d, char %d', $sev, $line, $char), $code, $name); } $expect = trim($expect); if ($expect) { $expect = explode("\n", $expect); } else { $expect = array(); } foreach ($expect as $key => $expected) { $expect[$key] = head(explode(' ', $expected)); } $expect = array_fill_keys($expect, true); $seen = array_fill_keys($seen, true); if (!$raised) { $raised = array(pht('No messages.')); } $raised = sprintf( "%s:\n%s", pht('Actually raised'), implode("\n", $raised)); foreach (array_diff_key($expect, $seen) as $missing => $ignored) { $missing = explode(':', $missing); $sev = array_shift($missing); $pos = $missing; $this->assertFailure( pht( "In '%s', expected lint to raise %s on line %d at char %d, ". "but no %s was raised. %s", $file, $sev, idx($pos, 0), idx($pos, 1), $sev, $raised)); } foreach (array_diff_key($seen, $expect) as $surprising => $ignored) { $message = $message_map[$surprising]; $message_info = $message->getDescription(); list($sev, $line, $char) = explode(':', $surprising); $this->assertFailure( sprintf( "%s:\n\n%s\n\n%s", pht( "In '%s', lint raised %s on line %d at char %d, ". "but nothing was expected", $file, $sev, $line, $char), $message_info, $raised)); } } private function compareTransform($expected, $actual) { if (!strlen($expected)) { return; } $this->assertEqual( $expected, $actual, pht('File as patched by lint did not match the expected patched file.')); } }