diff --git a/src/lint/ArcanistLintMessage.php b/src/lint/ArcanistLintMessage.php index d4de43b8..29f9ab92 100644 --- a/src/lint/ArcanistLintMessage.php +++ b/src/lint/ArcanistLintMessage.php @@ -1,259 +1,276 @@ setPath($dict['path']); $message->setLine($dict['line']); $message->setChar($dict['char']); $message->setCode($dict['code']); $message->setSeverity($dict['severity']); $message->setName($dict['name']); $message->setDescription($dict['description']); if (isset($dict['original'])) { $message->setOriginalText($dict['original']); } if (isset($dict['replacement'])) { $message->setReplacementText($dict['replacement']); } $message->setGranularity(idx($dict, 'granularity')); $message->setOtherLocations(idx($dict, 'locations', array())); if (isset($dict['bypassChangedLineFiltering'])) { $message->bypassChangedLineFiltering($dict['bypassChangedLineFiltering']); } return $message; } public function toDictionary() { return array( 'path' => $this->getPath(), 'line' => $this->getLine(), 'char' => $this->getChar(), 'code' => $this->getCode(), 'severity' => $this->getSeverity(), 'name' => $this->getName(), 'description' => $this->getDescription(), 'original' => $this->getOriginalText(), 'replacement' => $this->getReplacementText(), 'granularity' => $this->getGranularity(), 'locations' => $this->getOtherLocations(), 'bypassChangedLineFiltering' => $this->shouldBypassChangedLineFiltering(), ); } public function setPath($path) { $this->path = $path; return $this; } public function getPath() { return $this->path; } public function setLine($line) { $this->line = $this->validateInteger($line, 'setLine'); return $this; } public function getLine() { return $this->line; } public function setChar($char) { $this->char = $this->validateInteger($char, 'setChar'); return $this; } public function getChar() { return $this->char; } public function setCode($code) { + $code = (string)$code; + + $maximum_bytes = 128; + $actual_bytes = strlen($code); + + if ($actual_bytes > $maximum_bytes) { + throw new Exception( + pht( + 'Parameter ("%s") passed to "%s" when constructing a lint message '. + 'must be a scalar with a maximum string length of %s bytes, but is '. + '%s bytes in length.', + $code, + 'setCode()', + new PhutilNumber($maximum_bytes), + new PhutilNumber($actual_bytes))); + } + $this->code = $code; return $this; } public function getCode() { return $this->code; } public function setSeverity($severity) { $this->severity = $severity; return $this; } public function getSeverity() { return $this->severity; } public function setName($name) { $this->name = $name; return $this; } public function getName() { return $this->name; } public function setDescription($description) { $this->description = $description; return $this; } public function getDescription() { return $this->description; } public function setOriginalText($original) { $this->originalText = $original; return $this; } public function getOriginalText() { return $this->originalText; } public function setReplacementText($replacement) { $this->replacementText = $replacement; return $this; } public function getReplacementText() { return $this->replacementText; } /** * @param dict Keys 'path', 'line', 'char', 'original'. */ public function setOtherLocations(array $locations) { assert_instances_of($locations, 'array'); $this->otherLocations = $locations; return $this; } public function getOtherLocations() { return $this->otherLocations; } public function isError() { return $this->getSeverity() == ArcanistLintSeverity::SEVERITY_ERROR; } public function isWarning() { return $this->getSeverity() == ArcanistLintSeverity::SEVERITY_WARNING; } public function isAutofix() { return $this->getSeverity() == ArcanistLintSeverity::SEVERITY_AUTOFIX; } public function hasFileContext() { return ($this->getLine() !== null); } public function setObsolete($obsolete) { $this->obsolete = $obsolete; return $this; } public function getObsolete() { return $this->obsolete; } public function isPatchable() { return ($this->getReplacementText() !== null) && ($this->getReplacementText() !== $this->getOriginalText()); } public function didApplyPatch() { if ($this->appliedToDisk) { return $this; } $this->appliedToDisk = true; foreach ($this->dependentMessages as $message) { $message->didApplyPatch(); } return $this; } public function isPatchApplied() { return $this->appliedToDisk; } public function setGranularity($granularity) { $this->granularity = $granularity; return $this; } public function getGranularity() { return $this->granularity; } public function setDependentMessages(array $messages) { assert_instances_of($messages, __CLASS__); $this->dependentMessages = $messages; return $this; } public function setBypassChangedLineFiltering($bypass_changed_lines) { $this->bypassChangedLineFiltering = $bypass_changed_lines; return $this; } public function shouldBypassChangedLineFiltering() { return $this->bypassChangedLineFiltering; } /** * Validate an integer-like value, returning a strict integer. * * Further on, the pipeline is strict about types. We want to be a little * less strict in linters themselves, since they often parse command line * output or XML and will end up with string representations of numbers. * * @param mixed Integer or digit string. * @return int Integer. */ private function validateInteger($value, $caller) { if ($value === null) { // This just means that we don't have any information. return null; } // Strings like "234" are fine, coerce them to integers. if (is_string($value) && preg_match('/^\d+\z/', $value)) { $value = (int)$value; } if (!is_int($value)) { throw new Exception( pht( 'Parameter passed to "%s" must be an integer.', $caller.'()')); } return $value; } } diff --git a/src/lint/engine/ArcanistLintEngine.php b/src/lint/engine/ArcanistLintEngine.php index 4a092fdb..0bef3e0a 100644 --- a/src/lint/engine/ArcanistLintEngine.php +++ b/src/lint/engine/ArcanistLintEngine.php @@ -1,602 +1,617 @@ configurationManager = $configuration_manager; return $this; } final public function getConfigurationManager() { return $this->configurationManager; } final public function setWorkingCopy( ArcanistWorkingCopyIdentity $working_copy) { $this->workingCopy = $working_copy; return $this; } final public function getWorkingCopy() { return $this->workingCopy; } final public function setPaths($paths) { $this->paths = $paths; return $this; } public function getPaths() { return $this->paths; } final public function setPathChangedLines($path, $changed) { if ($changed === null) { $this->changedLines[$path] = null; } else { $this->changedLines[$path] = array_fill_keys($changed, true); } return $this; } final public function getPathChangedLines($path) { return idx($this->changedLines, $path); } final public function setFileData($data) { $this->fileData = $data + $this->fileData; return $this; } final public function setEnableAsyncLint($enable_async_lint) { $this->enableAsyncLint = $enable_async_lint; return $this; } final public function getEnableAsyncLint() { return $this->enableAsyncLint; } final public function loadData($path) { if (!isset($this->fileData[$path])) { $disk_path = $this->getFilePathOnDisk($path); $this->fileData[$path] = Filesystem::readFile($disk_path); } return $this->fileData[$path]; } public function pathExists($path) { $disk_path = $this->getFilePathOnDisk($path); return Filesystem::pathExists($disk_path); } final public function isDirectory($path) { $disk_path = $this->getFilePathOnDisk($path); return is_dir($disk_path); } final public function isBinaryFile($path) { try { $data = $this->loadData($path); } catch (Exception $ex) { return false; } return ArcanistDiffUtils::isHeuristicBinaryFile($data); } final public function isSymbolicLink($path) { return is_link($this->getFilePathOnDisk($path)); } final public function getFilePathOnDisk($path) { return Filesystem::resolvePath( $path, $this->getWorkingCopy()->getProjectRoot()); } final public function setMinimumSeverity($severity) { $this->minimumSeverity = $severity; return $this; } final public function run() { $linters = $this->buildLinters(); if (!$linters) { throw new ArcanistNoEffectException(pht('No linters to run.')); } foreach ($linters as $key => $linter) { $linter->setLinterID($key); } $linters = msort($linters, 'getLinterPriority'); foreach ($linters as $linter) { $linter->setEngine($this); } $have_paths = false; foreach ($linters as $linter) { if ($linter->getPaths()) { $have_paths = true; break; } } if (!$have_paths) { throw new ArcanistNoEffectException(pht('No paths are lintable.')); } $versions = array($this->getCacheVersion()); foreach ($linters as $linter) { $version = get_class($linter).':'.$linter->getCacheVersion(); $symbols = id(new PhutilSymbolLoader()) ->setType('class') ->setName(get_class($linter)) ->selectSymbolsWithoutLoading(); $symbol = idx($symbols, 'class$'.get_class($linter)); if ($symbol) { $version .= ':'.md5_file( phutil_get_library_root($symbol['library']).'/'.$symbol['where']); } $versions[] = $version; } $this->cacheVersion = crc32(implode("\n", $versions)); $runnable = $this->getRunnableLinters($linters); $this->stopped = array(); $exceptions = $this->executeLinters($runnable); foreach ($runnable as $linter) { foreach ($linter->getLintMessages() as $message) { + $this->validateLintMessage($linter, $message); + if (!$this->isSeverityEnabled($message->getSeverity())) { continue; } if (!$this->isRelevantMessage($message)) { continue; } $message->setGranularity($linter->getCacheGranularity()); $result = $this->getResultForPath($message->getPath()); $result->addMessage($message); } } if ($this->cachedResults) { foreach ($this->cachedResults as $path => $messages) { $messages = idx($messages, $this->cacheVersion, array()); $repository_version = idx($messages, 'repository_version'); unset($messages['stopped']); unset($messages['repository_version']); foreach ($messages as $message) { $use_cache = $this->shouldUseCache( idx($message, 'granularity'), $repository_version); if ($use_cache) { $this->getResultForPath($path)->addMessage( ArcanistLintMessage::newFromDictionary($message)); } } } } foreach ($this->results as $path => $result) { $disk_path = $this->getFilePathOnDisk($path); $result->setFilePathOnDisk($disk_path); if (isset($this->fileData[$path])) { $result->setData($this->fileData[$path]); } else if ($disk_path && Filesystem::pathExists($disk_path)) { // TODO: this may cause us to, e.g., load a large binary when we only // raised an error about its filename. We could refine this by looking // through the lint messages and doing this load only if any of them // have original/replacement text or something like that. try { $this->fileData[$path] = Filesystem::readFile($disk_path); $result->setData($this->fileData[$path]); } catch (FilesystemException $ex) { // Ignore this, it's noncritical that we access this data and it // might be unreadable or a directory or whatever else for plenty // of legitimate reasons. } } } if ($exceptions) { throw new PhutilAggregateException( pht('Some linters failed:'), $exceptions); } return $this->results; } final public function isSeverityEnabled($severity) { $minimum = $this->minimumSeverity; return ArcanistLintSeverity::isAtLeastAsSevere($severity, $minimum); } final private function shouldUseCache( $cache_granularity, $repository_version) { switch ($cache_granularity) { case ArcanistLinter::GRANULARITY_FILE: return true; case ArcanistLinter::GRANULARITY_DIRECTORY: case ArcanistLinter::GRANULARITY_REPOSITORY: return ($this->repositoryVersion == $repository_version); default: return false; } } /** * @param dict>> * @return this */ final public function setCachedResults(array $results) { $this->cachedResults = $results; return $this; } final public function getResults() { return $this->results; } final public function getStoppedPaths() { return $this->stopped; } abstract public function buildLinters(); final public function setRepositoryVersion($version) { $this->repositoryVersion = $version; return $this; } final private function isRelevantMessage(ArcanistLintMessage $message) { // When a user runs "arc lint", we default to raising only warnings on // lines they have changed (errors are still raised anywhere in the // file). The list of $changed lines may be null, to indicate that the // path is a directory or a binary file so we should not exclude // warnings. if (!$this->changedLines || $message->isError() || $message->shouldBypassChangedLineFiltering()) { return true; } $locations = $message->getOtherLocations(); $locations[] = $message->toDictionary(); foreach ($locations as $location) { $path = idx($location, 'path', $message->getPath()); if (!array_key_exists($path, $this->changedLines)) { continue; } $changed = $this->getPathChangedLines($path); if ($changed === null || !$location['line']) { return true; } $last_line = $location['line']; if (isset($location['original'])) { $last_line += substr_count($location['original'], "\n"); } for ($l = $location['line']; $l <= $last_line; $l++) { if (!empty($changed[$l])) { return true; } } } return false; } final protected function getResultForPath($path) { if (empty($this->results[$path])) { $result = new ArcanistLintResult(); $result->setPath($path); $result->setCacheVersion($this->cacheVersion); $this->results[$path] = $result; } return $this->results[$path]; } final public function getLineAndCharFromOffset($path, $offset) { if (!isset($this->charToLine[$path])) { $char_to_line = array(); $line_to_first_char = array(); $lines = explode("\n", $this->loadData($path)); $line_number = 0; $line_start = 0; foreach ($lines as $line) { $len = strlen($line) + 1; // Account for "\n". $line_to_first_char[] = $line_start; $line_start += $len; for ($ii = 0; $ii < $len; $ii++) { $char_to_line[] = $line_number; } $line_number++; } $this->charToLine[$path] = $char_to_line; $this->lineToFirstChar[$path] = $line_to_first_char; } $line = $this->charToLine[$path][$offset]; $char = $offset - $this->lineToFirstChar[$path][$line]; return array($line, $char); } protected function getCacheVersion() { return 1; } /** * Get a named linter resource shared by another linter. * * This mechanism allows linters to share arbitrary resources, like the * results of computation. If several linters need to perform the same * expensive computation step, they can use a named resource to synchronize * construction of the result so it doesn't need to be built multiple * times. * * @param string Resource identifier. * @param wild Optionally, default value to return if resource does not * exist. * @return wild Resource, or default value if not present. */ public function getLinterResource($key, $default = null) { return idx($this->linterResources, $key, $default); } /** * Set a linter resource that other linters can access. * * See @{method:getLinterResource} for a description of this mechanism. * * @param string Resource identifier. * @param wild Resource. * @return this */ public function setLinterResource($key, $value) { $this->linterResources[$key] = $value; return $this; } private function getRunnableLinters(array $linters) { assert_instances_of($linters, 'ArcanistLinter'); // TODO: The canRun() mechanism is only used by one linter, and just // silently disables the linter. Almost every other linter handles this // by throwing `ArcanistMissingLinterException`. Both mechanisms are not // ideal; linters which can not run should emit a message, get marked as // "skipped", and allow execution to continue. See T7045. $runnable = array(); foreach ($linters as $key => $linter) { if ($linter->canRun()) { $runnable[$key] = $linter; } } return $runnable; } private function executeLinters(array $runnable) { assert_instances_of($runnable, 'ArcanistLinter'); $all_paths = $this->getPaths(); $path_chunks = array_chunk($all_paths, 32, $preserve_keys = true); $exception_lists = array(); foreach ($path_chunks as $chunk) { $exception_lists[] = $this->executeLintersOnChunk($runnable, $chunk); } return array_mergev($exception_lists); } private function executeLintersOnChunk(array $runnable, array $path_list) { assert_instances_of($runnable, 'ArcanistLinter'); $path_map = array_fuse($path_list); $exceptions = array(); $did_lint = array(); foreach ($runnable as $linter) { $linter_id = $linter->getLinterID(); $paths = $linter->getPaths(); foreach ($paths as $key => $path) { // If we aren't running this path in the current chunk of paths, // skip it completely. if (empty($path_map[$path])) { unset($paths[$key]); continue; } // Make sure each path has a result generated, even if it is empty // (i.e., the file has no lint messages). $result = $this->getResultForPath($path); // If a linter has stopped all other linters for this path, don't // actually run the linter. if (isset($this->stopped[$path])) { unset($paths[$key]); continue; } // If we have a cached result for this path, don't actually run the // linter. if (isset($this->cachedResults[$path][$this->cacheVersion])) { $cached_result = $this->cachedResults[$path][$this->cacheVersion]; $use_cache = $this->shouldUseCache( $linter->getCacheGranularity(), idx($cached_result, 'repository_version')); if ($use_cache) { unset($paths[$key]); if (idx($cached_result, 'stopped') == $linter_id) { $this->stopped[$path] = $linter_id; } } } } $paths = array_values($paths); if (!$paths) { continue; } try { $this->executeLinterOnPaths($linter, $paths); $did_lint[] = array($linter, $paths); } catch (Exception $ex) { $exceptions[] = $ex; } } foreach ($did_lint as $info) { list($linter, $paths) = $info; try { $this->executeDidLintOnPaths($linter, $paths); } catch (Exception $ex) { $exceptions[] = $ex; } } return $exceptions; } private function beginLintServiceCall(ArcanistLinter $linter, array $paths) { $profiler = PhutilServiceProfiler::getInstance(); return $profiler->beginServiceCall( array( 'type' => 'lint', 'linter' => $linter->getInfoName(), 'paths' => $paths, )); } private function endLintServiceCall($call_id) { $profiler = PhutilServiceProfiler::getInstance(); $profiler->endServiceCall($call_id, array()); } private function executeLinterOnPaths(ArcanistLinter $linter, array $paths) { $call_id = $this->beginLintServiceCall($linter, $paths); try { $linter->willLintPaths($paths); foreach ($paths as $path) { $linter->setActivePath($path); $linter->lintPath($path); if ($linter->didStopAllLinters()) { $this->stopped[$path] = $linter->getLinterID(); } } } catch (Exception $ex) { $this->endLintServiceCall($call_id); throw $ex; } $this->endLintServiceCall($call_id); } private function executeDidLintOnPaths(ArcanistLinter $linter, array $paths) { $call_id = $this->beginLintServiceCall($linter, $paths); try { $linter->didLintPaths($paths); } catch (Exception $ex) { $this->endLintServiceCall($call_id); throw $ex; } $this->endLintServiceCall($call_id); } + private function validateLintMessage( + ArcanistLinter $linter, + ArcanistLintMessage $message) { + + $name = $message->getName(); + if (!strlen($name)) { + throw new Exception( + pht( + 'Linter "%s" generated a lint message that is invalid because it '. + 'does not have a name. Lint messages must have a name.', + get_class($linter))); + } + } } diff --git a/src/lint/linter/ArcanistPhpcsLinter.php b/src/lint/linter/ArcanistPhpcsLinter.php index da3d118c..9556ea0a 100644 --- a/src/lint/linter/ArcanistPhpcsLinter.php +++ b/src/lint/linter/ArcanistPhpcsLinter.php @@ -1,148 +1,150 @@ array( 'type' => 'optional string', 'help' => pht('The name or path of the coding standard to use.'), ), ); return $options + parent::getLinterConfigurationOptions(); } public function setLinterConfigurationValue($key, $value) { switch ($key) { case 'phpcs.standard': $this->standard = $value; return; default: return parent::setLinterConfigurationValue($key, $value); } } protected function getMandatoryFlags() { $options = array('--report=xml'); if ($this->standard) { $options[] = '--standard='.$this->standard; } return $options; } public function getDefaultBinary() { return 'phpcs'; } public function getVersion() { list($stdout) = execx('%C --version', $this->getExecutableCommand()); $matches = array(); $regex = '/^PHP_CodeSniffer version (?P\d+\.\d+\.\d+)\b/'; if (preg_match($regex, $stdout, $matches)) { return $matches['version']; } else { return false; } } protected function parseLinterOutput($path, $err, $stdout, $stderr) { // NOTE: Some version of PHPCS after 1.4.6 stopped printing a valid, empty // XML document to stdout in the case of no errors. If PHPCS exits with // error 0, just ignore output. if (!$err) { return array(); } $report_dom = new DOMDocument(); $ok = @$report_dom->loadXML($stdout); if (!$ok) { return false; } $files = $report_dom->getElementsByTagName('file'); $messages = array(); foreach ($files as $file) { foreach ($file->childNodes as $child) { if (!($child instanceof DOMElement)) { continue; } if ($child->tagName == 'error') { $prefix = 'E'; } else { $prefix = 'W'; } - $code = 'PHPCS.'.$prefix.'.'.$child->getAttribute('source'); - - $message = new ArcanistLintMessage(); - $message->setPath($path); - $message->setLine($child->getAttribute('line')); - $message->setChar($child->getAttribute('column')); - $message->setCode($code); - $message->setDescription($child->nodeValue); - $message->setSeverity($this->getLintMessageSeverity($code)); + $source = $child->getAttribute('source'); + $code = 'PHPCS.'.$prefix.'.'.$source; + + $message = id(new ArcanistLintMessage()) + ->setPath($path) + ->setName($source) + ->setLine($child->getAttribute('line')) + ->setChar($child->getAttribute('column')) + ->setCode($code) + ->setDescription($child->nodeValue) + ->setSeverity($this->getLintMessageSeverity($code)); $messages[] = $message; } } return $messages; } protected function getDefaultMessageSeverity($code) { if (preg_match('/^PHPCS\\.W\\./', $code)) { return ArcanistLintSeverity::SEVERITY_WARNING; } else { return ArcanistLintSeverity::SEVERITY_ERROR; } } protected function getLintCodeFromLinterConfigurationKey($code) { if (!preg_match('/^PHPCS\\.(E|W)\\./', $code)) { throw new Exception( pht( "Invalid severity code '%s', should begin with '%s.'.", $code, 'PHPCS')); } return $code; } }