diff --git a/src/unit/engine/ArcanistConfigurationDrivenUnitTestEngine.php b/src/unit/engine/ArcanistConfigurationDrivenUnitTestEngine.php index e24b4d5f..652a46d2 100644 --- a/src/unit/engine/ArcanistConfigurationDrivenUnitTestEngine.php +++ b/src/unit/engine/ArcanistConfigurationDrivenUnitTestEngine.php @@ -1,202 +1,217 @@ buildTestEngines(); foreach ($engines as $engine) { if ($engine->supportsRunAllTests()) { return true; } } return false; } public function buildTestEngines() { $working_copy = $this->getWorkingCopy(); $config_path = $working_copy->getProjectPath('.arcunit'); if (!Filesystem::pathExists($config_path)) { throw new ArcanistUsageException( pht( "Unable to find '%s' file to configure test engines. Create an ". "'%s' file in the root directory of the working copy.", '.arcunit', '.arcunit')); } $data = Filesystem::readFile($config_path); $config = null; try { $config = phutil_json_decode($data); } catch (PhutilJSONParserException $ex) { throw new PhutilProxyException( pht( "Expected '%s' file to be a valid JSON file, but ". "failed to decode '%s'.", '.arcunit', $config_path), $ex); } $test_engines = $this->loadAvailableTestEngines(); try { PhutilTypeSpec::checkMap( $config, array( 'engines' => 'map>', )); } catch (PhutilTypeCheckException $ex) { throw new PhutilProxyException( pht("Error in parsing '%s' file.", $config_path), $ex); } $built_test_engines = array(); $all_paths = $this->getPaths(); foreach ($config['engines'] as $name => $spec) { $type = idx($spec, 'type'); if ($type !== null) { if (empty($test_engines[$type])) { throw new ArcanistUsageException( pht( "Test engine '%s' specifies invalid type '%s'. ". "Available test engines are: %s.", $name, $type, implode(', ', array_keys($test_engines)))); } $test_engine = clone $test_engines[$type]; } else { // We'll raise an error below about the invalid "type" key. // TODO: Can we just do the type check first, and simplify this a bit? $test_engine = null; } try { PhutilTypeSpec::checkMap( $spec, array( 'type' => 'string', 'include' => 'optional regex | list', 'exclude' => 'optional regex | list', )); } catch (PhutilTypeCheckException $ex) { throw new PhutilProxyException( pht( "Error in parsing '%s' file, for test engine '%s'.", '.arcunit', $name), $ex); } if ($all_paths) { $include = (array)idx($spec, 'include', array()); $exclude = (array)idx($spec, 'exclude', array()); $paths = $this->matchPaths( $all_paths, $include, $exclude); $test_engine->setPaths($paths); } $built_test_engines[] = $test_engine; } return $built_test_engines; } public function run() { + $renderer = $this->renderer; + $this->setRenderer(null); + $paths = $this->getPaths(); // If we are running with `--everything` then `$paths` will be `null`. if (!$paths) { $paths = array(); } - $engines = $this->buildTestEngines(); - $results = array(); - $exceptions = array(); + $engines = $this->buildTestEngines(); + $all_results = array(); + $exceptions = array(); foreach ($engines as $engine) { $engine ->setWorkingCopy($this->getWorkingCopy()) - ->setEnableCoverage($this->getEnableCoverage()); + ->setEnableCoverage($this->getEnableCoverage()) + ->setRenderer($renderer); // TODO: At some point, maybe we should emit a warning here if an engine // doesn't support `--everything`, to reduce surprise when `--everything` // does not really mean `--everything`. if ($engine->supportsRunAllTests()) { $engine->setRunAllTests($this->getRunAllTests()); } try { // TODO: Type check the results. - $results[] = $engine->run(); + $results = $engine->run(); + $all_results[] = $results; + + foreach ($results as $result) { + if ($engine->shouldEchoTestResults()) { + echo $renderer->renderUnitResult($result); + } + } } catch (ArcanistNoEffectException $ex) { $exceptions[] = $ex; } } - if (!$results) { + if (!$all_results) { // If all engines throw an `ArcanistNoEffectException`, then we should // preserve this behavior. throw new ArcanistNoEffectException(pht('No tests to run.')); } - return array_mergev($results); + return array_mergev($all_results); + } + + public function shouldEchoTestResults() { + return false; } private function loadAvailableTestEngines() { return id(new PhutilClassMapQuery()) ->setAncestorClass('ArcanistUnitTestEngine') ->setUniqueMethod('getEngineConfigurationName', true) ->execute(); } /** * TODO: This is copied from @{class:ArcanistConfigurationDrivenLintEngine}. */ private function matchPaths(array $paths, array $include, array $exclude) { $match = array(); foreach ($paths as $path) { $keep = false; if (!$include) { $keep = true; } else { foreach ($include as $rule) { if (preg_match($rule, $path)) { $keep = true; break; } } } if (!$keep) { continue; } if ($exclude) { foreach ($exclude as $rule) { if (preg_match($rule, $path)) { continue 2; } } } $match[] = $path; } return $match; } } diff --git a/src/unit/engine/ArcanistUnitTestEngine.php b/src/unit/engine/ArcanistUnitTestEngine.php index e105f216..b54bafea 100644 --- a/src/unit/engine/ArcanistUnitTestEngine.php +++ b/src/unit/engine/ArcanistUnitTestEngine.php @@ -1,118 +1,98 @@ supportsRunAllTests() && $run_all_tests) { throw new Exception( pht( "Engine '%s' does not support %s.", get_class($this), '--everything')); } $this->runAllTests = $run_all_tests; return $this; } final public function getRunAllTests() { return $this->runAllTests; } protected function supportsRunAllTests() { return false; } final public function setConfigurationManager( ArcanistConfigurationManager $configuration_manager) { $this->configurationManager = $configuration_manager; return $this; } final public function getConfigurationManager() { return $this->configurationManager; } final public function setWorkingCopy( ArcanistWorkingCopyIdentity $working_copy) { $this->workingCopy = $working_copy; return $this; } final public function getWorkingCopy() { return $this->workingCopy; } final public function setPaths(array $paths) { $this->paths = $paths; return $this; } final public function getPaths() { return $this->paths; } - final public function setArguments(array $arguments) { - $this->arguments = $arguments; - return $this; - } - - final public function getArgument($key, $default = null) { - return idx($this->arguments, $key, $default); - } - - final public function setEnableAsyncTests($enable_async_tests) { - $this->enableAsyncTests = $enable_async_tests; - return $this; - } - - final public function getEnableAsyncTests() { - return $this->enableAsyncTests; - } - final public function setEnableCoverage($enable_coverage) { $this->enableCoverage = $enable_coverage; return $this; } final public function getEnableCoverage() { return $this->enableCoverage; } - final public function setRenderer(ArcanistUnitRenderer $renderer) { + final public function setRenderer(ArcanistUnitRenderer $renderer = null) { $this->renderer = $renderer; return $this; } abstract public function run(); /** * Modify the return value of this function in the child class, if you do * not need to echo the test results after all the tests have been run. This * is the case for example when the child class prints the tests results * while the tests are running. */ public function shouldEchoTestResults() { return true; } } diff --git a/src/unit/engine/PhutilUnitTestEngine.php b/src/unit/engine/PhutilUnitTestEngine.php index 110f8e35..42481434 100644 --- a/src/unit/engine/PhutilUnitTestEngine.php +++ b/src/unit/engine/PhutilUnitTestEngine.php @@ -1,228 +1,224 @@ getRunAllTests()) { $run_tests = $this->getAllTests(); } else { $run_tests = $this->getTestsForPaths(); } if (!$run_tests) { throw new ArcanistNoEffectException(pht('No tests to run.')); } $enable_coverage = $this->getEnableCoverage(); if ($enable_coverage !== false) { if (!function_exists('xdebug_start_code_coverage')) { if ($enable_coverage === true) { throw new ArcanistUsageException( pht( 'You specified %s but %s is not available, so '. 'coverage can not be enabled for %s.', '--coverage', 'XDebug', __CLASS__)); } } else { $enable_coverage = true; } } $test_cases = array(); foreach ($run_tests as $test_class) { $test_case = newv($test_class, array()) ->setEnableCoverage($enable_coverage) ->setWorkingCopy($this->getWorkingCopy()); if ($this->getPaths()) { $test_case->setPaths($this->getPaths()); } if ($this->renderer) { $test_case->setRenderer($this->renderer); } $test_cases[] = $test_case; } foreach ($test_cases as $test_case) { $test_case->willRunTestCases($test_cases); } $results = array(); foreach ($test_cases as $test_case) { $results[] = $test_case->run(); } $results = array_mergev($results); foreach ($test_cases as $test_case) { $test_case->didRunTestCases($test_cases); } return $results; } private function getAllTests() { $project_root = $this->getWorkingCopy()->getProjectRoot(); $symbols = id(new PhutilSymbolLoader()) ->setType('class') ->setAncestorClass('PhutilTestCase') ->setConcreteOnly(true) ->selectSymbolsWithoutLoading(); $in_working_copy = array(); $run_tests = array(); foreach ($symbols as $symbol) { if (!preg_match('@(?:^|/)__tests__/@', $symbol['where'])) { continue; } $library = $symbol['library']; if (!isset($in_working_copy[$library])) { $library_root = phutil_get_library_root($library); $in_working_copy[$library] = Filesystem::isDescendant( $library_root, $project_root); } if ($in_working_copy[$library]) { $run_tests[] = $symbol['name']; } } return $run_tests; } /** * Retrieve all relevant test cases. * * Looks for any class that extends @{class:PhutilTestCase} inside a * `__tests__` directory in any parent directory of every affected file. * * The idea is that "infrastructure/__tests__/" tests defines general tests * for all of "infrastructure/", and those tests run for any change in * "infrastructure/". However, "infrastructure/concrete/rebar/__tests__/" * defines more specific tests that run only when "rebar/" (or some * subdirectory) changes. * * @return list The names of the test case classes to be executed. */ private function getTestsForPaths() { $look_here = $this->getTestPaths(); $run_tests = array(); foreach ($look_here as $path_info) { $library = $path_info['library']; $path = $path_info['path']; $symbols = id(new PhutilSymbolLoader()) ->setType('class') ->setLibrary($library) ->setPathPrefix($path) ->setAncestorClass('PhutilTestCase') ->setConcreteOnly(true) ->selectAndLoadSymbols(); foreach ($symbols as $symbol) { $run_tests[$symbol['name']] = true; } } return array_keys($run_tests); } /** * Returns the paths in which we should look for tests to execute. * * @return list A list of paths in which to search for test cases. */ public function getTestPaths() { $root = $this->getWorkingCopy()->getProjectRoot(); $paths = array(); foreach ($this->getPaths() as $path) { $library_root = phutil_get_library_root_for_path($path); if (!$library_root) { continue; } $library_name = phutil_get_library_name_for_root($library_root); if (!$library_name) { throw new Exception( pht( "Attempting to run unit tests on a libphutil library which has ". "not been loaded, at:\n\n". " %s\n\n". "This probably means one of two things:\n\n". " - You may need to add this library to %s.\n". " - You may be running tests on a copy of libphutil or ". "arcanist using a different copy of libphutil or arcanist. ". "This operation is not supported.\n", $library_root, '.arcconfig.')); } $path = Filesystem::resolvePath($path, $root); $library_path = Filesystem::readablePath($path, $library_root); if (!Filesystem::isDescendant($path, $library_root)) { // We have encountered some kind of symlink maze -- for instance, $path // is some symlink living outside the library that links into some file // inside the library. Just ignore these cases, since the affected file // does not actually lie within the library. continue; } if (is_file($path) && preg_match('@(?:^|/)__tests__/@', $path)) { $paths[$library_name.':'.$library_path] = array( 'library' => $library_name, 'path' => $library_path, ); continue; } foreach (Filesystem::walkToRoot($path, $library_root) as $subpath) { if ($subpath == $library_root) { $paths[$library_name.':.'] = array( 'library' => $library_name, 'path' => '__tests__/', ); } else { $library_subpath = Filesystem::readablePath($subpath, $library_root); $paths[$library_name.':'.$library_subpath] = array( 'library' => $library_name, 'path' => $library_subpath.'/__tests__/', ); } } } return $paths; } - public function shouldEchoTestResults() { - return !$this->renderer; - } - } diff --git a/src/workflow/ArcanistUnitWorkflow.php b/src/workflow/ArcanistUnitWorkflow.php index 04d8c8e0..a1aadd3b 100644 --- a/src/workflow/ArcanistUnitWorkflow.php +++ b/src/workflow/ArcanistUnitWorkflow.php @@ -1,435 +1,427 @@ array( 'param' => 'revision', 'help' => pht( 'Run unit tests covering changes since a specific revision.'), 'supports' => array( 'git', 'hg', ), 'nosupport' => array( 'svn' => pht( 'Arc unit does not currently support %s in SVN.', '--rev'), ), ), 'engine' => array( 'param' => 'classname', 'help' => pht('Override configured unit engine for this project.'), ), 'coverage' => array( 'help' => pht('Always enable coverage information.'), 'conflicts' => array( 'no-coverage' => null, ), ), 'no-coverage' => array( 'help' => pht('Always disable coverage information.'), ), 'detailed-coverage' => array( 'help' => pht( 'Show a detailed coverage report on the CLI. Implies %s.', '--coverage'), ), 'json' => array( 'help' => pht('Report results in JSON format.'), ), 'output' => array( 'param' => 'format', 'help' => pht( "With 'full', show full pretty report (Default). ". "With 'json', report results in JSON format. ". "With 'ugly', use uglier (but more efficient) JSON formatting. ". "With 'none', don't print results."), 'conflicts' => array( 'json' => pht('Only one output format allowed'), 'ugly' => pht('Only one output format allowed'), ), ), 'target' => array( 'param' => 'phid', 'help' => pht( '(PROTOTYPE) Record a copy of the test results on the specified '. 'Harbormaster build target.'), ), 'everything' => array( 'help' => pht('Run every test.'), 'conflicts' => array( 'rev' => pht('%s runs all tests.', '--everything'), ), ), 'ugly' => array( 'help' => pht( 'With %s, use uglier (but more efficient) formatting.', '--json'), ), '*' => 'paths', ); } public function requiresWorkingCopy() { return true; } public function requiresRepositoryAPI() { return true; } public function requiresConduit() { return $this->shouldUploadResults(); } public function requiresAuthentication() { return $this->shouldUploadResults(); } public function getEngine() { return $this->engine; } public function run() { $working_copy = $this->getWorkingCopy(); $paths = $this->getArgument('paths'); $rev = $this->getArgument('rev'); $everything = $this->getArgument('everything'); if ($everything && $paths) { throw new ArcanistUsageException( pht( 'You can not specify paths with %s. The %s flag runs every test.', '--everything', '--everything')); } if ($everything) { $paths = iterator_to_array($this->getRepositoryApi()->getAllFiles()); } else { $paths = $this->selectPathsForWorkflow($paths, $rev); } $this->engine = $this->newUnitTestEngine($this->getArgument('engine')); if ($everything) { $this->engine->setRunAllTests(true); } else { $this->engine->setPaths($paths); } - $this->engine->setArguments($this->getPassthruArgumentsAsMap('unit')); $renderer = new ArcanistUnitConsoleRenderer(); $this->engine->setRenderer($renderer); $enable_coverage = null; // Means "default". if ($this->getArgument('coverage') || $this->getArgument('detailed-coverage')) { $enable_coverage = true; } else if ($this->getArgument('no-coverage')) { $enable_coverage = false; } $this->engine->setEnableCoverage($enable_coverage); - // Enable possible async tests only for 'arc diff' not 'arc unit' - if ($this->getParentWorkflow()) { - $this->engine->setEnableAsyncTests(true); - } else { - $this->engine->setEnableAsyncTests(false); - } - $results = $this->engine->run(); $this->validateUnitEngineResults($this->engine, $results); $this->testResults = $results; $console = PhutilConsole::getConsole(); $output_format = $this->getOutputFormat(); if ($output_format !== 'full') { $console->disableOut(); } $unresolved = array(); $coverage = array(); foreach ($results as $result) { $result_code = $result->getResult(); if ($this->engine->shouldEchoTestResults()) { $console->writeOut('%s', $renderer->renderUnitResult($result)); } if ($result_code != ArcanistUnitTestResult::RESULT_PASS) { $unresolved[] = $result; } if ($result->getCoverage()) { foreach ($result->getCoverage() as $file => $report) { $coverage[$file][] = $report; } } } if ($coverage) { $file_coverage = array_fill_keys( $paths, 0); $file_reports = array(); foreach ($coverage as $file => $reports) { $report = ArcanistUnitTestResult::mergeCoverage($reports); $cov = substr_count($report, 'C'); $uncov = substr_count($report, 'U'); if ($cov + $uncov) { $coverage = $cov / ($cov + $uncov); } else { $coverage = 0; } $file_coverage[$file] = $coverage; $file_reports[$file] = $report; } $console->writeOut("\n__%s__\n", pht('COVERAGE REPORT')); asort($file_coverage); foreach ($file_coverage as $file => $coverage) { $console->writeOut( " **%s%%** %s\n", sprintf('% 3d', (int)(100 * $coverage)), $file); $full_path = $working_copy->getProjectRoot().'/'.$file; if ($this->getArgument('detailed-coverage') && Filesystem::pathExists($full_path) && is_file($full_path) && array_key_exists($file, $file_reports)) { $console->writeOut( '%s', $this->renderDetailedCoverageReport( Filesystem::readFile($full_path), $file_reports[$file])); } } } $this->unresolvedTests = $unresolved; $overall_result = self::RESULT_OKAY; foreach ($results as $result) { $result_code = $result->getResult(); if ($result_code == ArcanistUnitTestResult::RESULT_FAIL || $result_code == ArcanistUnitTestResult::RESULT_BROKEN) { $overall_result = self::RESULT_FAIL; break; } else if ($result_code == ArcanistUnitTestResult::RESULT_UNSOUND) { $overall_result = self::RESULT_UNSOUND; } } if ($output_format !== 'full') { $console->enableOut(); } $data = array_values(mpull($results, 'toDictionary')); switch ($output_format) { case 'ugly': $console->writeOut('%s', json_encode($data)); break; case 'json': $json = new PhutilJSON(); $console->writeOut('%s', $json->encodeFormatted($data)); break; case 'full': // already printed break; case 'none': // do nothing break; } $target_phid = $this->getArgument('target'); if ($target_phid) { $this->uploadTestResults($target_phid, $overall_result, $results); } return $overall_result; } public function getUnresolvedTests() { return $this->unresolvedTests; } public function getTestResults() { return $this->testResults; } private function renderDetailedCoverageReport($data, $report) { $data = explode("\n", $data); $out = ''; $n = 0; foreach ($data as $line) { $out .= sprintf('% 5d ', $n + 1); $line = str_pad($line, 80, ' '); if (empty($report[$n])) { $c = 'N'; } else { $c = $report[$n]; } switch ($c) { case 'C': $out .= phutil_console_format( ' %s ', $line); break; case 'U': $out .= phutil_console_format( ' %s ', $line); break; case 'X': $out .= phutil_console_format( ' %s ', $line); break; default: $out .= ' '.$line.' '; break; } $out .= "\n"; $n++; } return $out; } private function getOutputFormat() { if ($this->getArgument('ugly')) { return 'ugly'; } if ($this->getArgument('json')) { return 'json'; } $format = $this->getArgument('output'); $known_formats = array( 'none' => 'none', 'json' => 'json', 'ugly' => 'ugly', 'full' => 'full', ); return idx($known_formats, $format, 'full'); } /** * Raise a tailored error when a unit test engine returns results in an * invalid format. * * @param ArcanistUnitTestEngine The engine. * @param wild Results from the engine. */ private function validateUnitEngineResults( ArcanistUnitTestEngine $engine, $results) { if (!is_array($results)) { throw new Exception( pht( 'Unit test engine (of class "%s") returned invalid results when '. 'run (with method "%s"). Expected a list of "%s" objects as results.', get_class($engine), 'run()', 'ArcanistUnitTestResult')); } foreach ($results as $key => $result) { if (!($result instanceof ArcanistUnitTestResult)) { throw new Exception( pht( 'Unit test engine (of class "%s") returned invalid results when '. 'run (with method "%s"). Expected a list of "%s" objects as '. 'results, but value with key "%s" is not valid.', get_class($engine), 'run()', 'ArcanistUnitTestResult', $key)); } } } public static function getHarbormasterTypeFromResult($unit_result) { switch ($unit_result) { case self::RESULT_OKAY: case self::RESULT_SKIP: $type = 'pass'; break; default: $type = 'fail'; break; } return $type; } private function shouldUploadResults() { return ($this->getArgument('target') !== null); } private function uploadTestResults( $target_phid, $unit_result, array $unit) { // TODO: It would eventually be nice to stream test results up to the // server as we go, but just get things working for now. $message_type = self::getHarbormasterTypeFromResult($unit_result); foreach ($unit as $key => $result) { $dictionary = $result->toDictionary(); $unit[$key] = $this->getModernUnitDictionary($dictionary); } $this->getConduit()->callMethodSynchronous( 'harbormaster.sendmessage', array( 'buildTargetPHID' => $target_phid, 'unit' => array_values($unit), 'type' => $message_type, )); } }