diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -210,6 +210,7 @@ 'ArcanistXUnitTestResultParser' => 'unit/parser/ArcanistXUnitTestResultParser.php', 'CSharpToolsTestEngine' => 'unit/engine/CSharpToolsTestEngine.php', 'ComprehensiveLintEngine' => 'lint/engine/ComprehensiveLintEngine.php', + 'DotNetXUnitUnitTestEngine' => 'unit/engine/DotNetXUnitUnitTestEngine.php', 'NoseTestEngine' => 'unit/engine/NoseTestEngine.php', 'PhpunitTestEngine' => 'unit/engine/PhpunitTestEngine.php', 'PhpunitTestEngineTestCase' => 'unit/engine/__tests__/PhpunitTestEngineTestCase.php', @@ -387,6 +388,7 @@ 'ArcanistXMLLinterTestCase' => 'ArcanistLinterTestCase', 'CSharpToolsTestEngine' => 'XUnitTestEngine', 'ComprehensiveLintEngine' => 'ArcanistComprehensiveLintEngine', + 'DotNetXUnitUnitTestEngine' => 'ArcanistUnitTestEngine', 'NoseTestEngine' => 'ArcanistUnitTestEngine', 'PhpunitTestEngine' => 'ArcanistUnitTestEngine', 'PhpunitTestEngineTestCase' => 'ArcanistTestCase', diff --git a/src/unit/engine/DotNetXUnitUnitTestEngine.php b/src/unit/engine/DotNetXUnitUnitTestEngine.php new file mode 100644 --- /dev/null +++ b/src/unit/engine/DotNetXUnitUnitTestEngine.php @@ -0,0 +1,504 @@ + "bin/Release/$1.dll"}. + * + * Paths will always be passed in with forward slashes, starting from the + * root of the repo. For instance, 'Source/Foo/MyProject.csproj'. Delimit + * the pattern with something that won't appear in the regex, such as '@'. + * + * - unit.dotnet.xunit.binary: the path to the xUnit.NET console runner + * binary. If it's not in $PATH, it will be resolved relative to the project + * root (i.e. where .arcconfig lives.) + * + * - unit.dotnet.xunit.args: additional arguments to pass to the test runner. + * This can be useful, for instance, to narrow the scope of testing down to + * specific traits. Example, listing some traits that should not be run: + * + * ["/-trait", "Speed=Slow", "/-trait", "Environment=SQA"] + * + * - unit.dotnet.xunit.args.mono: arguments to pass to the test runner only + * when running under mono. If omitted, uses the same value as + * unit.dotnet.xunit.args. + * + * @concrete-extensible + */ +class DotNetXUnitUnitTestEngine extends ArcanistUnitTestEngine { + + protected $projectRoot; + protected $testProjects; + protected $buildEngine; + protected $buildArgs; + protected $testEngine; + protected $xunitArgs; + protected $runtimeEngine; + + /** + * This test engine supports running all tests. + */ + protected function supportsRunAllTests() { + return true; + } + + /** + * Main entry point for the test engine. Determines what assemblies to build + * and test based on the files that have changed. + * + * @return array Array of test results. + */ + public function run() { + + $this->loadEnvironment(); + + if ($this->getRunAllTests()) { + $paths = id(new FileFinder($this->projectRoot))->find(); + } else { + $paths = $this->getPaths(); + } + + $test_projects = $this->getImpactedTestProjects($paths); + if (empty($test_projects)) { + return array(); + } + + $results = array(); + $results[] = $this->buildTestProjects($test_projects); + if ($this->resultsContainFailures($results)) { + return array_mergev($results); + } + + $results[] = $this->runTests($test_projects); + return array_mergev($results); + } + + /** + * Determine what executables and test paths to use. Between platforms this + * also changes whether the test engine is run under .NET or Mono. It also + * ensures that all of the required binaries are available for the tests to + * run successfully. + * + * @return void + */ + protected function loadEnvironment() { + $this->projectRoot = $this->getWorkingCopy()->getProjectRoot(); + $this->buildEngine = $this->getBuildEngine(); + $this->runtimeEngine = $this->getRuntimeEngine(); + + // Any relative paths will be relative to the root of the repo. + // See https://secure.phabricator.com/T6912 + chdir($this->projectRoot); + + $conf = $this->getConfigurationManager(); + + $proj_patterns = $conf->getConfigFromAnySource( + 'unit.dotnet.testprojects', array()); + if (empty($proj_patterns)) { + throw new Exception( + 'Please configure at least one project containing unit tests '. + 'via the `unit.dotnet.testprojects` option in .arcconfig.'); + } + + $all_files = id(new FileFinder($this->projectRoot))->find(); + $this->testProjects = $this->resolveProjectOutputs( + $proj_patterns, $all_files); + if (empty($this->testProjects)) { + throw new Exception('Found no test projects that match configuration.'); + } + + $msbuild_args = $conf->getConfigFromAnySource( + 'unit.msbuild.args', array()); + $xbuild_args = $conf->getConfigFromAnySource( + 'unit.xbuild.args', $msbuild_args); + if ($this->buildEngine == 'msbuild') { + $this->buildArgs = $msbuild_args; + } else { + $this->buildArgs = $xbuild_args; + } + + $this->testEngine = $conf->getConfigFromAnySource( + 'unit.dotnet.xunit.binary', 'xunit.console.clr4.exe'); + If (!Filesystem::binaryExists($this->testEngine)) { + throw new Exception( + 'Unable to locate the xUnit.NET test runner binary. Please update '. + 'your $PATH or configure it via the `unit.dotnet.xunit.binary` option '. + 'in .arcconfig'); + } + + $xunit_args = $conf->getConfigFromAnySource( + 'unit.dotnet.xunit.args', array()); + $xunit_mono_args = $conf->getConfigFromAnySource( + 'unit.dotnet.xunit.args.mono', $xunit_args); + if ($this->runtimeEngine == 'mono') { + $this->xunitArgs = $xunit_mono_args; + } else { + $this->xunitArgs = $xunit_args; + } + } + + /** + * Check if any build engine candidates are on the $PATH. + * + * @return string Either 'msbuild' or 'xbuild' + */ + private function getBuildEngine() { + if (Filesystem::binaryExists('msbuild')) { + return 'msbuild'; + } else if (Filesystem::binaryExists('xbuild')) { + return 'xbuild'; + } else { + throw new Exception('Unable to find msbuild or xbuild in PATH!'); + } + } + + /** + * Check if the mono runtime is required & available. + * + * @return string 'mono' if mono is required, null otherwise. + */ + private function getRuntimeEngine() { + if (phutil_is_windows()) { + return null; + } else if (Filesystem::binaryExists('mono')) { + return 'mono'; + } else { + throw new Exception('Unable to find Mono and you are not on Windows!'); + } + } + + /** + * Find paths matching the keys of the input array, and return the matches + * along with the dlls they map to. Use forward slashes in the paths. + * + * Example: + * + * array('@([^/]*\.Tests)\.csproj$@' => 'bin/Release/$1.dll') + * + * @param array A map of search & replace rules, in which the keys are + * patterns of build engine inputs (project files,) and the + * values are patterns of xUnit.NET inputs (assemblies.) + * @return array A map from search matches (actual project files) to + * replacement values. Forward slashes are used as separators. + */ + private function resolveProjectOutputs(array $rules, array $files) { + $results = array(); + foreach ($rules as $input_regex => $output_pattern) { + foreach ($files as $test_path) { + if (preg_match($input_regex, $test_path) === 1) { + $dll_path = preg_replace($input_regex, $output_pattern, $test_path); + $results[$test_path] = $dll_path; + } + } + } + return $results; + } + + /** + * Deduce which test projects are either directly or indirectly impacted by + * changes to some source files. + * + * @param array Array of paths to modified files. + * @return array Array of test projects that should be run. + */ + private function getImpactedTestProjects(array $changed_paths) { + $changed_paths = array_map('Filesystem::resolvePath', $changed_paths); + $impacted_projects = array(); + foreach ($this->testProjects as $project_path => $dll_path) { + $impactful_paths = $this->getFilesImpactingProject($project_path); + $impactful_paths[] = $project_path; + + // Removed source files won't be listed in the project file, but the + // project file itself should have been modified. + $intersection = array_intersect($impactful_paths, $changed_paths); + if (!empty($intersection)) { + $impacted_projects[$project_path] = $dll_path; + } + } + return $impacted_projects; + } + + /** + * Collect files belonging to the project, as well as all the files belonging + * to its dependencies. This serves as a reasonable lower-bound for the + * project's surface area for bugs. + * + * @param string Path to the project file. + * @return array Paths to all files impacting the project. + */ + private function getFilesImpactingProject($csproj_path) { + list($files, $dependent_projs) = $this->parseProjectFile($csproj_path); + $all_files = array($files); + foreach ($dependent_projs as $dependent_proj) { + $all_files[] = $this->getFilesImpactingProject($dependent_proj); + } + // Note: no uniqueness guarantee + return array_mergev($all_files); + } + + /** + * Extract a list of files to be compiled and a list of dependency projects + * from the XML of a .NET project file. + * + * Notes: + * - No glob support (.\**\*.cs or even .\*.cs) + * - The XML parser is case sensitive, which differs from how VS/MSBuild + * will parse the same file. + * - Paths are returned either relative to the working directory, or as + * absolute paths. + * + * @param string Path to the project file. + * @return array A list of files referenced by project, and a list of + * direct dependency projects. + */ + private function parseProjectFile($csproj_path) { + if (!file_exists($csproj_path)) { + throw new Exception('Project file not found: '.$csproj_path); + } + + $content = file_get_contents($csproj_path); + $xml = simplexml_load_string($content); + $rel_dir = dirname($csproj_path); + + $files = array(); + $refs = array(); + + foreach ($xml->ItemGroup as $item_group) { + foreach ($item_group->Compile as $compile) { + $files[] = $this->resolveWindowsPath($compile['Include'], $rel_dir); + } + foreach ($item_group->ProjectReference as $project_ref) { + $refs[] = $this->resolveWindowsPath($project_ref['Include'], $rel_dir); + } + } + + return array($files, $refs); + } + + private function resolveWindowsPath($path, $relatie_to) { + // Normalize all paths to use forward slashes. Mono is capable of building + // project that use backslashes as the dir separators, so we should handle + // that scenario gracefully. + $normalized_path = str_replace('\\', '/', $path); + return Filesystem::resolvePath($normalized_path, $relatie_to); + } + + /** + * Build the projects relevant for the specified test assemblies and return + * the results of the builds as test results. + * + * @param array Array of test assemblies. + * @return array Array of test results. + */ + private function buildTestProjects(array $test_projects) { + $build_futures = array(); + foreach ($test_projects as $project_path => $dll_path) { + $build_future = new ExecFuture( + '%s %Ls %s', $this->buildEngine, $this->buildArgs, $project_path); + $build_futures[$project_path] = $build_future; + } + + $build_start = microtime(true); + $results = array(); + $iterator = new FutureIterator($build_futures); + foreach ($iterator->limit(1) as $project_path => $build_future) { + $result = new ArcanistUnitTestResult(); + $result->setName('(build) '.$project_path); + + try { + $build_future->resolvex(); + $result->setResult(ArcanistUnitTestResult::RESULT_PASS); + } catch(CommandException $exc) { + if ($exc->getError() > 1) { + throw $exc; + } + $result->setResult(ArcanistUnitTestResult::RESULT_FAIL); + $result->setUserdata($exc->getStdout()); + } + + $result->setDuration(microtime(true) - $build_start); + $results[] = $result; + } + return $results; + } + + /** + * Determine whether or not results contain any failures. This is needed + * since we build the assemblies as part of the unit tests, but we can't run + * any of the unit tests if the build fails. + * + * @param array Array of results to check. + * @return bool If there are any failures in the results. + */ + private function resultsContainFailures(array $results) { + $results = array_mergev($results); + foreach ($results as $result) { + if ($result->getResult() != ArcanistUnitTestResult::RESULT_PASS) { + return true; + } + } + return false; + } + + /** + * Build the future for running a unit test. Override this to provide support + * for code coverage. + * + * @param string Name of the test assembly. + * @return array The future, output filename and coverage filename + * stored in an array. + */ + protected function buildTestFuture($dll_path) { + // FIXME: Can't use TempFile here as xUnit.NET doesn't like UNIX-style full + // paths. It sees the leading / as the start of an option flag. + $xunit_temp = Filesystem::readRandomCharacters(10).'.results.xml'; + if (file_exists($xunit_temp)) { + unlink($xunit_temp); + } + $command = array($this->testEngine, $dll_path); + if ($this->runtimeEngine !== null) { + array_unshift($command, $this->runtimeEngine); + } + $future = new ExecFuture( + '%Ls %Ls /xml %s', $command, $this->xunitArgs, $xunit_temp); + $folder = Filesystem::resolvePath($this->projectRoot); + $combined = $folder.DIRECTORY_SEPARATOR.$xunit_temp; + return array($future, $combined, null); + } + + /** + * Run the xUnit.NET test runner on each of the assemblies and parse the + * resulting XML. + * + * @param array Array of test assemblies. + * @return array Array of test results. + */ + private function runTests(array $test_projects) { + $results = array(); + + // Build the futures for running the tests. + $futures = array(); + $outputs = array(); + $coverages = array(); + foreach ($test_projects as $dll_path) { + list($future_r, $xunit_temp, $coverage) = + $this->buildTestFuture($dll_path); + $futures[$dll_path] = $future_r; + $outputs[$dll_path] = $xunit_temp; + $coverages[$dll_path] = $coverage; + } + + // Run all of the tests. + $iterator = new FutureIterator($futures); + foreach ($iterator->limit(8) as $dll_path => $future) { + list($err, $stdout, $stderr) = $future->resolve(); + + if (file_exists($outputs[$dll_path])) { + $result = $this->parseTestResult( + $outputs[$dll_path], + $coverages[$dll_path]); + $results[] = $result; + unlink($outputs[$dll_path]); + } else { + // FIXME: There's a bug in Mono which causes a segmentation fault + // when xUnit.NET runs; this causes the XML file to not appear + // (depending on when the segmentation fault occurs). See + // https://bugzilla.xamarin.com/show_bug.cgi?id=16379 + // for more information. + + // Since it's not possible for the user to correct this error, we + // ignore the fact the tests didn't run here. + } + } + + return array_mergev($results); + } + + /** + * Returns null for this implementation as xUnit.NET does not support code + * coverage directly. Override this method in another class to provide code + * coverage information. + * + * @param string The name of the coverage file if one was provided by + * `buildTestFuture`. + * @return array Code coverage results, or null. + */ + protected function parseCoverageResult($coverage) { + return null; + } + + /** + * Parse the test results from xUnit.NET. + * + * @param string The name of the xUnit.NET results file. + * @param string The name of the coverage file if one was provided by + * `buildTestFuture`. This is passed through to + * `parseCoverageResult`. + * @return array Test results. + */ + private function parseTestResult($xunit_tmp, $coverage) { + $xunit_dom = new DOMDocument(); + $xunit_dom->loadXML(Filesystem::readFile($xunit_tmp)); + + $results = array(); + $tests = $xunit_dom->getElementsByTagName('test'); + foreach ($tests as $test) { + $name = $test->getAttribute('name'); + $time = $test->getAttribute('time'); + $status = ArcanistUnitTestResult::RESULT_UNSOUND; + switch ($test->getAttribute('result')) { + case 'Pass': + $status = ArcanistUnitTestResult::RESULT_PASS; + break; + case 'Fail': + $status = ArcanistUnitTestResult::RESULT_FAIL; + break; + case 'Skip': + $status = ArcanistUnitTestResult::RESULT_SKIP; + break; + } + $userdata = ''; + $reason = $test->getElementsByTagName('reason'); + $failure = $test->getElementsByTagName('failure'); + if ($reason->length > 0 || $failure->length > 0) { + $node = ($reason->length > 0) ? $reason : $failure; + $message = $node->item(0)->getElementsByTagName('message'); + if ($message->length > 0) { + $userdata = $message->item(0)->nodeValue; + } + $stacktrace = $node->item(0)->getElementsByTagName('stack-trace'); + if ($stacktrace->length > 0) { + $userdata .= "\n".$stacktrace->item(0)->nodeValue; + } + } + + $result = new ArcanistUnitTestResult(); + $result->setName($name); + $result->setResult($status); + $result->setDuration($time); + $result->setUserData($userdata); + if ($coverage != null) { + $result->setCoverage($this->parseCoverageResult($coverage)); + } + $results[] = $result; + } + + return $results; + } + +}