diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -215,6 +215,7 @@ 'PhutilUnitTestEngineTestCase' => 'unit/engine/__tests__/PhutilUnitTestEngineTestCase.php', 'PytestTestEngine' => 'unit/engine/PytestTestEngine.php', 'XUnitTestEngine' => 'unit/engine/XUnitTestEngine.php', + 'XUnit2TestEngine' => 'unit/engine/XUnit2TestEngine.php', 'XUnitTestResultParserTestCase' => 'unit/parser/__tests__/XUnitTestResultParserTestCase.php', ), 'function' => array(), @@ -390,6 +391,7 @@ 'PhutilUnitTestEngineTestCase' => 'ArcanistTestCase', 'PytestTestEngine' => 'ArcanistUnitTestEngine', 'XUnitTestEngine' => 'ArcanistUnitTestEngine', + 'XUnit2TestEngine' => 'ArcanistUnitTestEngine', 'XUnitTestResultParserTestCase' => 'ArcanistTestCase', ), )); diff --git a/src/unit/engine/XUnit2TestEngine.php b/src/unit/engine/XUnit2TestEngine.php new file mode 100644 --- /dev/null +++ b/src/unit/engine/XUnit2TestEngine.php @@ -0,0 +1,490 @@ +projectRoot = $this->getWorkingCopy()->getProjectRoot(); + + // Determine build engine. + if (Filesystem::binaryExists('msbuild')) { + $this->buildEngine = 'msbuild'; + } else if (Filesystem::binaryExists('xbuild')) { + $this->buildEngine = 'xbuild'; + } else { + throw new Exception('Unable to find msbuild or xbuild in PATH!'); + } + + // Determine runtime engine (.NET or Mono). + if (phutil_is_windows()) { + $this->runtimeEngine = ''; + } else if (Filesystem::binaryExists('mono')) { + $this->runtimeEngine = Filesystem::resolveBinary('mono'); + } else { + throw new Exception('Unable to find Mono and you are not on Windows!'); + } + + // Read the discovery rules. + $this->discoveryRules = + $this->getConfigurationManager()->getConfigFromAnySource( + 'unit.csharp.discovery'); + if ($this->discoveryRules === null) { + throw new Exception( + 'You must configure discovery rules to map C# files '. + 'back to test projects (`unit.csharp.discovery` in .arcconfig).'); + } + + // Determine xUnit test runner path. + if ($this->xunitHintPath === null) { + $this->xunitHintPath = + $this->getConfigurationManager()->getConfigFromAnySource( + 'unit.csharp.xunit.binary'); + } + $xunit = $this->projectRoot.DIRECTORY_SEPARATOR.$this->xunitHintPath; + if (file_exists($xunit) && $this->xunitHintPath !== null) { + $this->testEngine = Filesystem::resolvePath($xunit); + } else if (Filesystem::binaryExists('xunit.console.exe')) { + $this->testEngine = 'xunit.console.exe'; + } else { + throw new Exception( + "Unable to locate xUnit console runner. Configure ". + "it with the `unit.csharp.xunit.binary' option in .arcconfig"); + } + } + + /** + * Main entry point for the test engine. Determines what assemblies to build + * and test based on the files that have changed. + * + * @return array Array of test results. + */ + public function run() { + + $this->loadEnvironment(); + + if ($this->getRunAllTests()) { + $paths = id(new FileFinder($this->projectRoot))->find(); + } else { + $paths = $this->getPaths(); + } + + return $this->runAllTests($this->mapPathsToResults($paths)); + } + + /** + * Applies the discovery rules to the set of paths specified. + * + * @param array Array of paths. + * @return array Array of paths to test projects and assemblies. + */ + public function mapPathsToResults(array $paths) { + $results = array(); + foreach ($this->discoveryRules as $regex => $targets) { + $regex = str_replace('/', '\\/', $regex); + foreach ($paths as $path) { + if (preg_match('/'.$regex.'/', $path) === 1) { + foreach ($targets as $target) { + // Index 0 is the test project (.csproj file) + // Index 1 is the output assembly (.dll file) + $project = preg_replace('/'.$regex.'/', $target[0], $path); + $project = $this->projectRoot.DIRECTORY_SEPARATOR.$project; + $assembly = preg_replace('/'.$regex.'/', $target[1], $path); + $assembly = $this->projectRoot.DIRECTORY_SEPARATOR.$assembly; + if (file_exists($project)) { + $project = Filesystem::resolvePath($project); + $assembly = Filesystem::resolvePath($assembly); + + // Check to ensure uniqueness. + $exists = false; + foreach ($results as $existing) { + if ($existing['assembly'] === $assembly) { + $exists = true; + break; + } + } + + if (!$exists) { + $results[] = array( + 'project' => $project, + 'assembly' => $assembly, + ); + } + } + } + } + } + } + return $results; + } + + /** + * Builds and runs the specified test assemblies. + * + * @param array Array of paths to test project files. + * @return array Array of test results. + */ + public function runAllTests(array $test_projects) { + if (empty($test_projects)) { + return array(); + } + + $results = array(); + $results[] = $this->generateProjects(); + if ($this->resultsContainFailures($results)) { + return array_mergev($results); + } + $results[] = $this->buildProjects($test_projects); + if ($this->resultsContainFailures($results)) { + return array_mergev($results); + } + $results[] = $this->testAssemblies($test_projects); + + return array_mergev($results); + } + + /** + * Determine whether or not a current set of results contains any failures. + * This is needed since we build the assemblies as part of the unit tests, but + * we can't run any of the unit tests if the build fails. + * + * @param array Array of results to check. + * @return bool If there are any failures in the results. + */ + private function resultsContainFailures(array $results) { + $results = array_mergev($results); + foreach ($results as $result) { + if ($result->getResult() != ArcanistUnitTestResult::RESULT_PASS) { + return true; + } + } + return false; + } + + /** + * If the `Build` directory exists, we assume that this is a multi-platform + * project that requires generation of C# project files. Because we want to + * test that the generation and subsequent build is whole, we need to + * regenerate any projects in case the developer has added files through an + * IDE and then forgotten to add them to the respective `.definitions` file. + * By regenerating the projects we ensure that any missing definition entries + * will cause the build to fail. + * + * @return array Array of test results. + */ + private function generateProjects() { + // No "Build" directory; so skip generation of projects. + if (!is_dir(Filesystem::resolvePath($this->projectRoot.'/Build'))) { + return array(); + } + + // No "Protobuild.exe" file; so skip generation of projects. + if (!is_file(Filesystem::resolvePath( + $this->projectRoot.'/Protobuild.exe'))) { + + return array(); + } + + // Work out what platform the user is building for already. + $platform = phutil_is_windows() ? 'Windows' : 'Linux'; + $files = Filesystem::listDirectory($this->projectRoot); + foreach ($files as $file) { + if (strtolower(substr($file, -4)) == '.sln') { + $parts = explode('.', $file); + $platform = $parts[count($parts) - 2]; + break; + } + } + + $regenerate_start = microtime(true); + $regenerate_future = new ExecFuture( + '%C Protobuild.exe --resync %s', + $this->runtimeEngine, + $platform); + $regenerate_future->setCWD(Filesystem::resolvePath( + $this->projectRoot)); + $results = array(); + $result = new ArcanistUnitTestResult(); + $result->setName("(regenerate projects for $platform)"); + + try { + $regenerate_future->resolvex(); + $result->setResult(ArcanistUnitTestResult::RESULT_PASS); + } catch(CommandException $exc) { + if ($exc->getError() > 1) { + throw $exc; + } + $result->setResult(ArcanistUnitTestResult::RESULT_FAIL); + $result->setUserdata($exc->getStdout()); + } + + $result->setDuration(microtime(true) - $regenerate_start); + $results[] = $result; + return $results; + } + + /** + * Build the projects relevant for the specified test assemblies and return + * the results of the builds as test results. This build also passes the + * "SkipTestsOnBuild" parameter when building the projects, so that MSBuild + * conditionals can be used to prevent any tests running as part of the + * build itself (since the unit tester is about to run each of the tests + * individually). + * + * @param array Array of test assemblies. + * @return array Array of test results. + */ + private function buildProjects(array $test_assemblies) { + $build_start = microtime(true); + + // Build an MSBuild project that imports all of the projects we want. + // This allows MSBuild to build all of our targets in parallel, instead + // of executing lots of MSBuild commands in sequence. + $imports = ""; + foreach ($test_assemblies as $test_assembly) { + $path = Filesystem::resolvePath($test_assembly['project']); + if (file_exists($path)) { + $result = new ArcanistUnitTestResult(); + $result->setName('Discovered: '.$path); + $result->setResult(ArcanistUnitTestResult::RESULT_PASS); + $results[] = $result; + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + } else { + $result = new ArcanistUnitTestResult(); + $result->setName('Missing: '.$path); + $result->setResult(ArcanistUnitTestResult::RESULT_UNSOUND); + $result->setUserData('C# project not found'); + $results[] = $result; + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + } + + $imports .= "\n"; + } + $xml = << + + +$imports + + + + + +EOF; + $temp = new TempFile(); + file_put_contents((string)$temp, $xml); + + $result = new ArcanistUnitTestResult(); + $result->setName('(build code)'); + + $args = array(); + $args[] = (string)$temp; + $args[] = '/p:SkipTestsOnBuild=True'; + if ($this->buildEngine === 'msbuild') { + $args[] = '/m'; + } + + $build_future = new ExecFuture( + '%C %Ls', + $this->buildEngine, + $args); + try { + $build_future->resolvex(); + $result->setResult(ArcanistUnitTestResult::RESULT_PASS); + } catch (CommandException $exc) { + if ($exc->getError() > 1) { + throw $exc; + } + $result->setResult(ArcanistUnitTestResult::RESULT_FAIL); + $result->setUserdata($exc->getStdout()); + } + + $result->setDuration(microtime(true) - $build_start); + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + + return array($result); + } + + /** + * Run the xUnit test runner on each of the assemblies and parse the + * resulting XML. + * + * @param array Array of test assemblies. + * @return array Array of test results. + */ + private function testAssemblies(array $test_assemblies) { + $results = array(); + + // Make sure we only pass in test assemblies that exist. + $existing_test_assemblies = array(); + foreach (ipull($test_assemblies, 'assembly') as $assembly_path) { + if (file_exists($assembly_path)) { + $existing_test_assemblies[] = $assembly_path; + + $result = new ArcanistUnitTestResult(); + $result->setName('Discovered: '.$assembly_path); + $result->setResult(ArcanistUnitTestResult::RESULT_PASS); + $results[] = $result; + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + } else { + $result = new ArcanistUnitTestResult(); + $result->setName('Missing: '.$assembly_path); + $result->setResult(ArcanistUnitTestResult::RESULT_UNSOUND); + $result->setUserData('Assembly not found'); + $results[] = $result; + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + } + } + + $future = new ExecFuture( + '%C %Ls -teamcity', + trim($this->runtimeEngine.' '.$this->testEngine), + $existing_test_assemblies); + $futures = new FutureIterator(array($future)); + + $stdout_buffer = ''; + + foreach ($futures->setUpdateInterval(0.1) as $key => $future_iter) { + list($stdout, $stderr) = $future->read(); + $stdout_buffer .= $stdout; + echo $stderr; + $future->discardBuffers(); + + $lines = phutil_split_lines($stdout_buffer); + if (strlen($stdout_buffer) > 0 && + $stdout_buffer[strlen($stdout_buffer) - 1] !== "\n") { + + $stdout_buffer = $lines[count($lines) - 1]; + array_pop($lines); + } else { + $stdout_buffer = ""; + } + + if (count($lines) > 0) { + foreach ($lines as $line) { + if (substr($line, 0, 10) === '##teamcity') { + if (!preg_match('/\[([a-zA-Z]+)\ /', $line, $matches)) { + continue; + } + + if (count($matches) < 2) { + continue; + } + + $result_code = null; + + switch ($matches[1]) { + case "testFinished": + $result_code = ArcanistUnitTestResult::RESULT_PASS; + break; + case "testIgnored": + $result_code = ArcanistUnitTestResult::RESULT_SKIP; + break; + case "testFailed": + $result_code = ArcanistUnitTestResult::RESULT_FAIL; + break; + case "testStarted": + case "testSuiteStarted": + case "testSuiteFinished": + break; + default: + throw new Exception('Unknown test event \''.$matches[1].'\' occurred!'); + } + + if ($result_code === null) { + // Some kind of event that we don't care about. + continue; + } + + preg_match_all("/([a-zA-Z]+)=\'([^'\\|]*(?:\\|.[^'\\|]*)*)\'/s", $line, $matches); + $attributes = array(); + if (isset($matches[1])) { + for ($i = 0; $i < count($matches[1]); $i++) { + $attributes[$matches[1][$i]] = $matches[2][$i]; + } + } + + $userdata = ''; + if (idx($attributes, 'message')) { + $userdata .= idx($attributes, 'message'); + } + if (idx($attributes, 'details')) { + $userdata .= + str_replace("|n", "\n", + str_replace("|r", "\r", + str_replace("|'", "'", + str_replace("||", "|", + idx($attributes, 'details'))))); + } + + $result = new ArcanistUnitTestResult(); + $result->setName(idx($attributes, 'name')); + $result->setResult($result_code); + $result->setDuration(idx($attributes, 'duration') / 1000); + $result->setUserData($userdata); + $results[] = $result; + + if ($this->renderer) { + echo $this->renderer->renderUnitResult($result); + } + } + } + } + + if ($future_iter !== null) { + break; + } + } + + return $results; + } + + public function shouldEchoTestResults() { + return !$this->renderer; + } + +}