Changeset View
Changeset View
Standalone View
Standalone View
src/unit/engine/XUnitTestEngine.php
- This file was copied to src/unit/engine/ArcanistXUnitTestEngine.php.
<?php | <?php | ||||
/** | /** | ||||
* Uses xUnit (http://xunit.codeplex.com/) to test C# code. | |||||
* | |||||
* Assumes that when modifying a file with a path like `SomeAssembly/MyFile.cs`, | |||||
* that the test assembly that verifies the functionality of `SomeAssembly` is | |||||
* located at `SomeAssembly.Tests`. | |||||
* | |||||
* @concrete-extensible | * @concrete-extensible | ||||
* @deprecated | |||||
*/ | */ | ||||
class XUnitTestEngine extends ArcanistUnitTestEngine { | class XUnitTestEngine extends ArcanistXUnitTestEngine { | ||||
protected $runtimeEngine; | |||||
protected $buildEngine; | |||||
protected $testEngine; | |||||
protected $projectRoot; | |||||
protected $xunitHintPath; | |||||
protected $discoveryRules; | |||||
/** | |||||
* This test engine supports running all tests. | |||||
*/ | |||||
protected function supportsRunAllTests() { | |||||
return true; | |||||
} | |||||
/** | |||||
* Determines what executables and test paths to use. Between platforms this | |||||
* also changes whether the test engine is run under .NET or Mono. It also | |||||
* ensures that all of the required binaries are available for the tests to | |||||
* run successfully. | |||||
* | |||||
* @return void | |||||
*/ | |||||
protected function loadEnvironment() { | |||||
$this->projectRoot = $this->getWorkingCopy()->getProjectRoot(); | |||||
// Determine build engine. | |||||
if (Filesystem::binaryExists('msbuild')) { | |||||
$this->buildEngine = 'msbuild'; | |||||
} else if (Filesystem::binaryExists('xbuild')) { | |||||
$this->buildEngine = 'xbuild'; | |||||
} else { | |||||
throw new Exception( | |||||
pht( | |||||
'Unable to find %s or %s in %s!', | |||||
'msbuild', | |||||
'xbuild', | |||||
'PATH')); | |||||
} | |||||
// Determine runtime engine (.NET or Mono). | |||||
if (phutil_is_windows()) { | |||||
$this->runtimeEngine = ''; | |||||
} else if (Filesystem::binaryExists('mono')) { | |||||
$this->runtimeEngine = Filesystem::resolveBinary('mono'); | |||||
} else { | |||||
throw new Exception( | |||||
pht('Unable to find Mono and you are not on Windows!')); | |||||
} | |||||
// Read the discovery rules. | |||||
$this->discoveryRules = | |||||
$this->getConfigurationManager()->getConfigFromAnySource( | |||||
'unit.csharp.discovery'); | |||||
if ($this->discoveryRules === null) { | |||||
throw new Exception( | |||||
pht( | |||||
'You must configure discovery rules to map C# files '. | |||||
'back to test projects (`%s` in %s).', | |||||
'unit.csharp.discovery', | |||||
'.arcconfig')); | |||||
} | |||||
// Determine xUnit test runner path. | |||||
if ($this->xunitHintPath === null) { | |||||
$this->xunitHintPath = | |||||
$this->getConfigurationManager()->getConfigFromAnySource( | |||||
'unit.csharp.xunit.binary'); | |||||
} | |||||
$xunit = $this->projectRoot.DIRECTORY_SEPARATOR.$this->xunitHintPath; | |||||
if (file_exists($xunit) && $this->xunitHintPath !== null) { | |||||
$this->testEngine = Filesystem::resolvePath($xunit); | |||||
} else if (Filesystem::binaryExists('xunit.console.clr4.exe')) { | |||||
$this->testEngine = 'xunit.console.clr4.exe'; | |||||
} else { | |||||
throw new Exception( | |||||
pht( | |||||
"Unable to locate xUnit console runner. Configure ". | |||||
"it with the `%s' option in %s.", | |||||
'unit.csharp.xunit.binary', | |||||
'.arcconfig')); | |||||
} | |||||
} | |||||
/** | |||||
* Main entry point for the test engine. Determines what assemblies to build | |||||
* and test based on the files that have changed. | |||||
* | |||||
* @return array Array of test results. | |||||
*/ | |||||
public function run() { | public function run() { | ||||
phutil_deprecated( | |||||
$this->loadEnvironment(); | __CLASS__, | ||||
pht( | |||||
if ($this->getRunAllTests()) { | 'You should use `%s` instead.', | ||||
$paths = id(new FileFinder($this->projectRoot))->find(); | 'ArcanistXUnitTestEngine')); | ||||
} else { | parent::run(); | ||||
$paths = $this->getPaths(); | |||||
} | |||||
return $this->runAllTests($this->mapPathsToResults($paths)); | |||||
} | |||||
/** | |||||
* Applies the discovery rules to the set of paths specified. | |||||
* | |||||
* @param array Array of paths. | |||||
* @return array Array of paths to test projects and assemblies. | |||||
*/ | |||||
public function mapPathsToResults(array $paths) { | |||||
$results = array(); | |||||
foreach ($this->discoveryRules as $regex => $targets) { | |||||
$regex = str_replace('/', '\\/', $regex); | |||||
foreach ($paths as $path) { | |||||
if (preg_match('/'.$regex.'/', $path) === 1) { | |||||
foreach ($targets as $target) { | |||||
// Index 0 is the test project (.csproj file) | |||||
// Index 1 is the output assembly (.dll file) | |||||
$project = preg_replace('/'.$regex.'/', $target[0], $path); | |||||
$project = $this->projectRoot.DIRECTORY_SEPARATOR.$project; | |||||
$assembly = preg_replace('/'.$regex.'/', $target[1], $path); | |||||
$assembly = $this->projectRoot.DIRECTORY_SEPARATOR.$assembly; | |||||
if (file_exists($project)) { | |||||
$project = Filesystem::resolvePath($project); | |||||
$assembly = Filesystem::resolvePath($assembly); | |||||
// Check to ensure uniqueness. | |||||
$exists = false; | |||||
foreach ($results as $existing) { | |||||
if ($existing['assembly'] === $assembly) { | |||||
$exists = true; | |||||
break; | |||||
} | |||||
} | |||||
if (!$exists) { | |||||
$results[] = array( | |||||
'project' => $project, | |||||
'assembly' => $assembly, | |||||
); | |||||
} | |||||
} | |||||
} | |||||
} | |||||
} | |||||
} | |||||
return $results; | |||||
} | |||||
/** | |||||
* Builds and runs the specified test assemblies. | |||||
* | |||||
* @param array Array of paths to test project files. | |||||
* @return array Array of test results. | |||||
*/ | |||||
public function runAllTests(array $test_projects) { | |||||
if (empty($test_projects)) { | |||||
return array(); | |||||
} | |||||
$results = array(); | |||||
$results[] = $this->generateProjects(); | |||||
if ($this->resultsContainFailures($results)) { | |||||
return array_mergev($results); | |||||
} | |||||
$results[] = $this->buildProjects($test_projects); | |||||
if ($this->resultsContainFailures($results)) { | |||||
return array_mergev($results); | |||||
} | |||||
$results[] = $this->testAssemblies($test_projects); | |||||
return array_mergev($results); | |||||
} | |||||
/** | |||||
* Determine whether or not a current set of results contains any failures. | |||||
* This is needed since we build the assemblies as part of the unit tests, but | |||||
* we can't run any of the unit tests if the build fails. | |||||
* | |||||
* @param array Array of results to check. | |||||
* @return bool If there are any failures in the results. | |||||
*/ | |||||
private function resultsContainFailures(array $results) { | |||||
$results = array_mergev($results); | |||||
foreach ($results as $result) { | |||||
if ($result->getResult() != ArcanistUnitTestResult::RESULT_PASS) { | |||||
return true; | |||||
} | |||||
} | |||||
return false; | |||||
} | |||||
/** | |||||
* If the `Build` directory exists, we assume that this is a multi-platform | |||||
* project that requires generation of C# project files. Because we want to | |||||
* test that the generation and subsequent build is whole, we need to | |||||
* regenerate any projects in case the developer has added files through an | |||||
* IDE and then forgotten to add them to the respective `.definitions` file. | |||||
* By regenerating the projects we ensure that any missing definition entries | |||||
* will cause the build to fail. | |||||
* | |||||
* @return array Array of test results. | |||||
*/ | |||||
private function generateProjects() { | |||||
// No "Build" directory; so skip generation of projects. | |||||
if (!is_dir(Filesystem::resolvePath($this->projectRoot.'/Build'))) { | |||||
return array(); | |||||
} | |||||
// No "Protobuild.exe" file; so skip generation of projects. | |||||
if (!is_file(Filesystem::resolvePath( | |||||
$this->projectRoot.'/Protobuild.exe'))) { | |||||
return array(); | |||||
} | |||||
// Work out what platform the user is building for already. | |||||
$platform = phutil_is_windows() ? 'Windows' : 'Linux'; | |||||
$files = Filesystem::listDirectory($this->projectRoot); | |||||
foreach ($files as $file) { | |||||
if (strtolower(substr($file, -4)) == '.sln') { | |||||
$parts = explode('.', $file); | |||||
$platform = $parts[count($parts) - 2]; | |||||
break; | |||||
} | |||||
} | |||||
$regenerate_start = microtime(true); | |||||
$regenerate_future = new ExecFuture( | |||||
'%C Protobuild.exe --resync %s', | |||||
$this->runtimeEngine, | |||||
$platform); | |||||
$regenerate_future->setCWD(Filesystem::resolvePath( | |||||
$this->projectRoot)); | |||||
$results = array(); | |||||
$result = new ArcanistUnitTestResult(); | |||||
$result->setName(pht('(regenerate projects for %s)', $platform)); | |||||
try { | |||||
$regenerate_future->resolvex(); | |||||
$result->setResult(ArcanistUnitTestResult::RESULT_PASS); | |||||
} catch(CommandException $exc) { | |||||
if ($exc->getError() > 1) { | |||||
throw $exc; | |||||
} | |||||
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL); | |||||
$result->setUserdata($exc->getStdout()); | |||||
} | |||||
$result->setDuration(microtime(true) - $regenerate_start); | |||||
$results[] = $result; | |||||
return $results; | |||||
} | |||||
/** | |||||
* Build the projects relevant for the specified test assemblies and return | |||||
* the results of the builds as test results. This build also passes the | |||||
* "SkipTestsOnBuild" parameter when building the projects, so that MSBuild | |||||
* conditionals can be used to prevent any tests running as part of the | |||||
* build itself (since the unit tester is about to run each of the tests | |||||
* individually). | |||||
* | |||||
* @param array Array of test assemblies. | |||||
* @return array Array of test results. | |||||
*/ | |||||
private function buildProjects(array $test_assemblies) { | |||||
$build_futures = array(); | |||||
$build_failed = false; | |||||
$build_start = microtime(true); | |||||
$results = array(); | |||||
foreach ($test_assemblies as $test_assembly) { | |||||
$build_future = new ExecFuture( | |||||
'%C %s', | |||||
$this->buildEngine, | |||||
'/p:SkipTestsOnBuild=True'); | |||||
$build_future->setCWD(Filesystem::resolvePath( | |||||
dirname($test_assembly['project']))); | |||||
$build_futures[$test_assembly['project']] = $build_future; | |||||
} | |||||
$iterator = id(new FutureIterator($build_futures))->limit(1); | |||||
foreach ($iterator as $test_assembly => $future) { | |||||
$result = new ArcanistUnitTestResult(); | |||||
$result->setName('(build) '.$test_assembly); | |||||
try { | |||||
$future->resolvex(); | |||||
$result->setResult(ArcanistUnitTestResult::RESULT_PASS); | |||||
} catch(CommandException $exc) { | |||||
if ($exc->getError() > 1) { | |||||
throw $exc; | |||||
} | |||||
$result->setResult(ArcanistUnitTestResult::RESULT_FAIL); | |||||
$result->setUserdata($exc->getStdout()); | |||||
$build_failed = true; | |||||
} | |||||
$result->setDuration(microtime(true) - $build_start); | |||||
$results[] = $result; | |||||
} | |||||
return $results; | |||||
} | |||||
/** | |||||
* Build the future for running a unit test. This can be overridden to enable | |||||
* support for code coverage via another tool. | |||||
* | |||||
* @param string Name of the test assembly. | |||||
* @return array The future, output filename and coverage filename | |||||
* stored in an array. | |||||
*/ | |||||
protected function buildTestFuture($test_assembly) { | |||||
// FIXME: Can't use TempFile here as xUnit doesn't like | |||||
// UNIX-style full paths. It sees the leading / as the | |||||
// start of an option flag, even when quoted. | |||||
$xunit_temp = Filesystem::readRandomCharacters(10).'.results.xml'; | |||||
if (file_exists($xunit_temp)) { | |||||
unlink($xunit_temp); | |||||
} | |||||
$future = new ExecFuture( | |||||
'%C %s /xml %s', | |||||
trim($this->runtimeEngine.' '.$this->testEngine), | |||||
$test_assembly, | |||||
$xunit_temp); | |||||
$folder = Filesystem::resolvePath($this->projectRoot); | |||||
$future->setCWD($folder); | |||||
$combined = $folder.'/'.$xunit_temp; | |||||
if (phutil_is_windows()) { | |||||
$combined = $folder.'\\'.$xunit_temp; | |||||
} | |||||
return array($future, $combined, null); | |||||
} | |||||
/** | |||||
* Run the xUnit test runner on each of the assemblies and parse the | |||||
* resulting XML. | |||||
* | |||||
* @param array Array of test assemblies. | |||||
* @return array Array of test results. | |||||
*/ | |||||
private function testAssemblies(array $test_assemblies) { | |||||
$results = array(); | |||||
// Build the futures for running the tests. | |||||
$futures = array(); | |||||
$outputs = array(); | |||||
$coverages = array(); | |||||
foreach ($test_assemblies as $test_assembly) { | |||||
list($future_r, $xunit_temp, $coverage) = | |||||
$this->buildTestFuture($test_assembly['assembly']); | |||||
$futures[$test_assembly['assembly']] = $future_r; | |||||
$outputs[$test_assembly['assembly']] = $xunit_temp; | |||||
$coverages[$test_assembly['assembly']] = $coverage; | |||||
} | |||||
// Run all of the tests. | |||||
$futures = id(new FutureIterator($futures)) | |||||
->limit(8); | |||||
foreach ($futures as $test_assembly => $future) { | |||||
list($err, $stdout, $stderr) = $future->resolve(); | |||||
if (file_exists($outputs[$test_assembly])) { | |||||
$result = $this->parseTestResult( | |||||
$outputs[$test_assembly], | |||||
$coverages[$test_assembly]); | |||||
$results[] = $result; | |||||
unlink($outputs[$test_assembly]); | |||||
} else { | |||||
// FIXME: There's a bug in Mono which causes a segmentation fault | |||||
// when xUnit.NET runs; this causes the XML file to not appear | |||||
// (depending on when the segmentation fault occurs). See | |||||
// https://bugzilla.xamarin.com/show_bug.cgi?id=16379 | |||||
// for more information. | |||||
// Since it's not possible for the user to correct this error, we | |||||
// ignore the fact the tests didn't run here. | |||||
} | |||||
} | |||||
return array_mergev($results); | |||||
} | |||||
/** | |||||
* Returns null for this implementation as xUnit does not support code | |||||
* coverage directly. Override this method in another class to provide code | |||||
* coverage information (also see @{class:CSharpToolsUnitEngine}). | |||||
* | |||||
* @param string The name of the coverage file if one was provided by | |||||
* `buildTestFuture`. | |||||
* @return array Code coverage results, or null. | |||||
*/ | |||||
protected function parseCoverageResult($coverage) { | |||||
return null; | |||||
} | |||||
/** | |||||
* Parses the test results from xUnit. | |||||
* | |||||
* @param string The name of the xUnit results file. | |||||
* @param string The name of the coverage file if one was provided by | |||||
* `buildTestFuture`. This is passed through to | |||||
* `parseCoverageResult`. | |||||
* @return array Test results. | |||||
*/ | |||||
private function parseTestResult($xunit_tmp, $coverage) { | |||||
$xunit_dom = new DOMDocument(); | |||||
$xunit_dom->loadXML(Filesystem::readFile($xunit_tmp)); | |||||
$results = array(); | |||||
$tests = $xunit_dom->getElementsByTagName('test'); | |||||
foreach ($tests as $test) { | |||||
$name = $test->getAttribute('name'); | |||||
$time = $test->getAttribute('time'); | |||||
$status = ArcanistUnitTestResult::RESULT_UNSOUND; | |||||
switch ($test->getAttribute('result')) { | |||||
case 'Pass': | |||||
$status = ArcanistUnitTestResult::RESULT_PASS; | |||||
break; | |||||
case 'Fail': | |||||
$status = ArcanistUnitTestResult::RESULT_FAIL; | |||||
break; | |||||
case 'Skip': | |||||
$status = ArcanistUnitTestResult::RESULT_SKIP; | |||||
break; | |||||
} | |||||
$userdata = ''; | |||||
$reason = $test->getElementsByTagName('reason'); | |||||
$failure = $test->getElementsByTagName('failure'); | |||||
if ($reason->length > 0 || $failure->length > 0) { | |||||
$node = ($reason->length > 0) ? $reason : $failure; | |||||
$message = $node->item(0)->getElementsByTagName('message'); | |||||
if ($message->length > 0) { | |||||
$userdata = $message->item(0)->nodeValue; | |||||
} | |||||
$stacktrace = $node->item(0)->getElementsByTagName('stack-trace'); | |||||
if ($stacktrace->length > 0) { | |||||
$userdata .= "\n".$stacktrace->item(0)->nodeValue; | |||||
} | |||||
} | |||||
$result = new ArcanistUnitTestResult(); | |||||
$result->setName($name); | |||||
$result->setResult($status); | |||||
$result->setDuration($time); | |||||
$result->setUserData($userdata); | |||||
if ($coverage != null) { | |||||
$result->setCoverage($this->parseCoverageResult($coverage)); | |||||
} | |||||
$results[] = $result; | |||||
} | |||||
return $results; | |||||
} | } | ||||
} | } |