Changeset View
Changeset View
Standalone View
Standalone View
src/search/__tests__/PhutilSearchStemmerTestCase.php
Show All 29 Lines | $tests = array( | ||||
// Stems should be normalized. | // Stems should be normalized. | ||||
'DOG' => 'dog', | 'DOG' => 'dog', | ||||
// If stemming would bring a token under 3 characters, it should not | // If stemming would bring a token under 3 characters, it should not | ||||
// be stemmed. | // be stemmed. | ||||
'dns' => 'dns', | 'dns' => 'dns', | ||||
'nis' => 'nis', | 'nis' => 'nis', | ||||
// Complex tokens with internal punctuation should be left untouched; | |||||
// these are usually things like domain names, API calls, informal tags, | |||||
// etc. | |||||
'apples' => 'appl', | |||||
'bananas' => 'banana', | |||||
'apples_bananas' => 'apples_bananas', | |||||
'apples_bananas.apples_bananas' => 'apples_bananas.apples_bananas', | |||||
); | ); | ||||
$stemmer = new PhutilSearchStemmer(); | $stemmer = new PhutilSearchStemmer(); | ||||
foreach ($tests as $input => $expect) { | foreach ($tests as $input => $expect) { | ||||
$stem = $stemmer->stemToken($input); | $stem = $stemmer->stemToken($input); | ||||
$this->assertEqual( | $this->assertEqual( | ||||
$expect, | $expect, | ||||
$stem, | $stem, | ||||
pht('Token stem of "%s".', $input)); | pht('Token stem of "%s".', $input)); | ||||
} | } | ||||
} | } | ||||
public function testStemDocuments() { | public function testStemDocuments() { | ||||
$tests = array( | $tests = array( | ||||
'The wild boar meandered erratically.' => | 'The wild boar meandered erratically.' => | ||||
'the wild boar meander errat', | 'the wild boar meander errat', | ||||
'Fool me onc, shame on you. Fool me twice, shame on me.' => | 'Fool me onc, shame on you. Fool me twice, shame on me.' => | ||||
'fool onc shame you twice', | 'fool onc shame you twice', | ||||
'Fireball is a seventh-level spell which deals 2d16 points of damage '. | 'Fireball is a seventh-level spell which deals 2d16 points of damage '. | ||||
'in a 1-meter radius around a target.' => | 'in a 1-meter radius around a target.' => | ||||
'firebal seventh level spell which deal 2d16 point damag meter '. | 'firebal seventh level spell which deal 2d16 point damag meter '. | ||||
'radiu around target', | 'radiu around target', | ||||
'apples-bananas' => 'appl banana', | |||||
'apples_bananas' => 'apples_bananas', | |||||
'apples.bananas' => 'apples.bananas', | |||||
'oddly-proportioned' => 'oddli proport', | |||||
); | ); | ||||
$stemmer = new PhutilSearchStemmer(); | $stemmer = new PhutilSearchStemmer(); | ||||
foreach ($tests as $input => $expect) { | foreach ($tests as $input => $expect) { | ||||
$stem = $stemmer->stemCorpus($input); | $stem = $stemmer->stemCorpus($input); | ||||
$this->assertEqual( | $this->assertEqual( | ||||
$expect, | $expect, | ||||
$stem, | $stem, | ||||
pht('Corpus stem of: %s', $input)); | pht('Corpus stem of: %s', $input)); | ||||
} | } | ||||
} | } | ||||
} | } |