protected function SearchApiUnitTest::checkTokenizer in Search API 7

Tests the functionality of the "Tokenizer" processor.

1 call to SearchApiUnitTest::checkTokenizer()

SearchApiUnitTest::testUnits in ./search_api.test

Tests the functionality of several components of the module.

File

./search_api.test, line 982

Contains the SearchApiWebTest and the SearchApiUnitTest classes.

Class

SearchApiUnitTest

Class with unit tests testing small fragments of the Search API.

Code

protected function checkTokenizer() {
  $orig = 'Foo bar1 BaZ,  La-la-la.';
  $processed1 = array(
    array(
      'value' => 'Foo',
      'score' => 1,
    ),
    array(
      'value' => 'bar1',
      'score' => 1,
    ),
    array(
      'value' => 'BaZ',
      'score' => 1,
    ),
    array(
      'value' => 'Lalala',
      'score' => 1,
    ),
  );
  $processed2 = array(
    array(
      'value' => 'Foob',
      'score' => 1,
    ),
    array(
      'value' => 'r1B',
      'score' => 1,
    ),
    array(
      'value' => 'Z,L',
      'score' => 1,
    ),
    array(
      'value' => 'l',
      'score' => 1,
    ),
    array(
      'value' => 'l',
      'score' => 1,
    ),
    array(
      'value' => '.',
      'score' => 1,
    ),
  );
  $items = array(
    1 => array(
      'name' => array(
        'type' => 'text',
        'original_type' => 'text',
        'value' => $orig,
      ),
      'search_api_language' => array(
        'type' => 'string',
        'original_type' => 'string',
        'value' => LANGUAGE_NONE,
      ),
    ),
  );
  $processor = new SearchApiTokenizer($this->index, array(
    'fields' => array(
      'name' => 'name',
    ),
    'spaces' => '[^\\p{L}\\p{N}]',
    'ignorable' => '[-]',
  ));
  $tmp = $items;
  $processor
    ->preprocessIndexItems($tmp);
  $this
    ->assertEqual($tmp[1]['name']['value'], $processed1, 'Value was correctly tokenized with default settings.');
  $query = new SearchApiQuery($this->index, array(
    'parse mode' => 'direct',
  ));
  $query
    ->keys("foo \"bar-baz\" \n\t foobar1");
  $processor
    ->preprocessSearchQuery($query);
  $this
    ->assertEqual($query
    ->getKeys(), 'foo barbaz foobar1', 'Search keys were processed correctly.');
  $processor = new SearchApiTokenizer($this->index, array(
    'fields' => array(
      'name' => 'name',
    ),
    'spaces' => '[-a]',
    'ignorable' => '\\s',
  ));
  $tmp = $items;
  $processor
    ->preprocessIndexItems($tmp);
  $this
    ->assertEqual($tmp[1]['name']['value'], $processed2, 'Value was correctly tokenized with custom settings.');
  $query = new SearchApiQuery($this->index, array(
    'parse mode' => 'direct',
  ));
  $query
    ->keys("foo \"bar-baz\" \n\t foobar1");
  $processor
    ->preprocessSearchQuery($query);
  $this
    ->assertEqual($query
    ->getKeys(), 'foo"b r b z"foob r1', 'Search keys were processed correctly.');
}