rustemmer.module in Russian stemming 6

Russian stemming algorith provided by Dr Martin Porter

http://snowball.tartarus.org/algorithms/russian/stemmer.html

Algorith implementation in PHP provided by Dmitry Koterov (dklab.ru): http://forum.dklab.ru/php/advises/HeuristicWithoutTheDictionaryExtractio...

Initial implementation adopted for Drupal by Algenon ([e-mail])

File

rustemmer.module

<?php

/**
 * @file
 * Russian stemming algorith provided by Dr Martin Porter
 *
 * http://snowball.tartarus.org/algorithms/russian/stemmer.html
 *
 * Algorith implementation in PHP provided by Dmitry Koterov (dklab.ru):
 * http://forum.dklab.ru/php/advises/HeuristicWithoutTheDictionaryExtractionOfARootFromRussianWord.html
 *
 * Initial implementation adopted for Drupal by Algenon ([e-mail])
 */
define('RUSTEMMER_CHARS', '_0-9a-zA-ZабвгдежзийклмнопрстуфхцчшщьыъэюяАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЬЫЪЭЮЯ');

/**
 * Implements hook_help().
 */
function rustemmer_help($path, $arg) {
  switch ($path) {
    case 'admin/help#rustemmer':
      return t('Improves search of Russian words by using the stemming algorithm for Russian language.');
  }
}

/**
 * Implements hook_search_preprocess().
 */
function rustemmer_search_preprocess($text) {
  $words = $text;
  $words = str_replace('ё', 'е', $words);
  $words = str_replace('Ё', 'Е', $words);

  // Split words from noise and remove apostrophes.
  $words = preg_split('/([^' . RUSTEMMER_CHARS . ']+)/u', str_replace("'", '', $words), -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
  $stemmer = new RussianStemmer();

  // Process each word.
  $odd = TRUE;
  foreach ($words as $k => $word) {
    if ($odd) {
      $words[$k] = $stemmer
        ->stem_word($word);
    }
    $odd = !$odd;
  }

  // Put it all back together.
  return implode('', $words);
}

/**
 * Implements hook_search().
 */
function rustemmer_search($op = 'search', $keys = NULL) {
  if ($op == 'admin') {
    $form = array();
    $form['rustemmer_stemcaching'] = array(
      '#type' => 'checkbox',
      '#title' => t('Stem caching'),
      '#default_value' => variable_get('rustemmer_stemcaching', 0),
      '#description' => t('Apply stem caching while performing search phrase stemming. Warning: This could lead to additional memory consumption while indexing.'),
    );
    return $form;
  }
}

/**
 * Implements Russian Stemming algorithm.
 */
class RussianStemmer {
  var $VERSION = "0.02";
  var $Stem_Caching = 0;
  var $Stem_Cache = array();
  var $VOWEL = '/аеиоуыэюя/u';
  var $PERFECTIVEGROUND = '/((ив|ивши|ившись|ыв|ывши|ывшись)|((?<=[ая])(в|вши|вшись)))$/u';
  var $REFLEXIVE = '/(с[яь])$/u';
  var $ADJECTIVE = '/(ее|ие|ые|ое|ими|ыми|ей|ий|ый|ой|ем|им|ым|ом|его|ого|ему|ому|их|ых|ую|юю|ая|яя|ою|ею)$/u';
  var $PARTICIPLE = '/((ивш|ывш|ующ)|((?<=[ая])(ем|нн|вш|ющ|щ)))$/u';
  var $VERB = '/((ила|ыла|ена|ейте|уйте|ите|или|ыли|ей|уй|ил|ыл|им|ым|ен|ило|ыло|ено|ят|ует|уют|ит|ыт|ены|ить|ыть|ишь|ую|ю)|((?<=[ая])(ла|на|ете|йте|ли|й|л|ем|н|ло|но|ет|ют|ны|ть|ешь|нно)))$/u';
  var $NOUN = '/(а|ев|ов|ие|ье|е|иями|ями|ами|еи|ии|и|ией|ей|ой|ий|й|иям|ям|ием|ем|ам|ом|о|у|ах|иях|ях|ы|ь|ию|ью|ю|ия|ья|я)$/u';
  var $RVRE = '/^(.*?[аеиоуыэюя])(.*)$/u';
  var $DERIVATIONAL = '/[^аеиоуыэюя][аеиоуыэюя]+[^аеиоуыэюя]+[аеиоуыэюя].*(?<=о)сть?$/u';
  function __construct() {
    $this->Stem_Caching = variable_get('rustemmer_stemcaching', 0);
  }
  function s(&$s, $re, $to) {
    $orig = $s;
    $s = preg_replace($re, $to, $s);
    return $orig !== $s;
  }
  function m($s, $re) {
    return preg_match($re, $s);
  }
  function stem_word($word) {
    $word = drupal_strtolower($word);
    $word = str_replace('ё', 'е', $word);

    // Check against cache of stemmed words
    if ($this->Stem_Caching && isset($this->Stem_Cache[$word])) {
      return $this->Stem_Cache[$word];
    }
    $stem = $word;
    do {
      if (!preg_match($this->RVRE, $word, $p)) {
        break;
      }
      $start = $p[1];
      $RV = $p[2];
      if (!$RV) {
        break;
      }

      // Step 1
      if (!$this
        ->s($RV, $this->PERFECTIVEGROUND, '')) {
        $this
          ->s($RV, $this->REFLEXIVE, '');
        if ($this
          ->s($RV, $this->ADJECTIVE, '')) {
          $this
            ->s($RV, $this->PARTICIPLE, '');
        }
        else {
          if (!$this
            ->s($RV, $this->VERB, '')) {
            $this
              ->s($RV, $this->NOUN, '');
          }
        }
      }

      // Step 2
      $this
        ->s($RV, '/и$/u', '');

      // Step 3
      if ($this
        ->m($RV, $this->DERIVATIONAL)) {
        $this
          ->s($RV, '/ость?$/u', '');
      }

      // Step 4
      if (!$this
        ->s($RV, '/ь$/u', '')) {
        $this
          ->s($RV, '/ейше?/u', '');
        $this
          ->s($RV, '/нн$/u', 'н');
      }
      $stem = $start . $RV;
    } while (FALSE);
    if ($this->Stem_Caching) {
      $this->Stem_Cache[$word] = $stem;
    }
    return $stem;
  }

}

Functions

Name	Description
rustemmer_help	Implements hook_help().
rustemmer_search	Implements hook_search().
rustemmer_search_preprocess	Implements hook_search_preprocess().

Constants

Name	Description
RUSTEMMER_CHARS	@file Russian stemming algorith provided by Dr Martin Porter

Classes

Name	Description
RussianStemmer	Implements Russian Stemming algorithm.