function _transliteration_process in Transliteration 5

Same name and namespace in other branches

1. 6 transliteration.inc \_transliteration_process()
2. 7.3 transliteration.inc \_transliteration_process()

Based on Mediawiki's UtfNormal::quickIsNFCVerify().

See also

transliteration_get()

2 calls to _transliteration_process()

transliteration_clean_filename in ./transliteration.inc

Sanitize a file name.

transliteration_get in ./transliteration.module

Transliterate UTF-8 input to plain ASCII.

File

./transliteration.inc, line 30

Code

function _transliteration_process($string, $unknown = '?') {

  // Screen out some characters that eg won't be allowed in XML.
  $string = preg_replace('/[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1f]/', $unknown, $string);

  // ASCII is always valid NFC!
  // If we're only ever given plain ASCII, we can avoid the overhead
  // of initializing the decomposition tables by skipping out early.
  if (!preg_match('/[\\x80-\\xff]/', $string)) {
    return $string;
  }
  static $tailBytes;
  if (!isset($tailBytes)) {

    // Each UTF-8 head byte is followed by a certain
    // number of tail bytes.
    $tailBytes = array();
    for ($n = 0; $n < 256; $n++) {
      if ($n < 0xc0) {
        $remaining = 0;
      }
      elseif ($n < 0xe0) {
        $remaining = 1;
      }
      elseif ($n < 0xf0) {
        $remaining = 2;
      }
      elseif ($n < 0xf8) {
        $remaining = 3;
      }
      elseif ($n < 0xfc) {
        $remaining = 4;
      }
      elseif ($n < 0xfe) {
        $remaining = 5;
      }
      else {
        $remaining = 0;
      }
      $tailBytes[chr($n)] = $remaining;
    }
  }

  // Chop the text into pure-ASCII and non-ASCII areas;
  // large ASCII parts can be handled much more quickly.
  // Don't chop up Unicode areas for punctuation, though,
  // that wastes energy.
  preg_match_all('/[\\x00-\\x7f]+|[\\x80-\\xff][\\x00-\\x40\\x5b-\\x5f\\x7b-\\xff]*/', $string, $matches);
  $result = '';
  foreach ($matches[0] as $str) {
    if ($str[0] < "") {

      // ASCII chunk: guaranteed to be valid UTF-8
      // and in normal form C, so skip over it.
      $result .= $str;
      continue;
    }

    // We'll have to examine the chunk byte by byte to ensure
    // that it consists of valid UTF-8 sequences, and to see
    // if any of them might not be normalized.
    //
    // Since PHP is not the fastest language on earth, some of
    // this code is a little ugly with inner loop optimizations.
    $head = '';
    $chunk = strlen($str);
    $len = $chunk + 1;

    // Counting down is faster. I'm *so* sorry.
    for ($i = -1; --$len;) {
      $c = $str[++$i];
      if ($remaining = $tailBytes[$c]) {

        // UTF-8 head byte!
        $sequence = $head = $c;
        do {

          // Look for the defined number of tail bytes...
          if (--$len && ($c = $str[++$i]) >= "" && $c < "") {

            // Legal tail bytes are nice.
            $sequence .= $c;
          }
          else {
            if ($len == 0) {

              // Premature end of string!
              // Drop a replacement character into output to
              // represent the invalid UTF-8 sequence.
              $result .= $unknown;
              break 2;
            }
            else {

              // Illegal tail byte; abandon the sequence.
              $result .= $unknown;

              // Back up and reprocess this byte; it may itself
              // be a legal ASCII or UTF-8 sequence head.
              --$i;
              ++$len;
              continue 2;
            }
          }
        } while (--$remaining);
        $n = ord($head);
        if ($n <= 0xdf) {
          $ord = ($n - 192) * 64 + (ord($sequence[1]) - 128);
        }
        else {
          if ($n <= 0xef) {
            $ord = ($n - 224) * 4096 + (ord($sequence[1]) - 128) * 64 + (ord($sequence[2]) - 128);
          }
          else {
            if ($n <= 0xf7) {
              $ord = ($n - 240) * 262144 + (ord($sequence[1]) - 128) * 4096 + (ord($sequence[2]) - 128) * 64 + (ord($sequence[3]) - 128);
            }
            else {
              if ($n <= 0xfb) {
                $ord = ($n - 248) * 16777216 + (ord($sequence[1]) - 128) * 262144 + (ord($sequence[2]) - 128) * 4096 + (ord($sequence[3]) - 128) * 64 + (ord($sequence[4]) - 128);
              }
              else {
                if ($n <= 0xfd) {
                  $ord = ($n - 252) * 1073741824 + (ord($sequence[1]) - 128) * 16777216 + (ord($sequence[2]) - 128) * 262144 + (ord($sequence[3]) - 128) * 4096 + (ord($sequence[4]) - 128) * 64 + (ord($sequence[5]) - 128);
                }
              }
            }
          }
        }
        $result .= _transliteration_replace($ord, $unknown);
        $head = '';
      }
      elseif ($c < "") {

        // ASCII byte.
        $result .= $c;
        $head = '';
      }
      elseif ($c < "") {

        // Illegal tail bytes.
        if ($head == '') {
          $result .= $unknown;
        }
      }
      else {

        // Miscellaneous freaks.
        $result .= $unknown;
        $head = '';
      }
    }
  }
  return $result;
}