public function Twig_Lexer::tokenize in Translation template extractor 7.3

Same name and namespace in other branches

1. 6.3 vendor/Twig/Lexer.php \Twig_Lexer::tokenize()
2. 7.2 vendor/Twig/Lexer.php \Twig_Lexer::tokenize()

Tokenizes a source code.

Parameters

string $code The source code:

string $filename A unique identifier for the source code:

Return value

Twig_TokenStream A token stream instance

Throws

Twig_Error_Syntax When the code is syntactically wrong

Overrides Twig_LexerInterface::tokenize

File

vendor/Twig/Lexer.php, line 78

Class

Twig_Lexer

Lexes a template string.

Code

public function tokenize($code, $filename = null) {
  if (function_exists('mb_internal_encoding') && (int) ini_get('mbstring.func_overload') & 2) {
    $mbEncoding = mb_internal_encoding();
    mb_internal_encoding('ASCII');
  }
  else {
    $mbEncoding = null;
  }
  $this->code = str_replace(array(
    "\r\n",
    "\r",
  ), "\n", $code);
  $this->filename = $filename;
  $this->cursor = 0;
  $this->lineno = 1;
  $this->end = strlen($this->code);
  $this->tokens = array();
  $this->state = self::STATE_DATA;
  $this->states = array();
  $this->brackets = array();
  $this->position = -1;

  // find all token starts in one go
  preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, PREG_OFFSET_CAPTURE);
  $this->positions = $matches;
  while ($this->cursor < $this->end) {

    // dispatch to the lexing functions depending
    // on the current state
    switch ($this->state) {
      case self::STATE_DATA:
        $this
          ->lexData();
        break;
      case self::STATE_BLOCK:
        $this
          ->lexBlock();
        break;
      case self::STATE_VAR:
        $this
          ->lexVar();
        break;
      case self::STATE_STRING:
        $this
          ->lexString();
        break;
      case self::STATE_INTERPOLATION:
        $this
          ->lexInterpolation();
        break;
    }
  }
  $this
    ->pushToken(Twig_Token::EOF_TYPE);
  if (!empty($this->brackets)) {
    list($expect, $lineno) = array_pop($this->brackets);
    throw new Twig_Error_Syntax(sprintf('Unclosed "%s".', $expect), $lineno, $this->filename);
  }
  if ($mbEncoding) {
    mb_internal_encoding($mbEncoding);
  }
  return new Twig_TokenStream($this->tokens, $this->filename);
}