abstract class AbstractLexer in Zircon Profile 8

Base class for writing simple lexers, i.e. for creating small DSLs.

@since 2.0 @author Guilherme Blanco <[e-mail]> @author Jonathan Wage <[e-mail]> @author Roman Borschel <[e-mail]>

Hierarchy

• class \Doctrine\Common\Lexer\AbstractLexer

3 files declare their use of AbstractLexer

DocLexer.php in vendor/doctrine/annotations/lib/Doctrine/Common/Annotations/DocLexer.php

EmailLexer.php in vendor/egulias/email-validator/src/Egulias/EmailValidator/EmailLexer.php

Lexer.php in vendor/doctrine/common/lib/Doctrine/Common/Lexer.php

File

vendor/doctrine/lexer/lib/Doctrine/Common/Lexer/AbstractLexer.php, line 30

Namespace

Doctrine\Common\Lexer

abstract class AbstractLexer {

  /**
   * Lexer original input string.
   *
   * @var string
   */
  private $input;

  /**
   * Array of scanned tokens.
   *
   * Each token is an associative array containing three items:
   *  - 'value'    : the string value of the token in the input string
   *  - 'type'     : the type of the token (identifier, numeric, string, input
   *                 parameter, none)
   *  - 'position' : the position of the token in the input string
   *
   * @var array
   */
  private $tokens = array();

  /**
   * Current lexer position in input string.
   *
   * @var integer
   */
  private $position = 0;

  /**
   * Current peek of current lexer position.
   *
   * @var integer
   */
  private $peek = 0;

  /**
   * The next token in the input.
   *
   * @var array
   */
  public $lookahead;

  /**
   * The last matched/seen token.
   *
   * @var array
   */
  public $token;

  /**
   * Sets the input data to be tokenized.
   *
   * The Lexer is immediately reset and the new input tokenized.
   * Any unprocessed tokens from any previous input are lost.
   *
   * @param string $input The input to be tokenized.
   *
   * @return void
   */
  public function setInput($input) {
    $this->input = $input;
    $this->tokens = array();
    $this
      ->reset();
    $this
      ->scan($input);
  }

  /**
   * Resets the lexer.
   *
   * @return void
   */
  public function reset() {
    $this->lookahead = null;
    $this->token = null;
    $this->peek = 0;
    $this->position = 0;
  }

  /**
   * Resets the peek pointer to 0.
   *
   * @return void
   */
  public function resetPeek() {
    $this->peek = 0;
  }

  /**
   * Resets the lexer position on the input to the given position.
   *
   * @param integer $position Position to place the lexical scanner.
   *
   * @return void
   */
  public function resetPosition($position = 0) {
    $this->position = $position;
  }

  /**
   * Retrieve the original lexer's input until a given position.
   *
   * @param integer $position
   *
   * @return string
   */
  public function getInputUntilPosition($position) {
    return substr($this->input, 0, $position);
  }

  /**
   * Checks whether a given token matches the current lookahead.
   *
   * @param integer|string $token
   *
   * @return boolean
   */
  public function isNextToken($token) {
    return null !== $this->lookahead && $this->lookahead['type'] === $token;
  }

  /**
   * Checks whether any of the given tokens matches the current lookahead.
   *
   * @param array $tokens
   *
   * @return boolean
   */
  public function isNextTokenAny(array $tokens) {
    return null !== $this->lookahead && in_array($this->lookahead['type'], $tokens, true);
  }

  /**
   * Moves to the next token in the input string.
   *
   * @return boolean
   */
  public function moveNext() {
    $this->peek = 0;
    $this->token = $this->lookahead;
    $this->lookahead = isset($this->tokens[$this->position]) ? $this->tokens[$this->position++] : null;
    return $this->lookahead !== null;
  }

  /**
   * Tells the lexer to skip input tokens until it sees a token with the given value.
   *
   * @param string $type The token type to skip until.
   *
   * @return void
   */
  public function skipUntil($type) {
    while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
      $this
        ->moveNext();
    }
  }

  /**
   * Checks if given value is identical to the given token.
   *
   * @param mixed   $value
   * @param integer $token
   *
   * @return boolean
   */
  public function isA($value, $token) {
    return $this
      ->getType($value) === $token;
  }

  /**
   * Moves the lookahead token forward.
   *
   * @return array|null The next token or NULL if there are no more tokens ahead.
   */
  public function peek() {
    if (isset($this->tokens[$this->position + $this->peek])) {
      return $this->tokens[$this->position + $this->peek++];
    }
    else {
      return null;
    }
  }

  /**
   * Peeks at the next token, returns it and immediately resets the peek.
   *
   * @return array|null The next token or NULL if there are no more tokens ahead.
   */
  public function glimpse() {
    $peek = $this
      ->peek();
    $this->peek = 0;
    return $peek;
  }

  /**
   * Scans the input string for tokens.
   *
   * @param string $input A query string.
   *
   * @return void
   */
  protected function scan($input) {
    static $regex;
    if (!isset($regex)) {
      $regex = sprintf('/(%s)|%s/%s', implode(')|(', $this
        ->getCatchablePatterns()), implode('|', $this
        ->getNonCatchablePatterns()), $this
        ->getModifiers());
    }
    $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
    $matches = preg_split($regex, $input, -1, $flags);
    foreach ($matches as $match) {

      // Must remain before 'value' assignment since it can change content
      $type = $this
        ->getType($match[0]);
      $this->tokens[] = array(
        'value' => $match[0],
        'type' => $type,
        'position' => $match[1],
      );
    }
  }

  /**
   * Gets the literal for a given token.
   *
   * @param integer $token
   *
   * @return string
   */
  public function getLiteral($token) {
    $className = get_class($this);
    $reflClass = new \ReflectionClass($className);
    $constants = $reflClass
      ->getConstants();
    foreach ($constants as $name => $value) {
      if ($value === $token) {
        return $className . '::' . $name;
      }
    }
    return $token;
  }

  /**
   * Regex modifiers
   *
   * @return string
   */
  protected function getModifiers() {
    return 'i';
  }

  /**
   * Lexical catchable patterns.
   *
   * @return array
   */
  protected abstract function getCatchablePatterns();

  /**
   * Lexical non-catchable patterns.
   *
   * @return array
   */
  protected abstract function getNonCatchablePatterns();

  /**
   * Retrieve token type. Also processes the token value if necessary.
   *
   * @param string $value
   *
   * @return integer
   */
  protected abstract function getType(&$value);

}

Members

Name	Modifiers	Type	Description	Overrides
AbstractLexer::$input	private	property	Lexer original input string.
AbstractLexer::$lookahead	public	property	The next token in the input.
AbstractLexer::$peek	private	property	Current peek of current lexer position.
AbstractLexer::$position	private	property	Current lexer position in input string.
AbstractLexer::$token	public	property	The last matched/seen token.
AbstractLexer::$tokens	private	property	Array of scanned tokens.
AbstractLexer::getCatchablePatterns	abstract protected	function	Lexical catchable patterns.	2
AbstractLexer::getInputUntilPosition	public	function	Retrieve the original lexer's input until a given position.
AbstractLexer::getLiteral	public	function	Gets the literal for a given token.
AbstractLexer::getModifiers	protected	function	Regex modifiers	1
AbstractLexer::getNonCatchablePatterns	abstract protected	function	Lexical non-catchable patterns.	2
AbstractLexer::getType	abstract protected	function	Retrieve token type. Also processes the token value if necessary.	2
AbstractLexer::glimpse	public	function	Peeks at the next token, returns it and immediately resets the peek.
AbstractLexer::isA	public	function	Checks if given value is identical to the given token.
AbstractLexer::isNextToken	public	function	Checks whether a given token matches the current lookahead.
AbstractLexer::isNextTokenAny	public	function	Checks whether any of the given tokens matches the current lookahead.
AbstractLexer::moveNext	public	function	Moves to the next token in the input string.	1
AbstractLexer::peek	public	function	Moves the lookahead token forward.
AbstractLexer::reset	public	function	Resets the lexer.	1
AbstractLexer::resetPeek	public	function	Resets the peek pointer to 0.
AbstractLexer::resetPosition	public	function	Resets the lexer position on the input to the given position.
AbstractLexer::scan	protected	function	Scans the input string for tokens.
AbstractLexer::setInput	public	function	Sets the input data to be tokenized.
AbstractLexer::skipUntil	public	function	Tells the lexer to skip input tokens until it sees a token with the given value.