look_ahead method of the lexer is now able to get any token.

        self._cursor = 0

        # The current processed token cache:

        self._current_token = None

        self._tokens_cache = []

    def get_token(self):

        '''

        Get the next token of the query and forward.

        '''

        if self._current_token is None:

            # Process the next token if the current token cache is empty and

            # return it (new token is not cached):

        if not self._tokens_cache:

            # If the cache is empty, get the next token and return it directly:

            return self._get_next_token()

        else:

            # If next token is already cached, return it and reset the cache:

            ctoken = self._current_token

            self._current_token = None

            return ctoken

            # If the cache is not empty, return its first token:

            return self._tokens_cache.pop(0)

    def look_ahead(self):

    def look_ahead(self, upto=1):

        '''

        Get the next token of the query but don't forward (multiple look_ahead

        calls will return the same token of get_token is not called).

        '''

        if self._current_token is None:

            # Process the next token if the cache is empty:

            self._current_token = self._get_next_token()

        if len(self._tokens_cache) < upto:

            # The cache is not populated with enough tokens, we process the

            # difference between the look ahead value and the current number

            # of cached tokens available:

            for _ in xrange(upto - len(self._tokens_cache)):

                self._tokens_cache.append(self._get_next_token())

        # And return the cached token:

        return self._current_token

        return self._tokens_cache[upto - 1]

    def _get_next_token(self):

        '''