Python sqlparse.tokens方法代碼示例

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def parse_partial_identifier(word):
    """Attempt to parse a (partially typed) word as an identifier

    word may include a schema qualification, like `schema_name.partial_name`
    or `schema_name.` There may also be unclosed quotation marks, like
    `"schema`, or `schema."partial_name`

    :param word: string representing a (partially complete) identifier
    :return: sqlparse.sql.Identifier, or None
    """

    p = sqlparse.parse(word)[0]
    n_tok = len(p.tokens)
    if n_tok == 1 and isinstance(p.tokens[0], Identifier):
        return p.tokens[0]
    elif p.token_next_by(m=(Error, '"'))[1]:
        # An unmatched double quote, e.g. '"foo', 'foo."', or 'foo."bar'
        # Close the double quote, then reparse
        return parse_partial_identifier(word + '"')
    else:
        return None 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def build_comparison(token):
    assert type(token) is S.Comparison

    m = M.Comparison()
    for tok in remove_whitespace(token.tokens):
        LOG.debug("  %s %s", tok, type(tok))
        if type(tok) is S.Parenthesis:
            subtokens = remove_whitespace(tok.tokens)
            subquery = tokens_to_sqla(subtokens[1:-1])
            if not m.left:
                m.left = subquery
            else:
                m.right = subquery
        else:
            m = sql_literal_to_model(tok, m)
            if not m:
                raise Exception("[BUG] Failed to convert %s to model" % tok)

    return m 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def test_simple(self):
        from cStringIO import StringIO

        stream = StringIO("SELECT 1; SELECT 2;")
        lex = lexer.Lexer()

        tokens = lex.get_tokens(stream)
        self.assertEqual(len(list(tokens)), 9)

        stream.seek(0)
        lex.bufsize = 4
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9)

        stream.seek(0)
        lex.bufsize = len(stream.getvalue())
        tokens = list(lex.get_tokens(stream))
        self.assertEqual(len(tokens), 9) 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def _validate_comparison(cls, tokens):
        base_error_string = "Invalid comparison clause"
        if len(tokens) != 3:
            raise MlflowException("{}. Expected 3 tokens found {}".format(base_error_string,
                                                                          len(tokens)),
                                  error_code=INVALID_PARAMETER_VALUE)
        if not isinstance(tokens[0], Identifier):
            raise MlflowException("{}. Expected 'Identifier' found '{}'".format(base_error_string,
                                                                                str(tokens[0])),
                                  error_code=INVALID_PARAMETER_VALUE)
        if not isinstance(tokens[1], Token) and tokens[1].ttype != TokenType.Operator.Comparison:
            raise MlflowException("{}. Expected comparison found '{}'".format(base_error_string,
                                                                              str(tokens[1])),
                                  error_code=INVALID_PARAMETER_VALUE)
        if not isinstance(tokens[2], Token) and \
                (tokens[2].ttype not in cls.STRING_VALUE_TYPES.union(cls.NUMERIC_VALUE_TYPES) or
                 isinstance(tokens[2], Identifier)):
            raise MlflowException("{}. Expected value token found '{}'".format(base_error_string,
                                                                               str(tokens[2])),
                                  error_code=INVALID_PARAMETER_VALUE) 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def _validate_order_by_and_generate_token(cls, order_by):
        try:
            parsed = sqlparse.parse(order_by)
        except Exception:
            raise MlflowException(f"Error on parsing order_by clause '{order_by}'",
                                  error_code=INVALID_PARAMETER_VALUE)
        if len(parsed) != 1 or not isinstance(parsed[0], Statement):
            raise MlflowException(f"Invalid order_by clause '{order_by}'. Could not be parsed.",
                                  error_code=INVALID_PARAMETER_VALUE)
        statement = parsed[0]
        if len(statement.tokens) == 1 and isinstance(statement[0], Identifier):
            token_value = statement.tokens[0].value
        elif len(statement.tokens) == 1 and \
                statement.tokens[0].match(ttype=TokenType.Keyword,
                                          values=[cls.ORDER_BY_KEY_TIMESTAMP]):
            token_value = cls.ORDER_BY_KEY_TIMESTAMP
        elif statement.tokens[0].match(ttype=TokenType.Keyword,
                                       values=[cls.ORDER_BY_KEY_TIMESTAMP])\
                and all([token.is_whitespace for token in statement.tokens[1:-1]])\
                and statement.tokens[-1].ttype == TokenType.Keyword.Order:
            token_value = cls.ORDER_BY_KEY_TIMESTAMP + ' ' + statement.tokens[-1].value
        else:
            raise MlflowException(f"Invalid order_by clause '{order_by}'. Could not be parsed.",
                                  error_code=INVALID_PARAMETER_VALUE)
        return token_value 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def parse_partial_identifier(word):
    """Attempt to parse a (partially typed) word as an identifier

    word may include a schema qualification, like `schema_name.partial_name`
    or `schema_name.` There may also be unclosed quotation marks, like
    `"schema`, or `schema."partial_name`

    :param word: string representing a (partially complete) identifier
    :return: sqlparse.sql.Identifier, or None
    """

    p = sqlparse.parse(word)[0]
    n_tok = len(p.tokens)
    if n_tok == 1 and isinstance(p.tokens[0], Identifier):
        return p.tokens[0]
    if p.token_next_by(m=(Error, '"'))[1]:
        # An unmatched double quote, e.g. '"foo', 'foo."', or 'foo."bar'
        # Close the double quote, then reparse
        return parse_partial_identifier(word + '"')
    return None 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def _extract_limit_from_query(statement: TokenList) -> Optional[int]:
    """
    Extract limit clause from SQL statement.

    :param statement: SQL statement
    :return: Limit extracted from query, None if no limit present in statement
    """
    idx, _ = statement.token_next_by(m=(Keyword, "LIMIT"))
    if idx is not None:
        _, token = statement.token_next(idx=idx)
        if token:
            if isinstance(token, IdentifierList):
                # In case of "LIMIT <offset>, <limit>", find comma and extract
                # first succeeding non-whitespace token
                idx, _ = token.token_next_by(m=(sqlparse.tokens.Punctuation, ","))
                _, token = token.token_next(idx=idx)
            if token and token.ttype == sqlparse.tokens.Literal.Number.Integer:
                return int(token.value)
    return None 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def _process_tokenlist(self, token_list: TokenList) -> None:
        """
        Add table names to table set

        :param token_list: TokenList to be processed
        """
        # exclude subselects
        if "(" not in str(token_list):
            table = self._get_table(token_list)
            if table and not table.table.startswith(CTE_PREFIX):
                self._tables.add(table)
            return

        # store aliases
        if token_list.has_alias():
            self._alias_names.add(token_list.get_alias())

        # some aliases are not parsed properly
        if token_list.tokens[0].ttype == Name:
            self._alias_names.add(token_list.tokens[0].value)
        self._extract_from_token(token_list) 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def get_query_tokens(query: str) -> List[sqlparse.sql.Token]:
    """
    :type query str
    :rtype: list[sqlparse.sql.Token]
    """
    query = preprocess_query(query)
    parsed = sqlparse.parse(query)

    # handle empty queries (#12)
    if not parsed:
        return []

    tokens = TokenList(parsed[0].tokens).flatten()
    # print([(token.value, token.ttype) for token in tokens])

    return [token for token in tokens if token.ttype is not Whitespace] 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def is_subselect(parsed):
    if not parsed.is_group():
        return False
    for item in parsed.tokens:
        if item.ttype is DML and item.value.upper() == 'SELECT':
            return True
    return False 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def find_prev_keyword(sql, n_skip=0):
    """ Find the last sql keyword in an SQL statement

    Returns the value of the last keyword, and the text of the query with
    everything after the last keyword stripped
    """
    if not sql.strip():
        return None, ""

    parsed = sqlparse.parse(sql)[0]
    flattened = list(parsed.flatten())
    flattened = flattened[: len(flattened) - n_skip]

    logical_operators = ("AND", "OR", "NOT", "BETWEEN")

    for t in reversed(flattened):
        if t.value == "(" or (
            t.is_keyword and (t.value.upper() not in logical_operators)
        ):
            # Find the location of token t in the original parsed statement
            # We can't use parsed.token_index(t) because t may be a child token
            # inside a TokenList, in which case token_index throws an error
            # Minimal example:
            #   p = sqlparse.parse('select * from foo where bar')
            #   t = list(p.flatten())[-3]  # The "Where" token
            #   p.token_index(t)  # Throws ValueError: not in list
            idx = flattened.index(t)

            # Combine the string values of all tokens in the original list
            # up to and including the target keyword token t, to produce a
            # query string with everything after the keyword token removed
            text = "".join(tok.value for tok in flattened[: idx + 1])
            return t, text

    return None, ""


# Postgresql dollar quote signs look like `$$` or `$tag$` 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def is_subselect(parsed):
    if not parsed.is_group:
        return False
    for item in parsed.tokens:
        if item.ttype is DML and item.value.upper() in (
            "SELECT",
            "INSERT",
            "UPDATE",
            "CREATE",
            "DELETE",
        ):
            return True
    return False 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def _identifier_is_function(identifier):
    return any(isinstance(t, Function) for t in identifier.tokens) 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def remove_whitespace(tokens):
    return [x for x in tokens if not x.is_whitespace] 

# 需要導入模塊: import sqlparse [as 別名]
# 或者: from sqlparse import tokens [as 別名]
def to_sqla(sql):
    sql = sql.strip()
    if not sql:
        raise Exception("Empty SQL string provided")

    tokens = sqlparse.parse(sql)[0].tokens
    tokens = remove_whitespace(tokens)
    return tokens_to_sqla(tokens).render()