Python asttokens.ASTTokens方法代码示例

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def _get_potential_mods(self):
        ip_inst = IPython.core.getipython.get_ipython()
        assert ip_inst, "SessionInfo() doesn't work outside of IPython"
        code = ip_inst.user_ns['In']
        # drop setup code if a reprex is running
        if os.environ.get('REPREX_RUNNING'):
            x = [
                i
                for i, j in enumerate(code) if re.search('REPREX_RUNNING', j)
            ]
            if x:
                code = code[(x[0] + 1):]
        scode = '\n'.join(code)
        tokes = asttokens.ASTTokens(scode, parse=True)

        def _get_one_mod(node):
            tnode = type(node).__name__
            if tnode == 'Import':
                return [i.name for i in node.names]
            elif tnode == 'ImportFrom':
                return [node.module]

        mlist = [_get_one_mod(i) for i in asttokens.util.walk(tokes.tree)]
        return {j for i in mlist if i is not None for j in i} 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def _get_input_chunks(code_str, si):
    tok = asttokens.ASTTokens(code_str, parse=True)

    ends = {statement.last_token.end[0] for statement in tok.tree.body}
    ends = list(sorted(ends))

    starts = [i + 1 for i in ends]
    starts.insert(0, 1)
    starts = starts[:-1]

    code_lines = code_str.splitlines()
    schunks = [code_lines[start - 1:end] for start, end in zip(starts, ends)]
    if si:
        schunks = schunks + [
            ['import reprexpy', 'print(reprexpy.SessionInfo())']
        ]
    return schunks 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def __init__(self, recomputed_values: Mapping[ast.AST, Any], variable_lookup: List[Mapping[str, Any]],
                 atok: asttokens.ASTTokens) -> None:
        """
        Initialize.

        :param recomputed_values: AST node of a condition function -> value associated with the node
        :param variable_lookup:
            list of lookup tables to look-up the values of the variables, sorted by precedence.
            The visitor needs it here to check whether we overrode a built-in variable (like ``id``).
        :param atok: parsed AST tree and tokens with additional positions in source code

        """
        self._recomputed_values = recomputed_values
        self._variable_lookup = variable_lookup
        self.reprs = dict()  # type: MutableMapping[str, str]
        self._atok = atok 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def print_timing(self):
    # pylint: disable=no-self-use
    # Test the implementation of asttokens.util.walk, which uses the same approach as
    # visit_tree(). This doesn't run as a normal unittest, but if you'd like to see timings, e.g.
    # after experimenting with the implementation, run this to see them:
    #
    #     nosetests -i print_timing -s tests.test_util
    #
    import timeit
    import textwrap
    setup = textwrap.dedent(
      '''
      import ast, asttokens
      source = "foo(bar(1 + 2), 'hello' + ', ' + 'world')"
      atok = asttokens.ASTTokens(source, parse=True)
      ''')
    print("ast", sorted(timeit.repeat(
      setup=setup, number=10000,
      stmt='len(list(ast.walk(atok.tree)))')))
    print("util", sorted(timeit.repeat(
      setup=setup, number=10000,
      stmt='len(list(asttokens.util.walk(atok.tree)))'))) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def test_walk_ast(self):
    atok = asttokens.ASTTokens(self.source, parse=True)

    def view(node):
      return "%s:%s" % (get_node_name(node), atok.get_text(node))

    scan = [view(n) for n in asttokens.util.walk(atok.tree)]
    self.assertEqual(scan, [
      "Module:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      "Expr:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      "Call:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      'Name:foo',
      'Call:bar(1 + 2)',
      'Name:bar',
      'BinOp:1 + 2',
      'Constant:1',
      'Constant:2',
      "BinOp:'hello' + ', ' + 'world'",
      "BinOp:'hello' + ', '",
      "Constant:'hello'",
      "Constant:', '",
      "Constant:'world'"
    ]) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def test_walk_astroid(self):
    atok = asttokens.ASTTokens(self.source, tree=astroid.builder.parse(self.source))

    def view(node):
      return "%s:%s" % (get_node_name(node), atok.get_text(node))

    scan = [view(n) for n in asttokens.util.walk(atok.tree)]
    self.assertEqual(scan, [
      "Module:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      "Expr:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      "Call:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
      'Name:foo',
      'Call:bar(1 + 2)',
      'Name:bar',
      'BinOp:1 + 2',
      'Const:1',
      'Const:2',
      "BinOp:'hello' + ', ' + 'world'",
      "BinOp:'hello' + ', '",
      "Const:'hello'",
      "Const:', '",
      "Const:'world'"
    ]) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def test_replace(self):
    self.assertEqual(asttokens.util.replace("this is a test", [(0, 4, "X"), (8, 9, "THE")]),
                     "X is THE test")
    self.assertEqual(asttokens.util.replace("this is a test", []), "this is a test")
    self.assertEqual(asttokens.util.replace("this is a test", [(7,7," NOT")]), "this is NOT a test")

    source = "foo(bar(1 + 2), 'hello' + ', ' + 'world')"
    atok = asttokens.ASTTokens(source, parse=True)
    names = [n for n in asttokens.util.walk(atok.tree) if isinstance(n, ast.Name)]
    strings = [n for n in asttokens.util.walk(atok.tree) if isinstance(n, ast.Str)]
    repl1 = [atok.get_text_range(n) + ('TEST',) for n in names]
    repl2 = [atok.get_text_range(n) + ('val',) for n in strings]
    self.assertEqual(asttokens.util.replace(source, repl1 + repl2),
                     "TEST(TEST(1 + 2), val + val + val)")
    self.assertEqual(asttokens.util.replace(source, repl2 + repl1),
                     "TEST(TEST(1 + 2), val + val + val)") 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def print_timing(self):
    # Print the timing of mark_tokens(). This doesn't normally run as a unittest, but if you'd like
    # to see timings, e.g. while optimizing the implementation, run this to see them:
    #
    #     nosetests -m print_timing -s tests.test_mark_tokens tests.test_astroid
    #
    # pylint: disable=no-self-use
    import timeit
    print("mark_tokens", sorted(timeit.repeat(
      setup=textwrap.dedent(
        '''
        import ast, asttokens
        source = "foo(bar(1 + 2), 'hello' + ', ' + 'world')"
        atok = asttokens.ASTTokens(source)
        tree = ast.parse(source)
        '''),
      stmt='atok.mark_tokens(tree)',
      repeat=3,
      number=1000))) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def test_expression(self):
        try:
            self.st = asttokens.ASTTokens(self.code, parse=True)  # ast.parse(self.code)
        except Exception as e:
            self.errors.append(f'Error executing expression: {e}')
            return
        if not self.st._tree or not self.st._tree.body:
            return
        self.test_correct_comparing(self.st._tree.body) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def test_expect_token():
  atok = asttokens.ASTTokens("a", parse=True)
  tok = atok.tokens[0]
  with pytest.raises(ValueError):
    asttokens.util.expect_token(tok, token.OP) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def create_asttokens(source):
    builder = astroid.builder.AstroidBuilder()
    tree = builder.string_build(source)
    return ASTTokens(source, tree=tree) 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def annotate_python_ast(
    parsed_ast: python_ast.AST,
    source_code: str,
    modification_offsets: Optional[ModificationOffsets] = None,
    source_id: int = 0,
) -> python_ast.AST:
    """
    Annotate and optimize a Python AST in preparation conversion to a Vyper AST.

    Parameters
    ----------
    parsed_ast : AST
        The AST to be annotated and optimized.
    source_code : str
        The originating source code of the AST.
    modification_offsets : dict, optional
        A mapping of class names to their original class types.

    Returns
    -------
        The annotated and optimized AST.
    """

    asttokens.ASTTokens(source_code, tree=parsed_ast)
    AnnotatingVisitor(source_code, modification_offsets, source_id).visit(parsed_ast)

    return parsed_ast 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def parse(self) -> Graph:
        """Parse the function into a Myia graph."""
        src0 = inspect.getsource(self.function)
        src = textwrap.dedent(src0)
        # We need col_offset to compensate for the dedent
        self.col_offset = len(src0.split("\n")[0]) - len(src.split("\n")[0])
        tree = asttokens.ASTTokens(src, parse=True).tree
        function_def = tree.body[0]
        assert isinstance(function_def, ast.FunctionDef)
        main_block, _finalize = self._create_function(None, function_def)
        _finalize()
        for node in dfs(main_block.graph.return_, succ_deeper):
            if node.is_constant_graph():
                if node.value.return_ is None:
                    current = node.value.debug
                    while getattr(current, "about", None) is not None:
                        current = getattr(current.about, "debug", None)
                    raise MyiaSyntaxError(
                        "Function doesn't return a value in all cases",
                        current.location,
                    )

        diff_cache = self.write_cache - self.read_cache
        if diff_cache:
            for _, varname, node in diff_cache:
                if varname != "_":
                    warnings.warn(
                        MyiaDisconnectedCodeWarning(
                            f"{varname} is not used "
                            + f"and will therefore not be computed",
                            node.debug.location,
                        )
                    )

        self.write_cache = OrderedSet()
        self.read_cache = OrderedSet()
        return main_block.graph 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def compile(self, source, filename, flags=0):
        traced_file = super(BirdsEye, self).compile(source, filename, flags)
        traced_file.tokens = ASTTokens(source, tree=traced_file.root)
        return traced_file 

# 需要导入模块: import asttokens [as 别名]
# 或者: from asttokens import ASTTokens [as 别名]
def parse(self, code):
        res = asttokens.ASTTokens(code, parse=True)
        return res, res.tree

    # add methods for retrieving parser outputs --------------------------