Python parser.Parser类代码示例

def analyzeJSCodesFinerBlock(script, display=False):
  try:
    t1 = time.time()
    parser = Parser()
    script = script.strip()
    if script.startswith('<!--') and script.endswith('-->'):
      script = script[4: -3]
    tree = parser.parse(script)
    visitor = MyVisitor( display)
    visitor.visit(tree, 0)
    if len(visitor.first_level_seq) != len(visitor.scripts):
      print >>sys.stderr, "error parsing script: scripts and seqs length inconsistent "+script[:100]
      return None, None
    t2 = time.time()
    total_time = t2 - t1
    total_len = float(len(script))
    try:
      portion = [len(x)/total_len for x in visitor.scripts]
      for i in range(len(portion)):
        t = total_time * portion[i]
        print "AST_TIME: %f %d" %(t, len(visitor.scripts[i]))    
    except:
      pass
    return visitor.first_level_seq, visitor.scripts
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || [START]"+script[:100]+"[END]"
    return None, None

 def find_first_match(test_file_content, pos):
   test_file_content = TestMethodMatcher.UnitTest.insert_mark(test_file_content, pos)
   if not test_file_content:
     return None
   parser = Parser()
   tree = parser.parse(test_file_content)
   return TestMethodMatcher.UnitTest.get_test_name(tree)

def extract_glow_lib():
    runjs = norm_path('untrusted/run.js')
    parser = JSParser()

    with open(runjs) as f:
        tree = parser.parse(f.read())

    for node in nodevisitor.visit(tree):
        if (isinstance(node, ast.Assign) and
            isinstance(node.left, ast.DotAccessor) and
            node.left.identifier.value == 'glowscript_libraries' and
            isinstance(node.right, ast.Object)):
                break
    else:
        print('Parsing {} failed'.format(runjs))
        exit(-1)

    return preproc_lib_path({
        prop.left.value:
            [
                eval(lib.value)
                for lib in prop.right.items
                if isinstance(lib, ast.String)
            ]
            for prop in node.right.properties
    })

    def extract(self, node):
        '''
        Given a BeautifulSoup representation of an HTML document,
        return a list of all code snippets in that document.
        '''

        node_code = None
        code_snippets = []

        # Attempt to parse content for a code node as JavaScript.
        # Mark the content as a code snippet if it is parsed successfully.
        # Skip nodes with nothing but whitespace content.
        if type(node) is Tag and node.name in self.TAGS:
            if node.text.strip() != '':
                try:
                    js_parser = JavaScriptParser()
                    js_parser.parse(node.text)
                except (SyntaxError, TypeError, AttributeError):
                    logger.debug("Code content could not be parsed as JavaScript.")
                else:
                    node_code = node.text
                    code_snippets.append(node_code)

        # If this node did not contain valid code, then visit all children
        # and check them for code.
        if node_code is None and hasattr(node, 'children'):
            for child_element in node.children:
                code_snippets.extend(self.extract(child_element))

        return code_snippets

def load_data():
    # Read list names (toplists.js)
    f = open("./data/masterbugtable.js", "r")
    tmp = f.read()
    f.close()
    masterbugtable = json.loads(tmp[tmp.index("{") :])
    # find and process latest test data
    f = open("./data/testing/index.json", "r")
    tmp = f.read()
    f.close()
    test_file_index = json.loads(tmp)
    f = open("./data/testing/%s" % test_file_index[-1], "r")
    test_data = {}
    test_reader = csv.reader(f)
    for line in test_reader:
        test_data[str(line[0])] = {"bug": line[0], "test_date": line[1], "ua": line[2], "test_state": line[3]}
    f.close()
    f = open("./data/sitedata.js", "r")
    parser = Parser()
    tree = parser.parse(f.read())
    f.close()
    return {
        "masterbugtable": masterbugtable,
        "test_result_files": test_file_index,
        "test_results": test_data,
        "tests_parsed": tree,
    }

def minify(text, mangle=False, mangle_toplevel=False):
    parser = Parser()
    tree = parser.parse(text)
    if mangle:
        mangler.mangle(tree, toplevel=mangle_toplevel)
    minified = ECMAMinifier().visit(tree)
    return minified

 def _parse_succeeds(self, text):
     try:
         parser = JsParser()
         parser.parse(text)
     except (SyntaxError, TypeError):
         return False
     else:
         return True

    def case(self, case):
        parser_a = Parser()
        result_a = parser_a.parse(case)

        parser_b = Parser()
        result_b = parser_b.parse(case)

        self.assertEqual(result_a, result_b)

 def test_func(self):
     parser = Parser()
     tree = parser.parse(input)
     mangle(tree, toplevel=True)
     self.assertMultiLineEqual(
         textwrap.dedent(tree.to_ecma()).strip(),
         textwrap.dedent(expected).strip()
         )

def whileExtract(s):
    '''Extracts all the while loops in the script. '''
    l = []
    parser = Parser()
    tree = parser.parse(s)
    for node in nodevisitor.visit(tree):
        if isinstance(node, ast.While):
            l+=[node.to_ecma()]
    return l

def _parse_redirect_to_security_challenge_script(script: str) -> str:
    """ Parses the script which redirects us to security challenge page and gets that URL. """
    parser = Parser()
    tree = parser.parse(script)
    nodes = [node for node in nodevisitor.visit(tree) if isinstance(node, ast.Assign)]
    for node in nodevisitor.visit(tree):
        if isinstance(node, ast.Assign) and hasattr(node, 'left') and isinstance(node.left, ast.DotAccessor):
            children = node.left.children()
            if len(children) == 2 and children[0].value == 'window' and children[1].value == 'location':
                return node.right.value.strip('\'"')

 def test_bug_no_semicolon_at_the_end_of_block_plus_newline_at_eof(self):
     # https://github.com/rspivak/slimit/issues/3
     text = textwrap.dedent("""
     function add(x, y) {
       return x + y;
     }
     """)
     parser = Parser()
     tree = parser.parse(text)
     self.assertTrue(bool(tree.children()))

def analyzeJSCodes(script, display=False):
  try:
    parser = Parser()
    tree = parser.parse(script)
    visitor = MyVisitor( display)
    visitor.visit(tree, 0)
    #print "first_level_seq: %d" %len(visitor.first_level_seq)
    return visitor.node_order_list
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || "+script
    return None

 def _test_function_expression(self):
     text = """
     if (true) {
       function() {
         foo;
         location = 'http://anywhere.com';
       }
     }
     """
     parser = Parser()
     parser.parse(text)

def rewriteJSCodes(script, display=False):
  try:
    parser = Parser()
    tree = parser.parse(script)
    visitor = RewriteVisitor( display)
    visitor.visit(tree, 0)
    x = ECMAVisitor().visit(tree)
    print x
    #print tree.to_ecma()
    #print "first_level_seq: %d" %len(visitor.first_level_seq)
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || "+script