Python Tree.load方法代码示例

# 需要导入模块: from tree import Tree [as 别名]
# 或者: from tree.Tree import load [as 别名]
def gen_simple_trees(criteria, stats):
    """ Generate simplified parse trees from articles matching the criteria """
    for a in Article.articles(criteria):
        if not a.root_domain or "raduneyti" in a.root_domain:
            # Skip ministry websites due to amount of chaff found there
            continue
        tree = Tree(url = a.url, authority = a.authority)
        # Note the parse timestamp
        stats["parsed"] = a.parsed
        tree.load(a.tree)
        for ix, stree in tree.simple_trees():
            yield stree, tree.score(ix), tree.length(ix)

# 需要导入模块: from tree import Tree [as 别名]
# 或者: from tree.Tree import load [as 别名]
    def go_single(self, url):
        """ Single article processor that will be called by a process within a
            multiprocessing pool """

        print("Processing article {0}".format(url))
        sys.stdout.flush()

        # If first article within a new process, import the processor modules
        if self.pmodules is None:
            self.pmodules = [
                importlib.import_module(modname) for modname in self.processors
            ]

        # Load the article
        with closing(self._db.session) as session:

            try:
                article = session.query(Article).filter_by(url=url).one_or_none()

                if article is None:
                    print("Article not found in scraper database")
                else:
                    if article.tree and article.tokens:
                        tree = Tree(url, article.authority)
                        tree.load(article.tree)

                        token_container = TokenContainer(article.tokens, url)

                        # Run all processors in turn
                        for p in self.pmodules:
                            if p.PROCESSOR_TYPE == "tree":
                                tree.process(session, p)
                            elif p.PROCESSOR_TYPE == "token":
                                token_container.process(session, p)

                    # Mark the article as being processed
                    article.processed = datetime.utcnow()

                # So far, so good: commit to the database
                session.commit()

            except Exception as e:
                # If an exception occurred, roll back the transaction
                session.rollback()
                print(
                    "Exception in article {0}, transaction rolled back\nException: {1}".format(
                        url, e
                    )
                )
                raise

        sys.stdout.flush()

# 需要导入模块: from tree import Tree [as 别名]
# 或者: from tree.Tree import load [as 别名]

#.........这里部分代码省略.........
                elif t[0] == TOK.P_BEGIN:
                    pass
                elif t[0] == TOK.P_END:
                    pass
                else:
                    sent.append(t)

        result = dict(num_sent=num_sent, num_parsed_sent=num_parsed_sent)
        return result, trees

    def parse(self, toklist, result):
        """ Parse the token list as a query, returning True if valid """

        self._tree = None  # Erase previous tree, if any
        self._error = None  # Erase previous error, if any
        self._qtype = None  # Erase previous query type, if any
        self._key = None
        self._toklist = None

        parse_result, trees = Query._parse(toklist)

        if not trees:
            # No parse at all
            self.set_error("E_NO_TREES")
            return False

        result.update(parse_result)

        if result["num_sent"] != 1:
            # Queries must be one sentence
            self.set_error("E_MULTIPLE_SENTENCES")
            return False
        if result["num_parsed_sent"] != 1:
            # Unable to parse the single sentence
            self.set_error("E_NO_PARSE")
            return False
        if 1 not in trees:
            # No sentence number 1
            self.set_error("E_NO_FIRST_SENTENCE")
            return False
        # Looks good
        # Store the resulting parsed query as a tree
        tree_string = "S1\n" + trees[1]
        # print("Query tree:\n{0}".format(tree_string))
        self._tree = Tree()
        self._tree.load(tree_string)
        # Store the token list
        self._toklist = toklist
        return True

    def execute(self):
        """ Execute the query contained in the previously parsed tree;
            return True if successful """
        if self._tree is None:
            self.set_error("E_QUERY_NOT_PARSED")
            return False

        self._error = None
        self._qtype = None
        # Process the tree, which has only one sentence
        self._tree.process(self._session, _THIS_MODULE, query=self)

        return self._error is None

    def set_qtype(self, qtype):
        """ Set the query type ('Person', 'Title', 'Company', 'Entity'...) """
        self._qtype = qtype

    def set_answer(self, answer):
        """ Set the answer to the query """
        self._answer = answer

    def set_key(self, key):
        """ Set the query key, i.e. the term or string used to execute the query """
        # This is for instance a person name in nominative case
        self._key = key

    def set_error(self, error):
        """ Set an error result """
        self._error = error

    def qtype(self):
        """ Return the query type """
        return self._qtype

    def answer(self):
        """ Return the query answer """
        return self._answer

    def key(self):
        """ Return the query key """
        return self._key

    def token_list(self):
        """ Return the token list for the query """
        return self._toklist

    def error(self):
        """ Return the query error, if any """
        return self._error