Python ElementHelper.get_xpath_by_element方法代码示例

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import get_xpath_by_element [as 别名]
 def print_cluster_record(cls, clusters, doctree):
     ElementHelper.print_element(doctree)
     for cluster in clusters:
         if len(cluster)>1:
             print '===='*10
             nodes = clusters[cluster]
             for node in nodes:
                 print ElementHelper.get_xpath_by_element(node, doctree), node.get(py)

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import get_xpath_by_element [as 别名]
def clean_body(clusters, doctree, title_node=None, debug = False):
    #filter user comments and all link records
    clusters = filter_cluster(clusters)
    if len(clusters) == 0:
        title = ElementHelper.element_text_content(title_node)
        return title, title

    #choose cluster which has most texts
    maxCluster = get_biggest_cluster(clusters)
    nodes = clusters[maxCluster]

    # m = {}
    # m[maxCluster] = nodes
    # filter_cluster(m)


    #get all children of max cluster record
    allnodes = []
    for node in nodes:
        children = ElementHelper.get_children(node)
        allnodes.extend(children)

    #sort max cluster nodes by its preorder index
    allnodes.sort(lambda p,q:ElementHelper.get_element_preorder_num(p)-ElementHelper.get_element_preorder_num(q))
    # debug=True
    if debug:
        print 'maxCluster: %s' % maxCluster
        for n in allnodes:
            print ElementHelper.get_xpath_by_element(n, doctree), n.get(py)
    s = ElementHelper.get_element_preorder_num(allnodes[0])
    t = ElementHelper.get_element_preorder_num(allnodes[-1])

    #correct start position by title node
    title_text = ''
    #====================================================================
    #在实验时借助title纠正正文文本的起始位置可以提高recall,但是实际过程中不需要
    if title_node is not None:
        index = ElementHelper.get_element_preorder_num(title_node)
        if index < s:
            s = index #ToDo:add 2016/03/09
            title_text = ElementHelper.element_text_content(title_node)
    #====================================================================

    body = ElementHelper.get_body(doctree)
    # remove nodes which not belong to main text
    set_text_mark(body, s, t)
    remove_nontext_element(body)

    return body, title_text

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import get_xpath_by_element [as 别名]
    def find_first_sibling_record_node(cls, element, doctree):
        '''找到element所在区域的起始节点
        1.首选查看element的xpath下表，如果其下表<2，表示element左边没有兄弟节点了，直接返回element
        2.如果element的xpath下标大于=2,表示element左边有兄弟节点，那么先找到element的父亲（以便于访问element的兄弟节点，然后index=设置为element的下表-2
        查看parentt[index]是否是数据区域，如果是继续项左寻找，否则返回parent[index+1]
        '''
        parent = element.getparent()
        if len(parent)<2:
            return element

        element_xpath = ElementHelper.get_xpath_by_element(element, doctree)
        # print 'xpath: %s' %element_xpath
        element_last_index = StringHelper.get_digits(element_xpath.split('/')[-1])

        if element_last_index < 2:
            return element

        index = element_last_index - 2
        # print 'parent length:%d' %len(parent)
        while index >= 0:
            # print index
            if parent[index].get(kg_record_mark) == '1':
                index -= 1
            else:
                break
        return parent[index+1]

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import get_xpath_by_element [as 别名]
def get_article_wish(clusters, doctree, title_node=None, debug = False):
    #filter
    clusters = filter_cluster(clusters)

    if len(clusters) == 0:
        return ElementHelper.element_text_content(title_node)

    maxCluster = get_biggest_cluster(clusters)

    nodes = clusters[maxCluster]

    m = {}
    m[maxCluster] = nodes
    filter_cluster(m)

    #============================================
    if debug:
    # for test
        print 'maxCluster: %s' %maxCluster
        for n in nodes:
            print ElementHelper.get_xpath_by_element(n, doctree)

    allnodes = []

    for node in nodes:
        children = ElementHelper.get_children(node)
        allnodes.extend(children)

    allnodes.sort(lambda p,q:ElementHelper.get_element_preorder_num(p)-ElementHelper.get_element_preorder_num(q))
    s = ElementHelper.get_element_preorder_num(allnodes[0])
    t = ElementHelper.get_element_preorder_num(allnodes[-1])
    title_text = ''
    if title_node is not None:
        index = ElementHelper.get_element_preorder_num(title_node)
        if index < s:
            s = index
            title_text = ElementHelper.element_text_content(title_node)

    body = ElementHelper.get_body(doctree)

    set_text_mark(body, s, t)

    remove_nontext_element(body)

    # ElementHelper.print_element(body)

    return ElementHelper.element_text_content_list(body), title_text

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import get_xpath_by_element [as 别名]
    def find_first_sibling_record_node(cls, element, doctree):
        parent = element.getparent()
        if len(parent)<2:
            return element

        element_xpath = ElementHelper.get_xpath_by_element(element, doctree)
        # print 'xpath: %s' %element_xpath
        element_last_index = StringHelper.get_digits(element_xpath.split('/')[-1])

        if element_last_index < 2:
            return element

        index = element_last_index - 2
        # print 'parent length:%d' %len(parent)
        while index >= 0:
            # print index
            if parent[index].get(kg_record_mark) == '1':
                index -= 1
            else:
                break
        return parent[index+1]