Python ElementHelper.element_text_content方法代码示例

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def clean_body(clusters, doctree, title_node=None, debug = False):
    #filter user comments and all link records
    clusters = filter_cluster(clusters)
    if len(clusters) == 0:
        title = ElementHelper.element_text_content(title_node)
        return title, title

    #choose cluster which has most texts
    maxCluster = get_biggest_cluster(clusters)
    nodes = clusters[maxCluster]

    # m = {}
    # m[maxCluster] = nodes
    # filter_cluster(m)


    #get all children of max cluster record
    allnodes = []
    for node in nodes:
        children = ElementHelper.get_children(node)
        allnodes.extend(children)

    #sort max cluster nodes by its preorder index
    allnodes.sort(lambda p,q:ElementHelper.get_element_preorder_num(p)-ElementHelper.get_element_preorder_num(q))
    # debug=True
    if debug:
        print 'maxCluster: %s' % maxCluster
        for n in allnodes:
            print ElementHelper.get_xpath_by_element(n, doctree), n.get(py)
    s = ElementHelper.get_element_preorder_num(allnodes[0])
    t = ElementHelper.get_element_preorder_num(allnodes[-1])

    #correct start position by title node
    title_text = ''
    #====================================================================
    #在实验时借助title纠正正文文本的起始位置可以提高recall,但是实际过程中不需要
    if title_node is not None:
        index = ElementHelper.get_element_preorder_num(title_node)
        if index < s:
            s = index #ToDo:add 2016/03/09
            title_text = ElementHelper.element_text_content(title_node)
    #====================================================================

    body = ElementHelper.get_body(doctree)
    # remove nodes which not belong to main text
    set_text_mark(body, s, t)
    remove_nontext_element(body)

    return body, title_text

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def get_title_util(body, title_text):
    if len(title_text) < 2:
        return None
    #1.计算节点文本与title_text的longest commen lenght
    candidate_nodes = []
    for tag in TITLE_TAG:
        nodes = ElementHelper.get_element_by_tag(body, tag)
        if nodes is None or len(nodes)<1: continue

        nodes = [node for node in nodes if is_possible_title_tag(node)]
        candidate_nodes.extend(nodes)

    mapping = {}
    for node in candidate_nodes:
        node_text = ElementHelper.element_text_content(node)
        # if len(node_text)==0 or len(node_text)>len(title_text): continue  #

        if len(node_text)==0: continue

        llength = longest_common_length(node_text, title_text)
        if llength >= 1:
            mapping[node] = llength

    if len(mapping)==0: return None
    #2.选择长度最大的作为title节点，如果存在多个最大的，选择最靠前的作为
    #title节点
    sorted_nodes = [node for node, _ in sorted(mapping.items(), key=lambda x:x[1], reverse=True)]
    max_len = mapping[sorted_nodes[0]]

    candidates = [node for node in sorted_nodes if mapping[node]==max_len]
    if len(candidate_nodes)==1:
        return sorted_nodes[0]
    else:
        candidates.sort(cmp=lambda x,y: ElementHelper.get_element_preorder_num(x)- ElementHelper.get_element_preorder_num(y), reverse=False)
        return candidates[0]

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def get_aricle_cetd(doctree):
    cetd_parse(doctree)
    body = ElementHelper.get_body(doctree)
    # ElementHelper.print_element(body)
    CleanTreeByMark(body)
    RemoveAttribute(body)
    return ElementHelper.element_text_content(body)

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def get_article_wish(clusters, doctree, title_node=None, debug = False):
    #filter
    clusters = filter_cluster(clusters)

    if len(clusters) == 0:
        return ElementHelper.element_text_content(title_node)

    maxCluster = get_biggest_cluster(clusters)

    nodes = clusters[maxCluster]

    m = {}
    m[maxCluster] = nodes
    filter_cluster(m)

    #============================================
    if debug:
    # for test
        print 'maxCluster: %s' %maxCluster
        for n in nodes:
            print ElementHelper.get_xpath_by_element(n, doctree)

    allnodes = []

    for node in nodes:
        children = ElementHelper.get_children(node)
        allnodes.extend(children)

    allnodes.sort(lambda p,q:ElementHelper.get_element_preorder_num(p)-ElementHelper.get_element_preorder_num(q))
    s = ElementHelper.get_element_preorder_num(allnodes[0])
    t = ElementHelper.get_element_preorder_num(allnodes[-1])
    title_text = ''
    if title_node is not None:
        index = ElementHelper.get_element_preorder_num(title_node)
        if index < s:
            s = index
            title_text = ElementHelper.element_text_content(title_node)

    body = ElementHelper.get_body(doctree)

    set_text_mark(body, s, t)

    remove_nontext_element(body)

    # ElementHelper.print_element(body)

    return ElementHelper.element_text_content_list(body), title_text

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
    def get_headline_content_in_cleaned_body(body):
        headlin_tag = ['h1', 'h2', 'h3', 'h4']

        headline_contents = [ElementHelper.element_text_content(node)
                             for node in ElementHelper.get_elements_by_tagnames(body, headlin_tag)
                             if not ElementHelper.is_element_content_none(node)]

        return '\n'.join(headline_contents)

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def ComputeDensitySum(element, ratio):
    density_sum, char_num_sum  = 0.0, 0
    _from, index, length = 0, 0, 0

    content = ElementHelper.element_text_content(element)
    if ElementHelper.is_element_has_child(element):
        for child in element:
            ComputeDensitySum(child, ratio)
        for child in element:
            density_sum += float(child.attrib.get(kg_text_density))
            char_num_sum += long(child.attrib.get(kg_char_num))

            #text before tag
            child_content = ElementHelper.element_text_content(child)
            index = -1
            if child_content != '':
                index = StringHelper.index_of(content, child_content, _from)

            if index > -1:
                length = index - _from
                if length > 0:
                    try:
                        tmp = length * qLn(1.0 * length) / qLn(qLn(ratio * length + qExp(1.0))) #此处的计算结果都为0
                        density_sum += tmp
                    except ZeroDivisionError:
                        pass
                _from = index + len(child_content)

        #text after tag
        length = len(ElementHelper.element_text_content(element)) - _from
        if length>0:
            try:
                density_sum += length * qLn(1.0 * length) / qLn(qLn(ratio * length + qExp(1.0)))
            except ZeroDivisionError:
                pass
    else:
        density_sum = float(element.attrib.get(kg_text_density))

    d2s_density_sum  = str(density_sum)
    element.set(kg_density_sum, d2s_density_sum)

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
    def get_article(doctree, debug = False):
        w = WISH()

        title = HtmlHelper.get_article_title_element(doctree)

        clusters = w.get_clustered_records(doctree)

        if debug:
            w.print_cluster_record(clusters, doctree)

        articles, title_text = get_article_wish(clusters, doctree, title, debug)
        if title_text is None or len(title_text)==0:
            title_text = ElementHelper.element_text_content(HtmlHelper.get_title(doctree))

        return articles, title_text

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def is_cluster_contain_user_comments(cluster):
    """ identify whether element or its children contain comment content, only consider <a> tag
    1.each node in cluster, at least has 3 children
    2.there is at least one <a> tag has same text
    """
    # can not identify
    if len(cluster) < 2: return False

    text_number_mapping = {}

    #at least have three children contain text
    for node in cluster:
        children = ElementHelper.get_children(node)
        link_nodes_contain_text = [n for n in children if is_link_node_with_text(n)]
        non_link_nodes_contain_text = [n for n in children if is_none_link_node_with_text(n)]

        if len(link_nodes_contain_text)<3: return False
        if len(non_link_nodes_contain_text)<2: return False

        for n in link_nodes_contain_text:
            text = ElementHelper.element_text_content(n)
            if text in text_number_mapping:
                text_number_mapping[text] += 1
            else:
                text_number_mapping[text] = 1
    #去除标点符号，出数字，空的文本
    tmp = copy.deepcopy(text_number_mapping)
    for text in tmp:
        if len(text)==0 or StringHelper.is_digits(text) :
            del text_number_mapping[text]

    text_number = text_number_mapping.values()

    # FOR TEST
    # for text, number in node_text_mapping.items():
    #     print text,':', number

    text_number_counter = collections.Counter(text_number).most_common()

    for number, counter in text_number_counter:
        if number > 1 and number==len(cluster) and counter>=2: #ToDo 2016/03/08  old:counter>=2 --> new:counter>=1
            print 'find comment!'
            return True
    return False

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def get_link_word_by_pair(docstring, base_url, supervisior=None, encoding='utf-8'):
    """ collect urls from
    :param html:
    :param base_url:
    :return:
    """
    h = HtmlHelper()
    doctree = h.create_doc(docstring, encoding)
    if isinstance(base_url, unicode):
        base_url = base_url.encode('utf-8')
    a_tags = ElementHelper.get_elements_by_tagnames(doctree, 'a')

    for a in a_tags:

        link = a.get('href',None)

        link = a.get('href',None)
        link = m_strip(link)

        if link is None or len(link)<2:continue
        if link[0]=='#': continue #link to itself
        link = normalize_url(link, base_url)

        #if url in non visited set
        if is_url_visited(link, unvisited_url_set):
            continue

        # if not should_collect_url(link, base_url):
        #     continue


        link_item = UrlItem()
        link_item['parent_url'] = base_url
        link_item['url'] = link
        link_item['anchor_text'] = ElementHelper.element_text_content(a).encode('utf-8')
        link_item['neigb_text'] = ''
        if supervisior is not None:
            link_item['label'], link_item['interestness'] = supervisior.predict(link_item['anchor_text'])
        else:
            link_item['label'], link_item['interestness'] = '1', 0.0  #1为负样本

        yield link_item

# 需要导入模块: from util import ElementHelper [as 别名]
# 或者: from util.ElementHelper import element_text_content [as 别名]
def CountChar(element):
    char_num = len(ElementHelper.element_text_content(element))
    l2s_char_num = str(char_num)
    element.set(kg_char_num, l2s_char_num)
    for child in element:
        CountChar(child)