Python ES_CLUSTER_FLOW1.get方法代码示例

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def get_user_influence(uid, date):
    date = str(date).replace("-","")
    index_name = pre_index + date
    try:
        bci_info = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        bci_info = {}
    result = {}
    for key in BCI_LIST:
        result[key] = bci_info.get(key, 0)

    user_index = result["user_index"]
    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "range":{
                        "user_index":{
                            "gt": user_index
                        }
                    }
                }
            }
        }
    }
    total_count = es_cluster.count(index=index_name, doc_type=influence_doctype)['count']
    order_count = es_cluster.count(index=index_name, doc_type=influence_doctype, body=query_body)['count']

    result["total_count"] = total_count
    result["order_count"] = order_count + 1

    return result

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def search_portrait_history_active_info(uid, date, index_name="copy_user_portrait", doctype="user"):
    # date.formate: 20130901
    date_list = time_series(date)

    try:
        result = es.get(index=index_name, doc_type=doctype, id=uid, _source=True)['_source']
    except NotFoundError:
        return "NotFound"
    except:
        return None
    
    date_max = {}
    for date_str in date_list:
        query_body = {
            'query':{
                'match_all':{}
                },
            'size': 1,
            'sort': [{date_str: {'order': 'desc'}}]
        }
        try:
            max_item = es.search(index=index_name, doc_type=doctype, body=query_body)['hits']['hits']
        except Exception, e:
            raise e
        date_max[date_str] = max_item[0]['_source'][date_str]

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def tag_vector(uid, date):
    date1 = str(date).replace('-', '')
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date
    result = []

    try:
        bci_result = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        tag = influence_tag["0"]
        result.append(tag)
        return result

    origin_retweeted = json.loads(bci_result["origin_weibo_retweeted_detail"])
    retweeted_retweeted = json.loads(bci_result["retweeted_weibo_retweeted_detail"])
    origin_comment = json.loads(bci_result["origin_weibo_comment_detail"])
    retweeted_comment = json.loads(bci_result["retweeted_weibo_comment_detail"])
    sum_retweeted = sum(origin_retweeted.values()) + sum(origin_comment.values())
    sum_comment = sum(retweeted_retweeted.values()) + sum(retweeted_comment.values())

    if sum_retweeted >= retweeted_threshold:
        if sum_comment >= comment_threshold:
            tag = influence_tag['3']
        else:
            tag = influence_tag['1']
    else:
        if sum_comment >= comment_threshold:
            tag = influence_tag['2']
        else:
            tag = influence_tag['4']
    result.append(tag)
    return result

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def influenced_detail(uid, date, style):
    date1 = str(date).replace("-", "")
    index_name = pre_index + date1
    # detail_text = {}
    style = int(style)
    try:
        user_info = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        result = {}
        return result
    origin_retweetd = json.loads(user_info["origin_weibo_retweeted_top"])
    origin_comment = json.loads(user_info["origin_weibo_comment_top"])
    retweeted_retweeted = json.loads(user_info["retweeted_weibo_retweeted_top"])
    retweeted_comment = json.loads(user_info["retweeted_weibo_comment_top"])

    if style == 0:
        detail_text = get_text(origin_retweetd, date, user_info, style)
    elif style == 1:
        detail_text = get_text(origin_comment, date, user_info, style)
    elif style == 2:
        detail_text = get_text(retweeted_retweeted, date, user_info, style)
    else:
        detail_text = get_text(retweeted_comment, date, user_info, style)
    # detail_text["origin_retweeted"] = get_text(origin_retweetd, date)
    # detail_text["origin_comment"] = get_text(origin_comment, date)
    # detail_text["retweeted_retweeted"] = get_text(retweeted_retweeted, date)
    # detail_text["retweeted_comment"] = get_text(retweeted_comment, date)

    return detail_text

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def statistics_influence_people(uid, date, style):
    # output: different retweeted and comment, uids' domain distribution, topic distribution, registeration geo distribution
    results = {}  # retwweted weibo people and comment weibo people
    date1 = str(date).replace("-", "")
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date

    try:
        bci_result = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        bci_result = []
        return results
    origin_retweeted_mid = []  # origin weibo mid
    retweeted_retweeted_mid = []  # retweeted weibo mid
    origin_comment_mid = []
    retweeted_comment_mid = []
    origin_retweeted = json.loads(bci_result["origin_weibo_retweeted_detail"])
    retweeted_retweeted = json.loads(bci_result["retweeted_weibo_retweeted_detail"])
    origin_comment = json.loads(bci_result["origin_weibo_comment_detail"])
    retweeted_comment = json.loads(bci_result["retweeted_weibo_comment_detail"])
    retweeted_total_number = sum(origin_retweeted.values()) + sum(retweeted_retweeted.values())
    comment_total_number = sum(origin_comment.values()) + sum(retweeted_comment.values())
    if origin_retweeted:
        origin_retweeted_mid = filter_mid(origin_retweeted)
    if retweeted_retweeted:
        retweeted_retweeted_mid = filter_mid(retweeted_retweeted)
    if origin_comment:
        origin_comment_mid = filter_mid(origin_comment)
    if retweeted_comment:
        retweeted_comment_mid = filter_mid(retweeted_comment)

    query_body = {"query": {"filtered": {"filter": {"bool": {"should": [], "must": []}}}}, "size": 10000}

    if int(style) == 0:  # retweeted
        retweeted_origin = []
        if retweeted_retweeted_mid:
            text_result = es.mget(
                index=index_flow_text, doc_type=flow_text_index_type, body={"ids": retweeted_retweeted_mid}
            )["docs"]
            for item in text_result:
                mid = item.get("source", {}).get("root_mid", "0")
                retweeted_origin.append(mid)
        retweeted_results = influenced_user_detail(uid, date, origin_retweeted_mid, retweeted_origin, 3)
        retweeted_results["total_number"] = retweeted_total_number
        results = retweeted_results
    else:
        retweeted_origin = []
        if retweeted_comment_mid:
            text_result = es.mget(
                index=index_flow_text, doc_type=flow_text_index_type, body={"ids": retweeted_comment_mid}
            )["docs"]
            for item in text_result:
                mid = item.get("source", {}).get("root_mid", "0")
                retweeted_origin.append(mid)
        comment_results = influenced_user_detail(uid, date, origin_comment_mid, retweeted_origin, 2)
        comment_results["total_number"] = comment_total_number
        results = comment_results

    return results

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def comment_on_influence(uid, date):
    date1 = str(date).replace('-', '')
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date
    result = []
    underline = []

    try:
        bci_result = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        description = CURRENT_INFLUENCE_CONCLUSION['0']
        result.append(description)
        return ([result, underline])

    user_index = bci_result['user_index']
    if user_index < CURRNET_INFLUENCE_THRESHOULD[0]:
        description = CURRENT_INFLUENCE_CONCLUSION['0']
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[0] and user_index < CURRNET_INFLUENCE_THRESHOULD[1]:
        description = CURRENT_INFLUENCE_CONCLUSION['1']
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[1] and user_index < CURRNET_INFLUENCE_THRESHOULD[2]:
        description = CURRENT_INFLUENCE_CONCLUSION['2']
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[2] and user_index < CURRNET_INFLUENCE_THRESHOULD[3]:
        description = CURRENT_INFLUENCE_CONCLUSION['3']
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[3] and user_index < CURRNET_INFLUENCE_THRESHOULD[4]:
        description = CURRENT_INFLUENCE_CONCLUSION['4']
    else:
        description = CURRENT_INFLUENCE_CONCLUSION['5']
    result.append(description)

    for i in range(4):
        if bci_result[INFLUENCE_TOTAL_LIST[i]] > INFLUENCE_TOTAL_THRESHOULD[i]:
            result.append(INFLUENCE_TOTAL_CONCLUSION[i])
            if bci_result[INFLUENCE_BRUST_LIST[i]] > INFLUENCE_BRUST_THRESHOULD[i]:
                result.append(INFLUENCE_BRUST_CONCLUSION[i])
                underline.append(UNDERLINE_CONCLUSION[i])
            else:
                result.append('')
                underline.append('')
        else:
            result.extend(['',''])
            underline.append('')

    return [result, underline]

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def comment_on_influence(uid, date):
    date1 = str(date).replace("-", "")
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date
    result = []
    underline = []

    try:
        bci_result = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        description = CURRENT_INFLUENCE_CONCLUSION["0"]
        result.append(description)
        return [result, underline]

    user_index = bci_result["user_index"]
    if user_index < CURRNET_INFLUENCE_THRESHOULD[0]:
        description = CURRENT_INFLUENCE_CONCLUSION["0"]
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[0] and user_index < CURRNET_INFLUENCE_THRESHOULD[1]:
        description = CURRENT_INFLUENCE_CONCLUSION["1"]
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[1] and user_index < CURRNET_INFLUENCE_THRESHOULD[2]:
        description = CURRENT_INFLUENCE_CONCLUSION["2"]
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[2] and user_index < CURRNET_INFLUENCE_THRESHOULD[3]:
        description = CURRENT_INFLUENCE_CONCLUSION["3"]
    elif user_index >= CURRNET_INFLUENCE_THRESHOULD[3] and user_index < CURRNET_INFLUENCE_THRESHOULD[4]:
        description = CURRENT_INFLUENCE_CONCLUSION["4"]
    else:
        description = CURRENT_INFLUENCE_CONCLUSION["5"]
    result.append(description)

    for i in range(4):
        if bci_result[INFLUENCE_TOTAL_LIST[i]] > INFLUENCE_TOTAL_THRESHOULD[i]:
            result.append(INFLUENCE_TOTAL_CONCLUSION[i])
            if bci_result[INFLUENCE_BRUST_LIST[i]] > INFLUENCE_BRUST_THRESHOULD[i]:
                result.append(INFLUENCE_BRUST_CONCLUSION[i])
                underline.append(UNDERLINE_CONCLUSION[i])
            else:
                result.append("")
                underline.append("")
        else:
            result.extend(["", ""])
            underline.append("")

    return [result, underline]

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def search_portrait_history_active_info(uid, date, index_name="copy_user_portrait", doctype="user"):
    # date.formate: 20130901
    date_list = time_series(date)

    try:
        result = es.get(index=index_name, doc_type=doctype, id=uid, _source=True)['_source']
    except NotFoundError:
        return "NotFound"
    except:
        return None

    return_dict = {}
    for item in date_list:
        return_dict[item] = result.get(item, 0)

    in_list = []
    for item in sorted(date_list):
        in_list.append(return_dict[item])
    #print 'in_list:', in_list
    max_influence = max(in_list)
    ave_influence = sum(in_list) / float(7)
    min_influence = min(in_list)
    if max_influence - min_influence <= 400 and ave_influence >= 900:
        mark = u'平稳高影响力'
    elif max_influence - min_influence > 400 and ave_influence >= 900:
        mark = u'波动高影响力'
    elif max_influence - min_influence <= 400 and ave_influence < 900 and ave_influence >= 500:
        mark = u'平稳一般影响力'
    elif max_influence - min_influence > 400 and ave_influence < 900 and ave_influence >= 500:
        mark = u'波动一般影响力'
    elif max_influence - min_influence <= 400 and ave_influence < 500:
        mark = u'平稳低影响力'
    else:
        mark = u'波动低影响力'
    description = [u'该用户为', mark]
    return [in_list, description]

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def statistics_influence_people(uid, date, style):
    # output: different retweeted and comment, uids' domain distribution, topic distribution, registeration geo distribution
    results = {} # retwweted weibo people and comment weibo people
    date1 = str(date).replace('-', '')
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date

    try:
        bci_result = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        bci_result = []
        return results
    origin_mid = [] # origin weibo mid
    retweeted_mid = [] # retweeted weibo mid

    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                        ]
                    }
                }
            }
        },
        "size":1000
    }

    body_1 = copy.deepcopy(query_body)
    body_2 = copy.deepcopy(query_body)

    body_1["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term":{"message_type": 1}}, {"term":{"uid": uid}}])
    result_1 = es.search(index=index_flow_text, doc_type=flow_text_index_type, body=body_1)["hits"]["hits"]
    if result_1:
        for item in result_1:
            origin_mid.append(item['_id'])

    body_1["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term":{"message_type": 3}}, {"term":{"uid": uid}}])
    result_2 = es.search(index=index_flow_text, doc_type=flow_text_index_type, body=body_2)["hits"]["hits"]
    if result_2:
        for item in result_2:
            if item['_source'].get('root_mid', ''):
                retweeted_mid.append(item['_source']['root_mid'])    
    
    origin_retweeted = json.loads(bci_result["origin_weibo_retweeted_detail"])
    retweeted_retweeted = json.loads(bci_result["retweeted_weibo_retweeted_detail"])
    origin_comment = json.loads(bci_result["origin_weibo_comment_detail"])
    retweeted_comment = json.loads(bci_result["retweeted_weibo_comment_detail"])

    """
    retweeted_total_number = sum(origin_retweeted.values()) + sum(retweeted_retweeted.values())
    comment_total_number = sum(origin_comment.values()) + sum(retweeted_comment.values())
    if origin_retweeted:
        origin_retweeted_mid = filter_mid(origin_retweeted)
    if retweeted_retweeted:
        retweeted_retweeted_mid = filter_mid(retweeted_retweeted)
    if origin_comment:
        origin_comment_mid = filter_mid(origin_comment)
    if retweeted_comment:
        retweeted_comment_mid = filter_mid(retweeted_comment)

    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "should":[
                        ],
                        "must": [
                        ]
                    }
                }
            }
        },
        "size":10000
    }
    """

    if int(style) == 0: # retweeted
        retweeted_results = influenced_user_detail(uid, date, origin_mid, retweeted_mid, 3)
        results = retweeted_results
    else:
        comment_results = influenced_user_detail(uid, date, origin_mid, retweeted_mid, 2)
        results = comment_results

    return results

# 需要导入模块: from user_portrait.global_utils import ES_CLUSTER_FLOW1 [as 别名]
# 或者: from user_portrait.global_utils.ES_CLUSTER_FLOW1 import get [as 别名]
def influenced_detail(uid, date, style):
    date1 = str(date).replace('-', '')
    index_name = pre_index + date1
    index_text = "flow_text_" + date
    #detail_text = {}
    style = int(style)
    try:
        user_info = es_cluster.get(index=index_name, doc_type=influence_doctype, id=uid)["_source"]
    except:
        result = {}
        return result
    origin_retweetd_dict = json.loads(user_info["origin_weibo_retweeted_detail"])
    origin_comment_dict = json.loads(user_info['origin_weibo_comment_detail'])
    retweeted_retweeted_dict = json.loads(user_info["retweeted_weibo_retweeted_detail"])
    retweeted_comment_dict = json.loads(user_info["retweeted_weibo_comment_detail"])

    origin_retweetd = sorted(origin_retweetd_dict.items(), key=lambda x:x[1], reverse=True)
    origin_comment = sorted(origin_comment_dict.items(), key=lambda x:x[1], reverse=True)
    retweeted_retweeted = sorted(retweeted_retweeted_dict.items(), key=lambda x:x[1], reverse=True)
    retweeted_comment = sorted(retweeted_comment_dict.items(), key=lambda x:x[1], reverse=True)

    query_body_origin = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                            {"term":{"message_type": 1}},
                            {"term":{"uid": uid}}
                        ]
                    }
                }
            }
        },
        "size": 10000
    }
    result_1 = es.search(index=index_text, doc_type="text", body=query_body_origin)['hits']['hits']
    origin_set = set()
    if result_1:
        for item in result_1:
            origin_set.add(item['_id'])

    query_body_retweeted = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                            {"term":{"message_type": 3}},
                            {"term":{"uid": uid}}
                        ]
                    }
                }
            }
        },
        "size": 10000
    }
    result_2 = es.search(index=index_text, doc_type="text", body=query_body_retweeted)['hits']['hits']
    retweeted_set = set()
    if result_2:
        for item in retweeted_set:
            retweeted_set.add(item['_id'])
    
    if origin_retweetd:
        for item in origin_retweetd:
            if item[0] not in origin_set:
                origin_retweetd.remove(item)

    if origin_comment:
        for item in origin_comment:
            if item[0] not in origin_set:
                origin_comment.remove(item)

    if retweeted_retweeted:
        for item in retweeted_retweeted:
            if item[0] not in retweeted_set:
                retweeted_retweeted.remove(item)

    if retweeted_comment:
        for item in retweeted_comment:
            if item[0] not in retweeted_set:
                retweeted_comment.remove(item)

    if style == 0:
        detail_text = get_text(origin_retweetd[:20], date, user_info, style)
    elif style == 1:
        detail_text = get_text(origin_comment[:20], date, user_info, style)
    elif style == 2:
        detail_text = get_text(retweeted_retweeted[:20], date, user_info, style)
    else:
        detail_text = get_text(retweeted_comment[:20], date, user_info, style)
    #detail_text["origin_retweeted"] = get_text(origin_retweetd, date)
    #detail_text["origin_comment"] = get_text(origin_comment, date)
    #detail_text["retweeted_retweeted"] = get_text(retweeted_retweeted, date)
    #detail_text["retweeted_comment"] = get_text(retweeted_comment, date)

    return detail_text