Python R_RECOMMENTATION.hget方法代码示例

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def recommentation_in_auto(seatch_date, submit_user):
    results = []
    #run type
    if RUN_TYPE == 1:
        now_date = ts2datetime(time.time() - DAY)
    else:
        now_date = ts2datetime(datetime2ts(RUN_TEST_TIME) - DAY)
    recomment_hash_name = 'recomment_' + now_date + '_auto'
    recomment_influence_hash_name = 'recomment_' + now_date + '_influence'
    recomment_sensitive_hash_name = 'recomment_' + now_date + '_sensitive'
    recomment_compute_hash_name = 'compute'
    #step1: get auto
    auto_result = r.hget(recomment_hash_name, 'auto')
    if auto_result:
        auto_user_list = json.loads(auto_result)
    else:
        auto_user_list = []
    #step2: get admin user result
    admin_result = r.hget(recomment_hash_name, submit_user)
    if admin_result:
        admin_user_list = json.loads(admin_result)
    else:
        admin_user_list = []
    #step3: get union user and filter compute/influence/sensitive
    union_user_auto_set = set(auto_user_list) | set(admin_user_list)
    influence_user = set(r.hkeys(recomment_influence_hash_name))
    sensitive_user = set(r.hkeys(recomment_sensitive_hash_name))
    compute_user = set(r.hkeys(recomment_compute_hash_name))
    filter_union_user = union_user_auto_set - (influence_user | sensitive_user | compute_user)
    auto_user_list = list(filter_union_user)
    #step4: get user detail
    results = get_user_detail(now_date, auto_user_list, 'show_in', 'auto')
    return results

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_uname(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    operation_type = input_data['operation_type']
    upload_data = input_data['upload_data']
    # get uname list from upload data
    uname_list_pre = upload_data.split('\n')
    uname_list = [item.split('\r')[0] for item in uname_list_pre]
    uid_list = []
    have_in_user_list = []
    invalid_user_list = []
    valid_uname_list = []
    #step1: get uid list from uname
    profile_exist_result = es_user_profile.search(index=profile_index_name, doc_type=profile_index_type, body={'query':{'terms':{'nick_name': uname_list}}}, _source=False, fields=['nick_name'])['hits']['hits']
    for profile_item in profile_exist_result:
        uid = profile_item['_id']
        uid_list.append(uid)
        uname = profile_item['fields']['nick_name'][0]
        valid_uname_list.append(uname)
    invalid_user_list = list(set(uname_list) - set(valid_uname_list))
    if len(invalid_user_list) != 0:
        return False, 'invalid user info', invalid_user_list
    #step2: filter user not in user_portrait and compute
    #step2.1: identify in user_portrait
    new_uid_list = []
    exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids': uid_list})['docs']
    new_uid_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==False]
    have_in_user_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==True]
    if not new_uid_list:
        return False, 'all user in'
    #step2.2: identify in compute
    new_uid_set = set(new_uid_list)
    compute_set = set(r.hkeys('compute'))
    in_uid_list = list(new_uid_set - compute_set)
    if not in_uid_list:
        return False, 'all user in'
    #step3: save submit
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    #identify final submit user list
    final_submit_user_list = []
    for in_item in in_uid_list:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, in_item))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system':'0', 'operation': submit_user}
        if operation_type == 'submit':
            r.hset(hashname_submit, in_item, json.dumps(tmp))
            r.hset(submit_user_recomment, in_item, '0')
        final_submit_user_list.append(in_item)
    return True, invalid_user_list, have_in_user_list, final_submit_user_list

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_url(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    operation_type = input_data['operation_type']
    upload_data = input_data['upload_data']
    #step1: get uid list from input_data url
    url_list_pre = upload_data.split('\n')
    url_list = [item.split('\r')[0] for item in url_list_pre]
    uid_list = []
    invalid_uid_list = []
    have_in_uid_list = []
    for url_item in url_list:
        try:
            #url_item = 'http://weibo.com/p/1002065727942146/album?.....'
            url_list = url_item.split('/')
            uid = url_list[4][-10:]
            uid_list.append(uid)
        except:
            invalid_uid_list.append(url_item)
    if len(invalid_uid_list)!=0:
        return False, 'invalid user info', invalid_uid_list
    #step2: identify uid list is not exist in user_portrait and compute
    #step2.1: identify in user_portrait
    new_uid_list = []
    exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids':uid_list}, _source=True)['docs']
    new_uid_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==False]
    have_in_uid_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==True]
    #step2.2: identify in compute
    new_uid_set = set(new_uid_list)
    compute_set = set(r.hkeys('compute'))
    in_uid_list = list(new_uid_set - compute_set)
    if len(in_uid_list)==0:
        return False, 'all user in'
    #step3: save
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    #identify the final submit user
    final_submit_user_list = []
    for in_item in in_uid_list:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, in_item))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system': '0', 'operation': submit_user}
        if operation_type == 'submit':
            r.hset(hashname_submit, in_item, json.dumps(tmp))
            r.hset(submit_user_recomment, in_item, '0')
        final_submit_user_list.append(in_item)
    return True, invalid_uid_list, have_in_uid_list, final_submit_user_list

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def identify_compute(data):
    results = False
    compute_status = 1
    hash_name = 'compute'
    uid2compute = r.hgetall(hash_name)
    for item in data:
        uid = item[1]
        result = r.hget(hash_name, uid)
        in_date = json.loads(result)[0]
        r.hset(hash_name, uid, json.dumps([in_date, compute_status]))

    return True

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_uname(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    upload_data = input_data['upload_data']
    # get uname list from upload data
    uname_list = upload_data.split('\n') 
    uid_list = []
    #step1: get uid list from uname
    profile_exist_result = es_user_profile.search(index=profile_index_name, doc_type=profile_index_type, body={'query':{'terms':{'nick_name': uname_list}}}, _source=False)['hits']['hits']
    for profile_item in profile_exist_result:
        uid = profile_item['_id']
        uid_list.append(uid)
    if not uid_list:
        return 'uname list valid'
    #step2: filter user not in user_portrait and compute
    #step2.1: identify in user_portrait
    new_uid_list = []
    exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids': uid_list})['docs']
    new_uid_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==False]
    if not new_uid_list:
        return 'uname list all in'
    #step2.2: identify in compute
    new_uid_set = set(new_uid_list)
    compute_set = r.hkeys('compute')
    in_uid_list = list(new_uid_set - compute_set)
    if not in_uid_list:
        return 'uname list all in'
    #step3: save submit
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    for in_item in in_uid_list:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, uid))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system':'0', 'operation': submit_user}
        r.hset(hashname_submit, uid, json.dumps(tmp))
        r.hset(submit_user_recomment, uid, '0')
    return True

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def new_identify_in(data, date, submit_user):
    in_status = 1
    compute_status = 0
    hashname_submit = "submit_recomment_" + date
    hashname_influence = "recomment_" + date + "_influence"
    hashname_sensitive = "recomment_" + date + "_sensitive"
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive)) # 系统自动推荐名单
    for item in data:
        date = item[0] # identify the date form '2013-09-01' with web
        uid = item[1]
        #status = item[2]
        if uid in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, uid))
            recommentor_list = (tmp['operation']).split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = "&".join(new_list)
        else:
            tmp = {"system":"0", "operation":submit_user}
        r.hset(hashname_submit, uid, json.dumps(tmp))
    return True

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_url(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    upload_data = input_data['upload_data']
    #step1: get uid list from input_data url
    url_list = upload_data.split('\n')
    uid_list = []
    for url_item in url_list:
        #url_item = 'weibo.com/p/1002065727942146/album?.....'
        url_list = url_item.split('/')
        uid = url_list[2][-10:]
        uid_list.append(uid)   
    #step2: identify uid list is not exist in user_portrait and compute
    #step2.1: identify in user_portrait
    new_uid_list = []
    exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids':uid_list}, _source=True)['docs']
    new_uid_list = [exist_item['_id'] for exist_item in exist_portrait_result if exist_item['found']==False]
    #step2.2: identify in compute
    new_uid_set = set(new_uid_list)
    compute_set = r.hkeys('compute')
    in_uid_list = list(new_uid_set - compute_set)
    #step3: save
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    for in_item in in_uid_list:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, uid))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system': '0', 'operation': submit_user}
        r.hset(hashname_submit, uid, json.dumps(tmp))
        r.hset(submit_user_recomment, uid, '0')
    return True

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_uid(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    upload_data = input_data['upload_data']
    line_list = upload_data.split('\n')
    uid_list = []
    for line in line_list:
        uid = line[:10]
        if len(uid)==10:
            uid_list.append(uid)
    #identify the uid is not exist in user_portrait and compute
    #step1: filter in user_portrait
    new_uid_list = []
    exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids':uid_list}, _source=False)['docs']
    for exist_item in exist_portrait_result:
        if exist_item['found'] == False:
            new_uid_list.append(exist_item['_id'])
    #step2: filter in compute
    new_uid_set = set(new_uid_list)
    compute_set = set(r.hkeys('compute'))
    in_uid_set = list(new_uid_set - compute_set)
    for in_item in in_uid_set:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashtname_submit, in_item))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system':'0', 'operation':submit_user}
        r.hset(hashname_submit, in_item, json.dumps(tmp))
        r.hset(submit_user_recomment, in_item, '0')
    return True

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def get_user_detail(date, input_result, status, user_type="influence", auth=""):
    bci_date = ts2datetime(datetime2ts(date) - DAY)
    results = []
    if status=='show_in':
        uid_list = input_result
    if status=='show_compute':
        uid_list = input_result.keys()
    if status=='show_in_history':
        uid_list = input_result.keys()
    if date!='all':
        index_name = 'bci_' + ''.join(bci_date.split('-'))
    else:
        now_ts = time.time()
        now_date = ts2datetime(now_ts)
        index_name = 'bci_' + ''.join(now_date.split('-'))
    index_type = 'bci'
    user_bci_result = es_cluster.mget(index=index_name, doc_type=index_type, body={'ids':uid_list}, _source=True)['docs']
    user_profile_result = es_user_profile.mget(index='weibo_user', doc_type='user', body={'ids':uid_list}, _source=True)['docs']
    max_evaluate_influ = get_evaluate_max(index_name)
    for i in range(0, len(uid_list)):
        uid = uid_list[i]
        bci_dict = user_bci_result[i]
        profile_dict = user_profile_result[i]
        try:
            bci_source = bci_dict['_source']
        except:
            bci_source = None
        if bci_source:
            influence = bci_source['user_index']
            influence = math.log(influence/max_evaluate_influ['user_index'] * 9 + 1 ,10)
            influence = influence * 100
        else:
            influence = ''
        try:
            profile_source = profile_dict['_source']
        except:
            profile_source = None
        if profile_source:
            uname = profile_source['nick_name'] 
            location = profile_source['user_location']
            fansnum = profile_source['fansnum']
            statusnum = profile_source['statusnum']
        else:
            uname = ''
            location = ''
            fansnum = ''
            statusnum = ''
        if status == 'show_in':
            if user_type == "sensitive":
                tmp_ts = datetime2ts(date) - DAY
                tmp_data = r_cluster.hget("sensitive_"+str(tmp_ts), uid)
                if tmp_data:
                    sensitive_dict = json.loads(tmp_data)
                    sensitive_words = sensitive_dict.keys()
                else:
                    sensitive_words = []
                results.append([uid, uname, location, fansnum, statusnum, influence, sensitive_words])
            else:
                results.append([uid, uname, location, fansnum, statusnum, influence])
            if auth:
                hashname_submit = "submit_recomment_" + date
                tmp_data = json.loads(r.hget(hashname_submit, uid))
                recommend_list = (tmp_data['operation']).split('&')
                admin_list = []
                admin_list.append(tmp_data['system'])
                admin_list.append(list(set(recommend_list)))
                admin_list.append(len(recommend_list))
                results[-1].extend(admin_list)
        if status == 'show_compute':
            in_date = json.loads(input_result[uid])[0]
            compute_status = json.loads(input_result[uid])[1]
            if compute_status == '1':
                compute_status = '3'
            results.append([uid, uname, location, fansnum, statusnum, influence, in_date, compute_status])
        if status == 'show_in_history':
            in_status = input_result[uid]
            if user_type == "sensitive":
                tmp_ts = datetime2ts(date) - DAY
                tmp_data = r_cluster.hget("sensitive_"+str(tmp_ts), uid)
                if tmp_data:
                    sensitive_dict = json.loads(tmp_data)
                    sensitive_words = sensitive_dict.keys()
                results.append([uid, uname, location, fansnum, statusnum, influence, in_status, sensitive_words])
            else:
                results.append([uid, uname, location, fansnum, statusnum, influence, in_status])

    return results

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
def submit_identify_in_uid(input_data):
    date = input_data['date']
    submit_user = input_data['user']
    operation_type = input_data['operation_type']
    hashname_submit = 'submit_recomment_' + date
    hashname_influence = 'recomment_' + date + '_influence'
    hashname_sensitive = 'recomment_' + date + '_sensitive'
    submit_user_recomment = 'recomment_' + submit_user + '_' + str(date)
    auto_recomment_set = set(r.hkeys(hashname_influence)) | set(r.hkeys(hashname_sensitive))
    upload_data = input_data['upload_data']
    line_list = upload_data.split('\n')
    uid_list = []
    invalid_uid_list = []
    for line in line_list:
        uid = line.split('\r')[0]
        #if len(uid)==10:
        #    uid_list.append(uid)
        if uid != '':
            uid_list.append(uid)
    if len(invalid_uid_list)!=0:
        return False, 'invalid user info', invalid_uid_list
    #identify the uid is not exist in user_portrait and compute
    #step1: filter in user_portrait
    new_uid_list = []
    have_in_uid_list = []
    try:
        exist_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, body={'ids':uid_list}, _source=False)['docs']
    except:
        exist_portrait_result = []
    if exist_portrait_result:
        for exist_item in exist_portrait_result:
            if exist_item['found'] == False:
                new_uid_list.append(exist_item['_id'])
            else:
                have_in_uid_list.append(exist_item['_id'])
    else:
        new_uid_list = uid_list
   
    #step2: filter in compute
    new_uid_set = set(new_uid_list)
    compute_set = set(r.hkeys('compute'))
    in_uid_set = list(new_uid_set - compute_set)
    print 'new_uid_set:', new_uid_set 
    print 'in_uid_set:', in_uid_set
    if len(in_uid_set)==0:
        return False, 'all user in'
    #identify the final add user
    final_submit_user_list = []
    for in_item in in_uid_set:
        if in_item in auto_recomment_set:
            tmp = json.loads(r.hget(hashname_submit, in_item))
            recommentor_list = tmp['operation'].split('&')
            recommentor_list.append(str(submit_user))
            new_list = list(set(recommentor_list))
            tmp['operation'] = '&'.join(new_list)
        else:
            tmp = {'system':'0', 'operation':submit_user}
        if operation_type == 'submit':
            r.hset(hashname_submit, in_item, json.dumps(tmp))
            r.hset(submit_user_recomment, in_item, '0')
        final_submit_user_list.append(in_item)
    return True, invalid_uid_list, have_in_uid_list, final_submit_user_list

# 需要导入模块: from user_portrait.global_utils import R_RECOMMENTATION [as 别名]
# 或者: from user_portrait.global_utils.R_RECOMMENTATION import hget [as 别名]
             sensitive_words = sensitive_dict.keys()
         else:
             sensitive_words = []
         if sensitive_history_dict.get('fields',0):
             #print sensitive_history_dict['fields'][sensitive_string][0]
             #print top_sensitive
             sensitive_value = math.log(sensitive_history_dict['fields'][sensitive_string][0]/float(top_sensitive)*9+1, 10)*100
             #print "sensitive_value", sensitive_value
         else:
             sensitive_value = 0
         results.append([uid, uname, location, fansnum, statusnum, influence, sensitive_words, sensitive_value])
     else:
         results.append([uid, uname, location, fansnum, statusnum, influence])
     if auth:
         hashname_submit = "submit_recomment_" + date
         tmp_data = json.loads(r.hget(hashname_submit, uid))
         recommend_list = (tmp_data['operation']).split('&')
         admin_list = []
         admin_list.append(tmp_data['system'])
         admin_list.append(list(set(recommend_list)))
         admin_list.append(len(recommend_list))
         results[-1].extend(admin_list)
 if status == 'show_compute':
     in_date = json.loads(input_result[uid])[0]
     compute_status = json.loads(input_result[uid])[1]
     if compute_status == '1':
         compute_status = '3'
     results.append([uid, uname, location, fansnum, statusnum, influence, in_date, compute_status])
 if status == 'show_in_history':
     in_status = input_result[uid]
     if user_type == "sensitive":