Python GeneralMethods.approximateEpoch方法代码示例

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def mapper(self, key, value):
     if False: yield # I'm a generator!
     hashtag_object = cjson.decode(value)
     if 'num_of_occurrences' in hashtag_object and\
             hashtag_object['num_of_occurrences'] >= MIN_HASHTAG_OCCURRENCES_FOR_PROPAGATION_ANALYSIS:
         ltuo_bucket_occ_time_and_occ_utm_id =\
                                     map(
                                            lambda (t, utm_id):
                                                 (GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS), utm_id),
                                            hashtag_object['ltuo_occ_time_and_occ_utm_id']
                                        )
         ltuo_bucket_occ_time_and_occ_utm_id.sort(key=itemgetter(1))
         ltuo_utm_id_and_bucket_occ_times =\
             [ (occ_utm_id,map(itemgetter(0), it_bucket_occ_time_and_occ_utm_id))
              for occ_utm_id, it_bucket_occ_time_and_occ_utm_id in
                 groupby(ltuo_bucket_occ_time_and_occ_utm_id, key=itemgetter(1))
             ]
         ltuo_utm_id_and_bucket_occ_times =\
                                         filter(
                                                lambda (_, occ_times): len(occ_times)>10,
                                                ltuo_utm_id_and_bucket_occ_times
                                            )
         for _, bucket_occ_times in ltuo_utm_id_and_bucket_occ_times:
             gap_perct = 0.05
             gaps = np.arange(gap_perct,1+gap_perct,gap_perct)
             bucket_occ_times = filter_outliers(bucket_occ_times)
             bucket_occ_times_at_gaps = get_items_at_gap(bucket_occ_times, gap_perct)
             start_time = float(bucket_occ_times_at_gaps[0])
             life_time = bucket_occ_times_at_gaps[-1] - start_time
             if life_time>0:
                 norm_num_of_occurrences =\
                                         map(lambda t: int(((t-start_time)/life_time)*100), bucket_occ_times_at_gaps)
                 for gap, norm_num_of_occurrence in zip(gaps, norm_num_of_occurrences):
                     self.mf_gap_to_norm_num_of_occurrences['%0.2f'%gap]+=norm_num_of_occurrence

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def getOccuranesInHighestActiveRegion(hashtagObject):
    def getActiveRegions(timeSeries):
        noOfZerosObserved, activeRegions = 0, []
        currentRegion, occurancesForRegion = None, 0
        for index, l in zip(range(len(timeSeries)),timeSeries):
            if l>0: 
                if noOfZerosObserved>MIN_NO_OF_TIME_UNITS_IN_INACTIVE_REGION or index==0:
                    currentRegion = [None, None, None]
                    currentRegion[0] = index
                    occurancesForRegion = 0
                noOfZerosObserved = 0
                occurancesForRegion+=l
            else: 
                noOfZerosObserved+=1
                if noOfZerosObserved>MIN_NO_OF_TIME_UNITS_IN_INACTIVE_REGION and currentRegion and currentRegion[1]==None:
                    currentRegion[1] = index-MIN_NO_OF_TIME_UNITS_IN_INACTIVE_REGION-1
                    currentRegion[2] = occurancesForRegion
                    activeRegions.append(currentRegion)
        if not activeRegions: activeRegions.append([0, len(timeSeries)-1, sum(timeSeries)])
        else: 
            currentRegion[1], currentRegion[2] = index, occurancesForRegion
            activeRegions.append(currentRegion)
        return activeRegions
    occurranceDistributionInEpochs = getOccurranceDistributionInEpochs(hashtagObject['oc'])
    startEpoch, endEpoch = min(occurranceDistributionInEpochs, key=itemgetter(0))[0], max(occurranceDistributionInEpochs, key=itemgetter(0))[0]
    dataX = range(startEpoch, endEpoch, TIME_UNIT_IN_SECONDS)
    occurranceDistributionInEpochs = dict(occurranceDistributionInEpochs)
    for x in dataX: 
        if x not in occurranceDistributionInEpochs: occurranceDistributionInEpochs[x]=0
    timeUnits, timeSeries = zip(*sorted(occurranceDistributionInEpochs.iteritems(), key=itemgetter(0)))
#    for k, v in zip(timeUnits, timeSeries):
#        print k, v
    hashtagPropagatingRegion = max(getActiveRegions(timeSeries), key=itemgetter(2))
    validTimeUnits = [timeUnits[i] for i in range(hashtagPropagatingRegion[0], hashtagPropagatingRegion[1]+1)]
    return [(p,t) for p,t in hashtagObject['oc'] if GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS) in validTimeUnits]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
    def combineLocationGraphs(graphMap, startingGraphId, startingTime, intervalInSeconds, linear=True, **kwargs):
        if intervalInSeconds%TIME_UNIT_IN_SECONDS==0 and int(intervalInSeconds/TIME_UNIT_IN_SECONDS)!=0: numberOfGraphs = int(intervalInSeconds/TIME_UNIT_IN_SECONDS)
        else: numberOfGraphs = int(intervalInSeconds/TIME_UNIT_IN_SECONDS)+1
        graphId = GeneralMethods.approximateEpoch(GeneralMethods.getEpochFromDateTimeObject(startingTime), TIME_UNIT_IN_SECONDS)
        currentLogarithmicId = LocationGraphs.getLogarithmicGraphId(startingGraphId, graphId)
        currentCollectedGraphs = 0
        graphIdsToCombine = []
        while currentCollectedGraphs!=numberOfGraphs and currentLogarithmicId>0:
            numberOfGraphsToCollect = 2**int(math.log(numberOfGraphs-currentCollectedGraphs,2))
            if not linear and currentLogarithmicId%2==0: 
                indices = [1]+map(lambda j: 2**j, filter(lambda j: currentLogarithmicId%(2**j)==0, range(1, int(math.log(currentLogarithmicId+1,2))+1)))
                if max(indices)>numberOfGraphsToCollect and numberOfGraphsToCollect in indices: index = numberOfGraphsToCollect
                else: index = max(indices)
            else: index=1
            logGraphId = '%s_%s'%(LocationGraphs.getGraphId(startingGraphId, currentLogarithmicId), index)
            if logGraphId in graphMap: graphIdsToCombine.append(logGraphId)
            currentLogarithmicId-=index
            currentCollectedGraphs+=index
        graphIdsToCombine = sorted(graphIdsToCombine, key=lambda id:int(id.split('_')[1]), reverse=True)
#        print graphIdsToCombine
#        for i in graphIdsToCombine:
#            ep, l = i.split('_')
#            print i, datetime.datetime.fromtimestamp(float(ep)), l, graphMap[i].number_of_nodes()
        graphsToCombine = [graphMap[id] for id in graphIdsToCombine]
        return combineGraphList(graphsToCombine, **kwargs)

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def mapper(self, key, hashtag_object):
     ltuo_occ_time_and_occ_location = hashtag_object['ltuo_occ_time_and_occ_location']
     if ltuo_occ_time_and_occ_location:
         ltuo_intvl_time_and_occ_location = [(
                                            GeneralMethods.approximateEpoch(occ_time, TIME_UNIT_IN_SECONDS),
                                            occ_location
                                             ) 
                                           for occ_time, occ_location in ltuo_occ_time_and_occ_location]
         ltuo_intvl_time_and_items =\
                                 GeneralMethods.group_items_by(ltuo_intvl_time_and_occ_location, key=itemgetter(0))
         ltuo_intvl_time_and_items.sort(key=itemgetter(0))
         first_time = ltuo_intvl_time_and_items[0][0]
         intvl_method = lambda (t, it): ((t-first_time)/TIME_UNIT_IN_SECONDS, (t, len(it)))
         ltuo_iid_and_tuo_interval_and_occurrence_count = map(intvl_method, ltuo_intvl_time_and_items)
         peak_tuo_iid_and_tuo_interval_and_occurrence_count = \
                                                         max(
                                                             ltuo_iid_and_tuo_interval_and_occurrence_count,
                                                             key=lambda (_, (__, occurrence_count)): occurrence_count
                                                         )
         peak_iid = peak_tuo_iid_and_tuo_interval_and_occurrence_count[0]
         current_val = 0.0
         total_occurrences = sum(data[1][1] for data in ltuo_iid_and_tuo_interval_and_occurrence_count)
         for iid, (_, occurrence_count) in ltuo_iid_and_tuo_interval_and_occurrence_count:
             is_peak = 0.0
             if iid==peak_iid: is_peak=1.0
             current_val+=occurrence_count
             yield iid, [is_peak, occurrence_count/total_occurrences, current_val/total_occurrences]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def iterate_hashtag_occurrences_with_high_accuracy_lid(line):
    data = cjson.decode(line)
    l = None
    if 'geo' in data: l = data['geo']
    else: l = data['bb']
    t = time.mktime(getDateTimeObjectFromTweetTimestamp(data['t']).timetuple())
    lid = getLatticeLid(l, accuracy=0.0001)
    for h in data['h']: yield h.lower(), [lid, GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS)]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def addHashtagDisplacementsInTime(hashtagObject, distanceMethod=getMeanDistanceFromSource, key='sit'):
    spread, occurencesDistribution = [], defaultdict(list)
    for oc in hashtagObject['oc']: occurencesDistribution[GeneralMethods.approximateEpoch(oc[1], HASHTAG_SPREAD_ANALYSIS_WINDOW_IN_SECONDS)].append(oc)
    for currentTime, oc in occurencesDistribution.iteritems():
        llidsToMeasureSpread = [i[0] for i in oc]
        if llidsToMeasureSpread: spread.append([currentTime, [len(llidsToMeasureSpread), distanceMethod(hashtagObject['src'][0], llidsToMeasureSpread)]])
        else: spread.append([currentTime, [len(llidsToMeasureSpread), 0]])
    hashtagObject[key] = spread

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def getOccurranceDistributionInEpochs(
     occ, timeUnit=TIME_UNIT_IN_SECONDS, fillInGaps=False, occurancesCount=True
 ):
     if occurancesCount:
         occurranceDistributionInEpochs = filter(
             lambda t: t[1] > 2,
             [
                 (k[0], len(list(k[1])))
                 for k in groupby(sorted([GeneralMethods.approximateEpoch(t, timeUnit) for t in zip(*occ)[1]]))
             ],
         )
     else:
         occurranceDistributionInEpochs = filter(
             lambda t: len(t[1]) > 2,
             [
                 (k[0], [t[1] for t in k[1]])
                 for k in groupby(
                     sorted(
                         [(GeneralMethods.approximateEpoch(t[1], timeUnit), t) for t in occ], key=itemgetter(0)
                     ),
                     key=itemgetter(0),
                 )
             ],
         )
     if not fillInGaps:
         return occurranceDistributionInEpochs
     else:
         if occurranceDistributionInEpochs:
             startEpoch, endEpoch = (
                 min(occurranceDistributionInEpochs, key=itemgetter(0))[0],
                 max(occurranceDistributionInEpochs, key=itemgetter(0))[0],
             )
             #            if not occurancesCount: startEpoch, endEpoch = startEpoch[0], endEpoch[0]
             dataX = range(startEpoch, endEpoch, timeUnit)
             occurranceDistributionInEpochs = dict(occurranceDistributionInEpochs)
             for x in dataX:
                 if x not in occurranceDistributionInEpochs:
                     if occurancesCount:
                         occurranceDistributionInEpochs[x] = 0
                     else:
                         occurranceDistributionInEpochs[x] = []
             return occurranceDistributionInEpochs
         else:
             return dict(occurranceDistributionInEpochs)

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def iterateHashtagObjectInstances(line):
    data = cjson.decode(line)
    l = None
    if 'geo' in data: l = data['geo']
    else: l = data['bb']
    t =  GeneralMethods.approximateEpoch(time.mktime(getDateTimeObjectFromTweetTimestamp(data['t']).timetuple()), TIME_UNIT_IN_SECONDS)
    if isWithinBoundingBox(l, BOUNDARY):
        point = getLatticeLid(l, LATTICE_ACCURACY)
        if point!='0.0000_0.0000':
            for h in data['h']: yield h.lower(), [point, t]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def get_tuo_hashtag_and_ltuo_occurrence_time_and_locations(mf_hashtag_to_ltuo_point_and_occurrence_time, top_hashtags):
     hashtag_and_ltuo_occurrence_time_and_locations = \
         [(
           top_hashtag.split()[0],
           [ (GeneralMethods.approximateEpoch(occurrence_time, UNIT_TIME_UNIT_IN_SECONDS), getLatticeLid(point, UNIT_LATTICE_ACCURACY))
                  for point, occurrence_time in mf_hashtag_to_ltuo_point_and_occurrence_time[top_hashtag.split()[0]]
              ]
           )
             for top_hashtag in top_hashtags
         ]
     tuo_hashtag_and_ltuo_occurrence_time_and_locations = []
     for hashtag, ltuo_occurrence_time_and_locations in hashtag_and_ltuo_occurrence_time_and_locations:
         ltuo_occurrence_time_and_locations = [(occurrence_time, zip(*ito_ltuo_occurrence_time_and_locations)[1])
             for occurrence_time, ito_ltuo_occurrence_time_and_locations in 
                 groupby(
                         sorted(ltuo_occurrence_time_and_locations, key=itemgetter(0)),
                         key=itemgetter(0)
                 )
          ]
         tuo_hashtag_and_ltuo_occurrence_time_and_locations.append((hashtag, ltuo_occurrence_time_and_locations))
     return tuo_hashtag_and_ltuo_occurrence_time_and_locations

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def mapper1(self, key, hashtag_object):
     if False: yield
     hashtag = hashtag_object['hashtag']
     ltuo_occ_time_and_occ_location = hashtag_object['ltuo_occ_time_and_occ_location']
     ltuo_location_and_items = GeneralMethods.group_items_by(ltuo_occ_time_and_occ_location, key=itemgetter(1))
     ltuo_location_and_occurrence_time =\
                         [(location, min(items, key=itemgetter(0))[0])for location, items in ltuo_location_and_items]
     ltuo_location_and_occurrence_time = [(
                                           location, 
                                           GeneralMethods.approximateEpoch(occurrence_time, TIME_UNIT_IN_SECONDS)
                                           ) 
                                          for location, occurrence_time in ltuo_location_and_occurrence_time]
     if ltuo_location_and_occurrence_time:
         occurrence_times = filter_outliers(zip(*ltuo_location_and_occurrence_time)[1])
         ltuo_location_and_occurrence_time =\
                                         filter(lambda (l, o): o in occurrence_times, ltuo_location_and_occurrence_time)
         for location, occurrence_time in ltuo_location_and_occurrence_time:
             self.mf_location_to_ltuo_hashtag_and_min_occ_time[location].append([hashtag, occurrence_time])
             for neighbor_location, _ in ltuo_location_and_occurrence_time:
                 if location!=neighbor_location:
                     self.mf_location_to_neighbor_locations[location].add(neighbor_location)

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
 def _TemporalDistribution(mf_hashtag_to_ltuo_point_and_occurrence_time, top_hashtags):
     tuo_hashtag_and_ltuo_occurrence_time_and_no_of_occurrences = []
     mf_occurrence_time_to_no_of_occurrences = defaultdict(float)
     for hashtag in top_hashtags:
         hashtag = hashtag.split()[0]
         ltuo_point_and_occurrence_time = mf_hashtag_to_ltuo_point_and_occurrence_time[hashtag]
         for _, occurrence_time in ltuo_point_and_occurrence_time:
             mf_occurrence_time_to_no_of_occurrences[GeneralMethods.approximateEpoch(occurrence_time, UNIT_TIME_UNIT_IN_SECONDS)] += 1
         tuo_hashtag_and_ltuo_occurrence_time_and_no_of_occurrences.append([
                                   hashtag,
                                   sorted(
                                          mf_occurrence_time_to_no_of_occurrences.iteritems(),
                                          key=itemgetter(0)
                                          )
                         ])
     chart_data = []
     for hashtag, ltuo_occurrence_time_and_no_of_occurrences in \
             tuo_hashtag_and_ltuo_occurrence_time_and_no_of_occurrences:
         chart_data.append({
                            'name': hashtag,
                            'data': [ (Charts.getTimeTuple(occurrence_time), no_of_occurrences) for occurrence_time, no_of_occurrences in ltuo_occurrence_time_and_no_of_occurrences],
                            'showInLegend': False
                            })
     return chart_data

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def getOccurencesFilteredByDistributionInTimeUnits(occ): 
    validTimeUnits = [t[0] for t in getOccurranceDistributionInEpochs(occ) if t[1]>=MIN_OCCUREANCES_PER_TIME_UNIT]
    return [(p,t) for p,t in occ if GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS) in validTimeUnits]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def getValidTimeUnits(occ):
    occurranceDistributionInEpochs = [(k[0], len(list(k[1]))) for k in groupby(sorted([GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS) for t in zip(*occ)[1]]))]
    return [t[0] for t in occurranceDistributionInEpochs if t[1]>=MIN_OBSERVATIONS_PER_TIME_UNIT]

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def addHashtagLocalityIndexInTime(hashtagObject):
    liInTime, occurencesDistribution = [], defaultdict(list)
    for oc in hashtagObject['oc']: occurencesDistribution[GeneralMethods.approximateEpoch(oc[1], HASHTAG_SPREAD_ANALYSIS_WINDOW_IN_SECONDS)].append(oc)
    for currentTime, oc in occurencesDistribution.iteritems(): liInTime.append([currentTime, getLocalityIndexAtK(zip(*oc)[0], K_VALUE_FOR_LOCALITY_INDEX)])
    hashtagObject['liInTime'] = liInTime

# 需要导入模块: from library.classes import GeneralMethods [as 别名]
# 或者: from library.classes.GeneralMethods import approximateEpoch [as 别名]
def getOccurencesFilteredByDistributionInTimeUnits(occ, validTimeUnits): return [(p,t) for p,t in occ if GeneralMethods.approximateEpoch(t, TIME_UNIT_IN_SECONDS) in validTimeUnits]
def plotDistributionGraphs(occurences, validTimeUnits, title, startingEpoch=None):