Python periods.Period类代码示例

 def scalar_plotpoints(
     cls, pk, dimensions=None, metrics=None, depth=0, period=None, flot_time=False, points_type=dict
 ):
     metrics = metrics or ["hits"]
     if isinstance(metrics, basestring):
         metrics = [metrics]
     period = Period.get(period)
     sparse = cls.whale_driver().retrieve(pk, dimensions, metrics, period=period)
     nonsparse = defaultdict(dict)
     if flot_time:
         points_type = list
     for dim, mets in sparse.items():
         for met, points in mets.items():
             dts = period.datetimes_strs()
             nonsparse[dim][met] = []
             for dt in dts:
                 if flot_time:
                     dt_t = to_flot_time(Period.parse_dt_str(dt))
                 else:
                     dt_t = dt
                 value = points[dt] if dt in points else 0
                 nonsparse[dim][met].append([dt_t, float(value)])
             nonsparse[dim][met] = points_type(nonsparse[dim][met])
     if depth > 0:
         for sub in cls.get_subdimensions(pk, dimensions):
             nonsparse = dict(
                 nonsparse.items()
                 + cls.plotpoints(
                     pk, sub, metrics, depth=depth - 1, period=period, flot_time=flot_time, points_type=points_type
                 ).items()
             )
     return nonsparse

def _store(redis, pk, dimension, metric, period, dt, count, method='set',
        rank=False):
    # Keep a list of graphs per pk
    key = keyify(pk, dimension, Period.get(period).interval, metric)
    # Store pk dimensions
    dimension_key = keyify('dimensions', pk)
    dimension_json = keyify(dimension)
    
    if not dimension_json in _added_dimensions[dimension_key]:
        redis.sadd(dimension_key, dimension_json)
        _added_dimensions[dimension_key].append(dimension_json)
    # Store dimensional subdimensions
    if dimension != '_':
        subdimension_key = keyify('subdimensions', pk, parent(dimension))
        if not dimension_json in _added_subdimensions[subdimension_key]:
            redis.sadd(subdimension_key, dimension_json)
            _added_subdimensions[subdimension_key].append(dimension_json)

    if method == 'set':
        new_val = float(count)
        redis.hset(key, dt, new_val)
    elif method == 'incr':
        new_val = redis.execute_command('HINCRBYFLOAT', key, dt, float(count))
    if rank and (isinstance(try_loads(pk), list) or dimension != '_'):
        if isinstance(pk, list) and dimension == '_':
            tgt_pk = parent(pk)
            tgt_dimension = dimension
        else:
            tgt_pk = pk
            tgt_dimension = parent(dimension)
        rank_key = keyify('rank', tgt_pk, tgt_dimension,
                Period.get(period).interval, dt, metric) 
        redis.zadd(rank_key, dimension_json, new_val)
    return new_val

 def zranked(cls, pk, parent_dimension='_', metric='hits', period=None,
         at=None, start=0, size=10, sort_dir=None, tzoffset=None):
     period, ats, tzoffset = Period.get_days(period, at)
     dt = ats or [Period.convert(cls.now(), tzoffset)]
     return map(try_loads,
             _ranked(cls.whale_driver(), pk, parent_dimension, metric,
                 period, dt, start, size, sort_dir=sort_dir))

 def totals(cls, pk, dimensions=None, metrics=None, periods=None, at=None):
     if not periods:
         periods = DEFAULT_PERIODS
     if not isinstance(periods, list):
         periods = [periods]
     metrics = metrics or ['hits']
     if not isinstance(metrics, list):
         metrics = [metrics]
     ratios = []
     for metric in metrics:
         if '/' in metric:
             metrics.remove(metric)
             ratios.append(metric)
             metrics += metric.split('/')
     d = {}
     for p in periods:
         period, ats, tzoffset = Period.get_days(p,at)
         p_data = cls.plotpoints(pk, dimensions, metrics, period=p,at=at)
         p_totals = dict()
         for dim in p_data.keys():
             p_totals[dim] = dict()
             for met, vals in p_data[dim].items():
                 p_totals[dim][met] = sum([
                     v for k, v in vals.items()
                     if k in ats])
             for rat in ratios:
                 top, bot = parse_formula(rat)
                 topt, bott = p_totals[dim][top], p_totals[dim][bot]
                 p_totals[dim][rat] = bott and topt / bott or 0
         d[str(p)] = p_totals
     return d

def _ranked(redis, pk, parent_dimension, metric, period, ats, start=0, size=10,
        sort_dir=None):
    top, bot = parse_formula(metric)
    rank_keyify = lambda ats, met: keyify('rank', pk, parent_dimension,
            Period.get(period).interval, ats, met)
    final_rank_key = rank_keyify(ats, metric)
    def squash_ats(met):
        if len(ats) > 1:
            map(lambda at: redis.zremrangebyscore(rank_keyify(at, met), 0, 0), ats)
            redis.zunionstore(rank_keyify(ats, met),
                    map(lambda at: rank_keyify(at, met), ats))
    squash_ats(top)
    if bot:
        squash_ats(bot)
        top_key, bot_key = rank_keyify(ats, top), rank_keyify(ats, bot)
        redis.execute_command("eval", """
        for key_i, key_n in ipairs(redis.call("zrange", KEYS[2], 0, -1)) do
            local top_s = tonumber(redis.call("zscore", KEYS[1], key_n))
            local bot_s = tonumber(redis.call("zscore", KEYS[2], key_n))
            if top_s and bot_s and bot_s > 0 then
                redis.call("zadd", KEYS[3], top_s/bot_s, key_n)
            end
        end
        """, 3, top_key, bot_key, final_rank_key)
        redis.zremrangebyscore(final_rank_key, 0, 0)
    return redis.zrange(final_rank_key, start, start + size,
                desc=not sort_dir or sort_dir.upper() in ['-', 'DESC', 'HIGH'])

 def total(cls, pk, metric, dimension='_', period=None, at=None, index=None,
         tzoffset=None):
     period, ats, tzoffset = Period.get_days(period, at, tzoffset=tzoffset)
     top, bot = parse_formula(metric)
     dimension = maybe_dumps(dimension)
     if not ats and not index:
         index = -1
     if isinstance(index, int):
         pps = cls.plotpoints(pk, dimension, metric, period=period, points_type=list)
         return pps[dimension][metric][index][1]
     else:
         if not bot:
             pps = cls.plotpoints(pk, dimension, metric, period=period,
                                  tzoffset=tzoffset)
             ppsm = pps[dimension][metric]
             return sum([ppsm[dt] for dt in ats if dt in ppsm])
         else:
             top_pps = cls.plotpoints(pk, dimension, top, period=period,
                                      tzoffset=tzoffset)
             bot_pps = cls.plotpoints(pk, dimension, bot, period=period,
                                      tzoffset=tzoffset)
             top_ppsm = top_pps[dimension][top]
             bot_ppsm = bot_pps[dimension][bot]
             top_tot = sum([top_ppsm[dt] for dt in ats if dt in top_ppsm])
             bot_tot = sum([bot_ppsm[dt] for dt in ats if dt in bot_ppsm])
             return bot_tot and top_tot/bot_tot or 0

    def rank_subdimensions_scalar(cls, pk, dimension='_', metric='hits', period=None):
        period = period or Period.default_size()
        d_k = keyify(dimension)
        total = cls.totals(pk, dimension, metric, periods=[period])[str(period)][d_k][metric]
        ranked = dict()

        def info(sub):
            pps = cls.plotpoints(pk, sub, metric, period=period)[sub][metric]
            sub_total = sum(pps.values())
            return {
                'points': pps,
                'total': sub_total,
                'important': sub_total > 10 and (sub_total > (total / 10)) or False
            }

        for sub in map(maybe_dumps, cls.all_subdimensions(pk, dimension)):
            ranked[sub] = info(sub)
        del(ranked[dimension])

        # Prune parents
        for sub, info in ranked.items():
            children = map(maybe_dumps, cls.get_subdimensions(pk, sub))
            children_total = sum(map(lambda s: ranked[s]['total'], children))
            if info['important'] and (info['total'] - children_total) < (total / 10):
                info['important'] = False
        return ranked

    def __call__(self, *args, **kwargs):
        from whale import Whale
        from periods import Period

        if len(args) and args[0] == Whale or issubclass(args[0], Whale):
            args = args[1:]
        clear_cache = kwargs.pop("unmemoize", False)
        self.get_cache()
        if "period" in kwargs:
            p = Period.get(kwargs["period"])
            kwargs["period"] = str(p)
            ttl = int(p.interval) / 5
        else:
            ttl = 60

        key_name = self.keyify(args, kwargs)

        if clear_cache:
            self.cache.delete(key_name)

        try:
            return json.loads(self.cache[key_name])
        except KeyError:
            value = self.func(Whale, *args, **kwargs)
            self.cache[key_name] = json.dumps(value)
            self.cache.expire(key_name, ttl)
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(Whale, *args, **kwargs)

    def rank_subdimensions_scalar(cls, pk, dimension='_', metric='hits',
            period=None, recursive=True, prune_parents=True, points=False):
        period = period or Period.default_size()
        d_k = keyify(dimension)
        total = cls.cached_totals(pk, dimension, metric, periods=[period])[period][d_k][metric]
        ranked = dict()

        def info(sub):
            pps = cls.plotpoints(pk, sub, metric, period=period)[sub][metric]
            sub_total = sum(pps.values())
            data = {
                'points': pps,
                'score': sub_total,
                'important': sub_total > 10 and (sub_total > (total / 10)) or False,
                'effect': total - sub_total,
                'difference': total - sub_total,
                'value': sub_total,
                'count': sub_total,
                'dimension': sub
            }
            if not points:
                del data['points']
            return data
        _subs = recursive and cls.all_subdimensions or cls.get_subdimensions
        for sub in map(maybe_dumps, _subs(pk, dimension)):
            ranked[sub] = info(sub)

        # Prune parents
        if recursive and prune_parents:
            for sub, info in ranked.items():
                children = map(maybe_dumps, cls.get_subdimensions(pk, sub))
                children_total = sum(map(lambda s: ranked[s]['score'], children))
                if info['important'] and (info['score'] - children_total) < (total / 10):
                    info['important'] = False
        return ranked

 def plotpoints(cls, pk, dimensions=None, metrics=None,
         period=None, overall=True):
     metrics = metrics or ['hits',]
     period = period or Period.default_size()
     sparse = cls.whale_driver().retrieve(pk,dimensions,metrics,
             period=period, overall=overall)
     nonsparse = defaultdict(dict)
     for dimensions, metrics in sparse.items():
         for metric, points in metrics.items():
             dts = Period(*period.split('x')).datetimes_strs()
             nonsparse[dimensions][metric] = []
             for dt in dts:
                 flot_time = to_flot_time(Period.parse_dt_str(dt))
                 value = points[dt] if dt in points else 0
                 nonsparse[dimensions][metric].append([flot_time,
                     float(value)])
     return nonsparse

 def retrieve(self, pk, dimensions, metrics, period=None, dt=None):
     nested = defaultdict(dict)
     period = str(Period.get(period))
     for dimension in map(maybe_dumps, iterate_dimensions(dimensions)):
         for metric in map(maybe_dumps, metrics):
             hash_key = keyify(pk, dimension, period, metric)
             value_dict = self.hgetall(hash_key)
             nested[dimension][metric] = dict([(k, float(v)) for k, v in value_dict.items()])
     return dict(nested)

 def update_count_to(cls, pk, dimensions='_', metrics=None, period=False,
         at=False, rank=False):
     period = Period.get(period)
     at = at or cls.now()
     dt = period.flatten_str(at)
     pipe = cls.whale_driver().pipeline(transaction=False)
     for (metric, i) in metrics.iteritems():
         _store(pipe, pk, dimensions, metric, period, dt, i,
                 rank=rank)
     pipe.execute()

 def plotpoints(self, categories=None, dimensions=None, metrics=None,
         period=None, depth=0):
     categories = categories or ''
     dimensions = dimensions or json.dumps(list(list()))
     # Convert categories to a list, if it's not
     if type(categories) in [str,unicode]: categories = [categories,]
     metrics = metrics or ['hits',]
     period = period or Period.default_size()
     sparse = self.driver().retrieve(categories,dimensions,metrics,
             period=period, depth=depth)
     nonsparse = defaultdict(dict)
     for dimensions, metrics in sparse.items():
         for metric, points in metrics.items():
             dts = Period(*period.split('x')).datetimes_strs()
             nonsparse[dimensions][metric] = []
             for dt in dts:
                 flot_time = to_flot_time(Period.parse_dt_str(dt))
                 value = points[dt] if dt in points else 0
                 nonsparse[dimensions][metric].append([flot_time,
                     float(value)])
     return nonsparse

    def plotpoints(cls, pk, dimensions=None, metrics=None,
            depth=0, period=None, flot_time=False, points_type=dict):
        metrics = metrics or ['hits']
        if isinstance(metrics, basestring):
            metrics = [metrics]
        period = Period.get(period)
        dts = period.datetimes_strs()
        nonsparse = defaultdict(defaultdict)

        # Hardwire time-based metrics for lulz
        time_metrics = {'second': 1, 'minute': 60, 'hour': 3600, 'day': 3600*24, 'week': 3600*24*7}
        #for t_m, factor in time_metrics.items():
        #    if t_m in metrics:
        #        metrics.remove(t_m)
        #        for dimension in dimensions:
        #            nonsparse[dimension][t_m] = list()
        #            for dt in dts:
        #                if flot_time:
        #                    dt = to_flot_time(Period.parse_dt_str(dt))
        #                nonsparse[dimension][t_m].append([dt, period.interval / factor])
        #            nonsparse[dimension][t_m] = points_type(nonsparse[dimension][t_m])
        # Pull the plotpoints that exist from Redis
        sparse = cls.whale_driver().retrieve(pk, dimensions, metrics, period=period)
        
        for dimensions, metrics in sparse.items():
            for metric, points in metrics.items():
                #if metric in time_metrics: continue
                nonsparse[dimensions][metric] = []
                for dt in dts:
                    if flot_time:
                        dt = to_flot_time(Period.parse_dt_str(dt))
                    value = points[dt] if dt in points else 0
                    nonsparse[dimensions][metric].append([dt, float(value)])
                nonsparse[dimensions][metric] = points_type(nonsparse[dimensions][metric])
        if depth > 0:
            for sub in cls.get_subdimensions(pk, dimensions):
                nonsparse = dict(nonsparse.items() +
                    cls.plotpoints(pk, sub, metrics, depth=depth - 1, period=period,
                        flot_time=flot_time, points_type=points_type).items())
        return nonsparse

def _retrieve(redis, pk, dimensions, metrics, period=None, dt=None):
    nested = defaultdict(dict)
    interval = Period.get(period).interval
    for dimension in iterate_dimensions(dimensions)+['_']:
        for metric in metrics:
            if ':' in metric:
                metric_name = metric.split(':')[0]
            else: metric_name = metric
            hash_key = keyify(pk, dimension, interval, metric_name)
            value_dict = redis.hgetall(hash_key)
            nested[maybe_dumps(dimension)][maybe_dumps(metric)] = dict([
                    (k, float(v)) for k, v in value_dict.items()])
    return dict(nested)