Python Timeseries.read_from_db方法代码示例

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def get(self, request, pk, format=None):
     ts = Timeseries(id=int(pk))
     self.check_object_permissions(request, ts)
     ts.read_from_db(connection)
     result = StringIO()
     ts.write(result)
     return HttpResponse(result.getvalue(), content_type="text/plain")

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def handle(self, *args, **options):
     try:
         username = args[0]
     except IndexError:
         print "I need a username!"
         return -1
     try:
         if username:
             user = User.objects.get(username=username)
             out = []
             print "output for {x}".format(x=username)
             household = Household.objects.get(user=user)
             timeseries = household \
                 .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                                 variable__id=VAR_PERIOD)
             series = TSeries(id=timeseries.id)
             series.read_from_db(db.connection)
             timestamps = sorted(series.keys())
             values = np.array([])
             for ts in timestamps:
                 val = series[ts]
                 if isnan(val) or val == 0:
                     continue
                 values = np.append(values, val)
                 perc = np.percentile(values, 90)
                 out.append([ts, val, perc])
             _outfile = "timeseries_%s.csv" % username
             _path = "data/"
             with open(path.join(_path, _outfile), 'w') as of:
                 a = csv.writer(of, delimiter=',',
                                quotechar='"',
                                quoting=csv.QUOTE_ALL)
                 a.writerows(out)
     except Exception as e:
         print "failed with %s" % repr(e)

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def update_ts_temp_file(cache_dir, connection, id):
    full_rewrite = False

    afilename = os.path.join(cache_dir, '%d.hts'%(id,))
    if os.path.exists(afilename):
        if os.path.getsize(afilename)<3:
            full_rewrite = True
#Update the file in the case of logged data, if this is possible
    if os.path.exists(afilename) and not full_rewrite:
        with open(afilename, 'r') as fileobject:
            xr = xreverse(fileobject, 2048)
            line = xr.next()
        lastdate = datetime_from_iso(line.split(',')[0])
        ts = Timeseries(id)
        ts.read_from_db(connection, bottom_only=True)
        if len(ts)>0:
            db_start, db_end = ts.bounding_dates()
            if db_start>lastdate:
                full_rewrite = True
            elif db_end>lastdate:
                lastindex = ts.index(lastdate)
                with open(afilename, 'a') as fileobject:
                    ts.write(fileobject, start=ts.keys()[lastindex+1])
#Check for tmmp file or else create it
    if not os.path.exists(afilename) or full_rewrite:
        ts = Timeseries(id)
        ts.read_from_db(connection)
        if not os.path.exists(cache_dir):
            os.mkdir(cache_dir)
        tempfile_handle, tempfile_name = tempfile.mkstemp(dir=cache_dir)
        with os.fdopen(tempfile_handle, 'w') as afile:
            ts.write(afile)
        shutil.move(tempfile_name, afilename)

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def process_dma(dma, bounds):
    """Process DMA timeseries by aggregating all the contained
    households in the DMA"""
    print "Process DMA %s" % (dma,)
    for dma_series in dma.timeseries.all():
        print "Process series %s" % (dma_series,)
        per_capita = dma_series.name.find('capita') > -1
        variable = dma_series.variable.id
        if dma_series.time_step.id == TSTEP_FIFTEEN_MINUTES:
            start = bounds[variable]['fifteen_start']
            end = bounds[variable]['fifteen_end']
            # Fifteen minutes process is DEACTIVATED!
            # We don't process fifteen minutes, it takes too long,
            # maybe we reactivate later after we optimize the
            # algorithm to process only new records
            continue
        elif dma_series.time_step.id == TSTEP_HOURLY:
            start = bounds[variable]['hourly_start']
            end = bounds[variable]['hourly_end']
        elif dma_series.time_step.id == TSTEP_DAILY:
            start = bounds[variable]['daily_start']
            end = bounds[variable]['daily_end']
        elif dma_series.time_step.id == TSTEP_MONTHLY:
            start = bounds[variable]['monthly_start']
            end = bounds[variable]['monthly_end']
        time_step = ReadTimeStep(dma_series.id, dma_series)
        tseries = TSeries(time_step = time_step, id=dma_series.id)
        nhseries = TSeries(time_step = time_step)
        pointer = start
        while pointer<=end:
            tseries[pointer] = 0
            nhseries[pointer] = 0
            pointer = tseries.time_step.next(pointer)
        for household in dma.households.all():
            for h_series_db in household.timeseries.filter(
                    time_step__id=dma_series.time_step.id,
                    variable__id=variable):
                hseries = TSeries(id=h_series_db.id)
                hseries.read_from_db(db.connection)
                pointer = start
                while pointer<=end:
                    try:
                        v = hseries[pointer]
                        if math.isnan(v):
                            pointer = tseries.time_step.next(pointer)
                            continue
                        if per_capita:
                            v = v/float(household.num_of_occupants)
                        tseries[pointer] += v
                        nhseries[pointer] += 1
                    except KeyError:
                        v = 0
                    pointer = tseries.time_step.next(pointer)
        pointer = start
        while pointer<=end:
            if per_capita and nhseries[pointer]>0:
                tseries[pointer] = tseries[pointer] / nhseries[pointer]
            pointer = tseries.time_step.next(pointer)
        tseries.write_to_db(db.connection, commit=True)#False)

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def testUploadTsDataUnauthenticated(self):
     # Attempt to upload some timeseries data, unauthenticated
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def testUploadTsDataGarbage(self):
     self.assert_(self.client.login(username='user1', password='password1'))
     response = self.client.put(
         "/api/tsdata/{}/".format(self.timeseries1.id),
         encode_multipart(BOUNDARY,
                          {'timeseries_records': '2012-aa-06 18:17,20,\n'}),
         content_type=MULTIPART_CONTENT)
     self.assertEqual(response.status_code, 400)
     t = Timeseries(self.timeseries1.id)
     t.read_from_db(connection)
     self.assertEqual(len(t), 0)
     self.client.logout()

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def testUploadTsDataGarbage(self):
     self.assert_(self.client.login(username="user1", password="password1"))
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-aa-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 400)
     self.assertEqual(len(t), 0)
     self.client.logout()

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def testUploadTsDataAsWrongUser(self):
     # Attempt to upload some timeseries data as user 2; should deny
     self.assert_(self.client.login(username='user2', password='password2'))
     response = self.client.put(
         "/api/tsdata/{}/".format(self.timeseries1.id),
         encode_multipart(BOUNDARY,
                          {'timeseries_records': '2012-11-06 18:17,20,\n'}),
         content_type=MULTIPART_CONTENT)
     t = Timeseries(self.timeseries1.id)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)
     self.client.logout()

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def testUploadTsDataAsWrongUser(self):
     # Attempt to upload some timeseries data as user 2; should deny
     self.assert_(self.client.login(username="user2", password="password2"))
     response = self.client.put(
         "/api/tsdata/1/",
         encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
         content_type=MULTIPART_CONTENT,
     )
     t = Timeseries(1)
     t.read_from_db(connection)
     self.assertEqual(response.status_code, 403)
     self.assertEqual(len(t), 0)
     self.client.logout()

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def process():
    for household in Household.objects.all():
        daily_series_db = household.timeseries.get(
                time_step__id=TSTEP_DAILY)
        series = TSeries(id=daily_series_db.id)
        series.read_from_db(db.connection)
        m = 1000.000*series.average()
        if math.isnan(m):
            continue
        num_of_occupants = max(1,
                int(round(m/AVERAGE_UNIT_WATER_CONSUMPTION)))
        print 'Household with id=%s, average daily consumption %.1f, '\
              'number of occupants set to %s'%(household.id, m,
                      num_of_occupants,)
        household.num_of_occupants = num_of_occupants
        household.save()

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def has_burst(household):
    """
    We won't be using this algorithm any more
    :param household:
    :return:
    """
    name = household.user.username
    if not name.startswith('GR'):
        return 0, 0
    timeseries = household \
        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                        variable__id=VAR_PERIOD)
    series = TSeries(id=timeseries.id)
    series.read_from_db(db.connection)
    timestamps = sorted(series.keys())
    today = []  # all today's values
    daily_maxes = {}
    for i in range(1, len(timestamps)):
        ts = timestamps[i]
        prev_ts = timestamps[i-1]
        date = ts.date()
        # if previous value is NaN we don't take this value into consideration
        # Because it might have all consumption of all the previous NaN times
        val = series[ts]
        prev_val = series[prev_ts]
        if isnan(prev_val):
            continue
        if i < len(timestamps) - 100:
            if not isnan(val) and not val == 0:
                daily_max = daily_maxes.get(date, 0)
                if val > daily_max:
                    daily_maxes[date] = val
        else:
            tm = "%s-%s-%s %s:%s" % (ts.year, ts.month, ts.day,
                                     ts.time().hour, ts.time().minute)
            if not isnan(val) and not val == 0:
                today.append((val, tm))

    if daily_maxes and today:
        maxes = np.array(daily_maxes.values())
        p = np.percentile(maxes, 90)
        for cons, tm in today:
            if cons > p:
                return cons, tm
    return 0, 0

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def has_burst_old(household):
    """
    We won't be using this algorithm any more
    :param household:
    :return:
    """
    name = household.user.username
    if not name.startswith('GR'):
        return 0, 0
    timeseries = household \
        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                        variable__id=VAR_PERIOD)
    series = TSeries(id=timeseries.id)
    series.read_from_db(db.connection)
    timestamps = sorted(series.keys())
    today = []  # all today's values
    _all = []
    for i in range(1, len(timestamps)):
        ts = timestamps[i]
        if household.user.username == "GR006047" \
                and ts.year == 2015 and ts.month == 2 and ts.day == 9 \
                and ts.hour == 17:
            pass
        prev_ts = timestamps[i-1]
        # if previous value is NaN we don't take this value into consideration
        # Because it might have all consumption of all the previous NaN times
        val = series[ts]
        prev_val = series[prev_ts]
        if isnan(prev_val):
            continue
        if i < len(timestamps) - 100:
            if not isnan(val) and not val == 0:
                _all.append(series[ts])
        else:
            tm = "%s:%s" % (ts.time().hour, ts.time().minute)
            if not isnan(val) and not val == 0:
                today.append((val, tm))

    if _all and today:
        all1 = np.array(_all)
        p = np.percentile(all1, 95)
        for cons, tm in today:
            if cons > p:
                return cons, tm
    return 0, 0

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def get_values_after(household, dt, variable):
    timeseries = None
    if variable == "WaterCold":
        timeseries = household \
            .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                            variable__id=VAR_PERIOD)
    elif variable == "Electricity":
        timeseries = household \
            .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                            variable__id=VAR_ENERGY_PERIOD)
    data = []
    if timeseries:
        series = TSeries(id=timeseries.id)
        series.read_from_db(db.connection)
        timestamps = sorted(series.keys())
        for ts in timestamps:
            val = series[ts]
            if ts <= dt:
                continue
            data.append((ts, val))
    return data

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
 def handle(self, *args, **options):
     try:
         username = args[0]
     except IndexError:
         print "I need a username!"
         return -1
     try:
         if username not in ["GR", "GB", "PT", "GBA"]:
             users = User.objects.filter(username=username)
         else:
             users = User.objects.filter(username__startswith=username)
         for user in users:
             out = []
             print "output for {x}".format(x=username)
             household = Household.objects.get(user=user)
             # ts_raw = household.timeseries.filter(time_step__isnull=True,
             #                                      variable__id=VAR_CUMULATIVE)[0]
             # series = TSeries(id=ts_raw.id)
             timeseries = household \
                 .timeseries.get(variable__id=VAR_CUMULATIVE)
             series = TSeries(id=timeseries.id)
             series.read_from_db(db.connection)
             timestamps = sorted(series.keys())
             values = np.array([])
             for ts in timestamps:
                 val = series[ts]
                 if isnan(val) or val == 0:
                     continue
                 values = np.append(values, val)
                 #perc = np.percentile(values, 90)
                 out.append([ts, val])
             _outfile = "timeseries_cumulative_%s.csv" % user.username
             _path = "data/"
             with open(path.join(_path, _outfile), 'w') as of:
                 a = csv.writer(of, delimiter=',',
                                quotechar='"',
                                quoting=csv.QUOTE_ALL)
                 a.writerows(out)
     except Exception as e:
         print "failed with %s" % repr(e)

# 需要导入模块: from pthelma.timeseries import Timeseries [as 别名]
# 或者: from pthelma.timeseries.Timeseries import read_from_db [as 别名]
def MultiTimeseriesProcessDb(method, timeseries_arg, out_timeseries_id,
                             db, read_tstep_func, transaction=None, 
                             commit=True, options={}):
    out_timeseries = Timeseries(id = out_timeseries_id)
    opts = copy.deepcopy(options)
    if 'append_only' in opts and opts['append_only']:
        bounds = timeseries_bounding_dates_from_db(db, 
                                                   id = out_timeseries_id)
        opts['start_date'] = bounds[1] if bounds else None;
        opts['interval_exclusive'] = True
    tseries_arg={}
    for key in timeseries_arg:
        ts = Timeseries(id=timeseries_arg[key])
        if ('append_only' in opts and opts['append_only']) \
                         and opts['start_date'] is not None:
            ts.read_from_db(db, bottom_only=True)
            if ts.bounding_dates()[0]>opts['start_date']:
                ts.read_from_db(db)
        else:
            ts.read_from_db(db)
        ts.time_step = read_tstep_func(ts.id)
        tseries_arg[key] = ts
    MultiTimeseriesProcess(method, tseries_arg, out_timeseries, opts)
    if 'append_only' in opts and opts['append_only']:
        out_timeseries.append_to_db(db=db, transaction=transaction,
                                    commit=commit)
    else:
        out_timeseries.write_to_db(db=db, transaction=transaction, 
                                   commit=commit)