Python Session.rollback方法代码示例

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        sess = None
        try:
            self._log(
                "Starting to parse the file with '" + self.parser_name + "'.")
            sess = Session()
            g_batch = GBatch.get_by_id(sess, self.g_batch_id)
            raw_bills = self.parser.make_raw_bills()
            self._log(
                "Successfully parsed the file, and now I'm starting to "
                "insert the raw bills.")
            for self.bill_num, raw_bill in enumerate(raw_bills):
                try:
                    bill_type = BillType.get_by_code(
                        sess, raw_bill['bill_type_code'])
                    g_bill = g_batch.insert_g_bill(
                        sess, bill_type, raw_bill['mprn'],
                        raw_bill['reference'], raw_bill['account'],
                        raw_bill['issue_date'], raw_bill['start_date'],
                        raw_bill['finish_date'], raw_bill['kwh'],
                        raw_bill['net_gbp'], raw_bill['vat_gbp'],
                        raw_bill['gross_gbp'], raw_bill['raw_lines'],
                        raw_bill['breakdown'])
                    sess.flush()
                    for raw_read in raw_bill['reads']:
                        prev_type = GReadType.get_by_code(
                            sess, raw_read['prev_type_code'])
                        pres_type = GReadType.get_by_code(
                            sess, raw_read['pres_type_code'])
                        g_units = GUnits.get_by_code(sess, raw_read['units'])
                        g_read = g_bill.insert_g_read(
                            sess, raw_read['msn'], raw_read['prev_value'],
                            raw_read['prev_date'], prev_type,
                            raw_read['pres_value'], raw_read['pres_date'],
                            pres_type, g_units, raw_read['correction_factor'],
                            raw_read['calorific_value'])

                        sess.expunge(g_read)
                    sess.commit()
                    self.successful_bills.append(raw_bill)
                    sess.expunge(g_bill)
                except BadRequest as e:
                    sess.rollback()
                    raw_bill['error'] = e.description
                    self.failed_bills.append(raw_bill)

            if len(self.failed_bills) == 0:
                self._log(
                    "All the bills have been successfully loaded and attached "
                    "to the batch.")
            else:
                self._log(
                    "The import has finished, but " +
                    str(len(self.failed_bills)) + " bills failed to load.")

        except:
            self._log("I've encountered a problem: " + traceback.format_exc())
        finally:
            if sess is not None:
                sess.close()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.run_inner(sess)
                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking GCv rates.")

            self.going.wait(30 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check BSUoS rates.")
                    contract = Contract.get_non_core_by_name(sess, 'bsuos')
                    latest_rs = sess.query(RateScript).filter(
                        RateScript.contract == contract).order_by(
                        RateScript.start_date.desc()).first()
                    latest_rs_id = latest_rs.id
                    this_month_start = latest_rs.start_date + \
                        relativedelta(months=1)
                    next_month_start = this_month_start + \
                        relativedelta(months=1)
                    now = Datetime.now(pytz.utc)
                    props = contract.make_properties()
                    if props.get('enabled', False):

                        if now > next_month_start:
                            url = props['url']
                            self.log(
                                "Checking to see if data is available from " +
                                str(this_month_start) + " to " +
                                str(next_month_start - HH) +
                                " at " + url)
                            res = requests.get(url)
                            self.log(
                                "Received " + str(res.status_code) + " " +
                                res.reason)
                            book = xlrd.open_workbook(
                                file_contents=res.content)
                            sheet = book.sheet_by_index(0)

                            ct_tz = pytz.timezone('Europe/London')

                            month_bsuos = {}
                            for row_index in range(1, sheet.nrows):
                                row = sheet.row(row_index)
                                raw_date = Datetime(
                                    *xlrd.xldate_as_tuple(
                                        row[0].value, book.datemode))
                                hh_date_ct = ct_tz.localize(raw_date)
                                hh_date = pytz.utc.normalize(
                                    hh_date_ct.astimezone(pytz.utc))
                                hh_date += relativedelta(
                                    minutes=30*int(row[1].value))
                                if not hh_date < this_month_start and \
                                        hh_date < next_month_start:
                                    month_bsuos[key_format(hh_date)] = \
                                        row[2].value

                            if key_format(next_month_start - HH) in \
                                    month_bsuos:
                                self.log("The whole month's data is there.")
                                script = "def rates_gbp_per_mwh():\n    " \
                                    "return {\n" + ',\n'.join(
                                        "'" + k + "': " + str(month_bsuos[k])
                                        for k in sorted(
                                            month_bsuos.keys())) + "}"
                                set_read_write(sess)
                                contract = Contract.get_non_core_by_name(
                                    sess, 'bsuos')
                                rs = RateScript.get_by_id(sess, latest_rs_id)
                                contract.update_rate_script(
                                    sess, rs, rs.start_date,
                                    rs.start_date + relativedelta(months=2) -
                                    HH, rs.script)
                                sess.flush()
                                contract.insert_rate_script(
                                    sess,
                                    rs.start_date + relativedelta(months=1),
                                    script)
                                sess.commit()
                                self.log("Added new rate script.")
                            else:
                                self.log(
                                    "There isn't a whole month there yet. The "
                                    "last date is " +
                                    sorted(month_bsuos.keys())[-1])
                    else:
                        self.log(
                            "The automatic importer is disabled. To "
                            "enable it, edit the contract properties to "
                            "set 'enabled' to True.")
                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking BSUoS rates.")

            self.going.wait(30 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check TLMs.")
                    contract = Contract.get_non_core_by_name(sess, 'tlms')
                    latest_rs = sess.query(RateScript).filter(
                        RateScript.contract_id == contract.id).order_by(
                        RateScript.start_date.desc()).first()
                    latest_rs_id = latest_rs.id
                    next_month_start = latest_rs.start_date + \
                        relativedelta(months=1)
                    next_month_finish = latest_rs.start_date + \
                        relativedelta(months=2) - HH

                    now = utc_datetime_now()
                    if now > next_month_start:
                        self.log(
                            "Checking to see if data is available from " +
                            str(next_month_start) + " to " +
                            str(next_month_finish) + " on Elexon Portal.")
                        config = Contract.get_non_core_by_name(
                            sess, 'configuration')
                        props = config.make_properties()

                        scripting_key = props.get(
                            ELEXON_PORTAL_SCRIPTING_KEY_KEY)
                        if scripting_key is None:
                            raise BadRequest(
                                "The property " +
                                ELEXON_PORTAL_SCRIPTING_KEY_KEY +
                                " cannot be found in the configuration " +
                                "properties.")

                        contract_props = contract.make_properties()
                        url_str = ''.join(
                            (
                                contract_props['url'],
                                'file/download/TLM_FILE?key=',
                                scripting_key))

                        r = requests.get(url_str)
                        parser = csv.reader(
                            (l.decode() for l in r.iter_lines()),
                            delimiter=',', quotechar='"')
                        self.log("Opened " + url_str + ".")

                        next(parser, None)
                        month_tlms = {}
                        for values in parser:
                            hh_date_ct = to_ct(
                                Datetime.strptime(values[0], "%d/%m/%Y"))
                            hh_date = to_utc(hh_date_ct)
                            hh_date += relativedelta(minutes=30*int(values[2]))
                            if next_month_start <= hh_date <= \
                                    next_month_finish:
                                month_tlms[key_format(hh_date)] = {
                                    'off-taking': values[3],
                                    'delivering': values[4]}

                        if key_format(next_month_finish) in month_tlms:
                            self.log("The whole month's data is there.")
                            script = "def tlms():\n    return {\n" + \
                                ',\n'.join(
                                    "'" + k + "': " +
                                    month_tlms[k]['off-taking'] for k in
                                    sorted(month_tlms.keys())) + "}"
                            contract = Contract.get_non_core_by_name(
                                sess, 'tlms')
                            rs = RateScript.get_by_id(sess, latest_rs_id)
                            contract.update_rate_script(
                                sess, rs, rs.start_date,
                                rs.start_date + relativedelta(months=2) - HH,
                                rs.script)
                            sess.flush()
                            contract.insert_rate_script(
                                sess, rs.start_date + relativedelta(months=1),
                                script)
                            sess.commit()
                            self.log("Added new rate script.")
                        else:
                            msg = "There isn't a whole month there yet."
                            if len(month_tlms) > 0:
                                msg += "The last date is " + \
                                    sorted(month_tlms.keys())[-1]
                            self.log(msg)
                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking TLM rates.")

            self.going.wait(30 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check bank holidays")
                    contract = Contract.get_non_core_by_name(
                        sess, 'bank_holidays')
                    contract_props = contract.make_properties()

                    if contract_props.get('enabled', False):
                        url_str = contract_props['url']

                        self.log("Downloading from " + url_str + ".")
                        res = requests.get(url_str)
                        self.log(
                            ' '.join(
                                (
                                    "Received", str(res.status_code),
                                    res.reason)))
                        PREFIX = 'DTSTART;VALUE=DATE:'
                        hols = collections.defaultdict(list)
                        for line in res.text.splitlines():
                            if line.startswith(PREFIX):
                                dt = utc_datetime_parse(line[-8:], "%Y%m%d")
                                hols[dt.year].append(dt)

                        for year in sorted(hols.keys()):
                            year_start = utc_datetime(year, 1, 1)
                            year_finish = year_start + \
                                relativedelta(years=1) - HH
                            rs = sess.query(RateScript).filter(
                                RateScript.contract == contract,
                                RateScript.start_date == year_start).first()
                            if rs is None:
                                self.log(
                                    "Adding a new rate script starting at " +
                                    hh_format(year_start) + ".")

                                latest_rs = sess.query(RateScript).filter(
                                    RateScript.contract == contract).\
                                    order_by(RateScript.start_date.desc()). \
                                    first()

                                contract.update_rate_script(
                                    sess, latest_rs, latest_rs.start_date,
                                    year_finish, latest_rs.script)
                                rs = contract.insert_rate_script(
                                    sess, year_start, '')

                            script = {
                                'bank_holidays': [
                                    v.strftime("%Y-%m-%d")
                                    for v in hols[year]]}

                            self.log(
                                "Updating rate script starting at " +
                                hh_format(year_start) + ".")
                            contract.update_rate_script(
                                sess, rs, rs.start_date, rs.finish_date,
                                json.dumps(
                                    script, indent='    ', sort_keys=True))
                            sess.commit()
                    else:
                        self.log(
                            "The automatic importer is disabled. To "
                            "enable it, edit the contract properties to "
                            "set 'enabled' to True.")

                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    if sess is not None:
                        sess.close()
                    self.lock.release()
                    self.log("Finished checking bank holidays.")

            self.going.wait(24 * 60 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]

#.........这里部分代码省略.........

                    for pair in pairs:
                        pair_hhs = (
                            (
                                pair['finish-date'] - pair['start-date']
                            ).total_seconds() + 30 * 60) / (60 * 30)
                        pair['pair_hhs'] = pair_hhs
                        for tpr_code, pair_kwh in pair['tprs'].items():
                            total_kwh[meter_type] += pair_kwh * pair_hhs

                    breakdown += 'pairs - \n' + str(pairs)

                elif meter_type in ('hh', 'amr'):
                    period_kwhs = list(
                        float(v[0]) for v in sess.query(HhDatum.value).
                        join(Channel).filter(
                            Channel.imp_related == true(),
                            Channel.channel_type == 'ACTIVE',
                            Channel.era == era,
                            HhDatum.start_date >= period_start,
                            HhDatum.start_date <= period_finish).order_by(
                                HhDatum.id))
                    year_kwhs = list(
                        float(v[0]) for v in sess.query(HhDatum.value).
                        join(Channel).join(Era).filter(
                            Channel.imp_related == true(),
                            Channel.channel_type == 'ACTIVE',
                            Era.supply == supply,
                            HhDatum.start_date >= year_start,
                            HhDatum.start_date <= year_finish).order_by(
                                HhDatum.id))

                    period_sum_kwhs = sum(period_kwhs)
                    year_sum_kwhs = sum(year_kwhs)
                    period_len_kwhs = len(period_kwhs)
                    year_len_kwhs = len(year_kwhs)
                    total_kwh[meter_type] += period_sum_kwhs
                    period_hhs = (
                        period_finish + HH - period_start
                        ).total_seconds() / (60 * 30)
                    if year_len_kwhs > 0:
                        filled_kwh[meter_type] += year_sum_kwhs / \
                            year_len_kwhs * (period_hhs - period_len_kwhs)
                    normal_days[meter_type] += sess.query(
                        func.count(HhDatum.value)).join(Channel). \
                        filter(
                            Channel.imp_related == true(),
                            Channel.channel_type == 'ACTIVE',
                            Channel.era == era,
                            HhDatum.start_date >= period_start,
                            HhDatum.start_date <= period_finish,
                            HhDatum.status == 'A').one()[0] / 48
                elif meter_type == 'unmetered':
                    bills = sess.query(Bill).filter(
                        Bill.supply == supply,
                        Bill.finish_date >= period_start,
                        Bill.start_date <= period_finish)
                    for bill in bills:
                        total_kwh[meter_type] += kwh
                    normal_days[meter_type] += (
                        (
                            period_finish - period_start).total_seconds() +
                        60 * 30) / (60 * 60 * 24)

            # for full year 183
            total_normal_days = sum(normal_days.values())
            total_max_normal_days = sum(max_normal_days.values())
            is_normal = total_normal_days / total_max_normal_days >= 183 / 365

            f.write(
                ','.join(
                    '"' + str(val) + '"' for val in
                    [
                        supply.id, mpan_core, site.code, site.name,
                        hh_format(year_start), hh_format(year_finish),
                        breakdown] +
                    [
                        normal_days[type] for type in meter_types] +
                    [
                        max_normal_days[type] for type in meter_types] +
                    [
                        total_normal_days, total_max_normal_days,
                        "Actual" if is_normal else "Estimated"] +
                    [total_kwh[type] for type in meter_types] +
                    [filled_kwh[type] for type in ('hh', 'amr')] +
                    [sum(total_kwh.values()) + sum(filled_kwh.values()), '']) +
                '\n')

            # avoid a long running transaction
            sess.rollback()
    except:
        msg = traceback.format_exc()
        sys.stderr.write(msg + '\n')
        f.write("Problem " + msg)
    finally:
        if sess is not None:
            sess.close()
        if f is not None:
            f.close()
            os.rename(running_name, finished_name)

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]

#.........这里部分代码省略.........
                            month_data['import-3rd-party-gbp'] += gbp
                        else:
                            month_data['import-net-gbp'] += gbp
                        month_data['used-gbp'] += gbp

                        if source_code in ('gen', 'gen-net'):
                            generator_type = supply.generator_type.code
                            site_gen_types.add(generator_type)
                        else:
                            generator_type = None

                        sup_category = era.make_meter_category()
                        if CATEGORY_ORDER[site_category] < \
                                CATEGORY_ORDER[sup_category]:
                            site_category = sup_category

                        for bill in sess.query(Bill).filter(
                                Bill.supply == supply,
                                Bill.start_date <= sss.finish_date,
                                Bill.finish_date >= sss.start_date):
                            bill_start = bill.start_date
                            bill_finish = bill.finish_date
                            bill_duration = (
                                bill_finish - bill_start).total_seconds() + \
                                (30 * 60)
                            overlap_duration = (
                                min(bill_finish, sss.finish_date) -
                                max(bill_start, sss.start_date)
                                ).total_seconds() + (30 * 60)
                            overlap_proportion = \
                                float(overlap_duration) / bill_duration
                            month_data['billed-import-net-kwh'] += \
                                overlap_proportion * float(bill.kwh)
                            month_data['billed-import-net-gbp'] += \
                                overlap_proportion * float(bill.net)

                        out = [
                            era.imp_mpan_core, era.exp_mpan_core,
                            sup_category, source_code,
                            generator_type, supply.name, era.msn, era.pc.code,
                            site.code, site.name,
                            ','.join(sorted(list(site_associates))),
                            month_finish] + [
                            month_data[t] for t in summary_titles] + [None] + [
                            (mop_bill[t] if t in mop_bill else None)
                            for t in title_dict['mop']] + [None] + \
                            [(dc_bill[t] if t in dc_bill else None)
                                for t in title_dict['dc']]
                        if imp_supplier_contract is None:
                            out += [None] * \
                                (len(title_dict['imp-supplier']) + 1)
                        else:
                            out += [None] + [
                                (
                                    imp_supplier_bill[t]
                                    if t in imp_supplier_bill else None)
                                for t in title_dict['imp-supplier']]
                        if exp_supplier_contract is not None:
                            out += [None] + [
                                (
                                    exp_supplier_bill[t]
                                    if t in exp_supplier_bill else None)
                                for t in title_dict['exp-supplier']]

                        for k, v in month_data.items():
                            site_month_data[k] += v
                        sup_tab.writerow(out)

                group_tab.writerow(
                    [
                        site.code, site.name,
                        ''.join(sorted(list(site_associates))),
                        month_finish, site_category,
                        ', '.join(sorted(list(site_sources))),
                        ', '.join(sorted(list(site_gen_types)))] +
                    [site_month_data[k] for k in summary_titles])
                sess.rollback()

            month_start += relativedelta(months=1)
    except BadRequest as e:
        msg = e.description + traceback.format_exc()
        sys.stderr.write(msg + '\n')
        group_tab.writerow(["Problem " + msg])
    except:
        msg = traceback.format_exc()
        sys.stderr.write(msg + '\n')
        group_tab.writerow(["Problem " + msg])
    finally:
        if sess is not None:
            sess.close()
        try:
            f.close()
            rf.close()
            os.rename(running_name, finished_name)
        except:
            msg = traceback.format_exc()
            r_name, f_name = chellow.dloads.make_names('error.txt', user)
            ef = open(r_name, "w")
            ef.write(msg + '\n')
            ef.close()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]

#.........这里部分代码省略.........
                        if url.scheme == 'https':
                            conn = http.client.HTTPSConnection(
                                url.hostname, url.port)
                        else:
                            conn = http.client.HTTPConnection(
                                url.hostname, url.port)
                        conn.request("GET", url.path + '?' + url.query)

                        res = conn.getresponse()
                        self.log(
                            "Received " + str(res.status) + " " + res.reason)
                        data = res.read()
                        book = xlrd.open_workbook(file_contents=data)
                        sbp_sheet = book.sheet_by_index(1)
                        ssp_sheet = book.sheet_by_index(2)

                        sp_months = []
                        sp_month = None
                        for row_index in range(1, sbp_sheet.nrows):
                            sbp_row = sbp_sheet.row(row_index)
                            ssp_row = ssp_sheet.row(row_index)
                            raw_date = datetime.datetime(
                                *xlrd.xldate_as_tuple(
                                    sbp_row[0].value, book.datemode))
                            hh_date_ct = to_ct(raw_date)
                            hh_date = to_utc(hh_date_ct)
                            run_code = sbp_row[1].value
                            for col_idx in range(2, 52):
                                if hh_date >= fill_start:
                                    sbp_val = sbp_row[col_idx].value
                                    if sbp_val != '':
                                        if hh_date.day == 1 and \
                                                hh_date.hour == 0 and \
                                                hh_date.minute == 0:
                                            sp_month = {}
                                            sp_months.append(sp_month)
                                        ssp_val = ssp_row[col_idx].value
                                        sp_month[hh_date] = {
                                            'run': run_code,
                                            'sbp': sbp_val, 'ssp': ssp_val}
                                hh_date += HH
                        self.log("Successfully extracted data.")
                        last_date = sorted(sp_months[-1].keys())[-1]
                        if last_date.month == (last_date + HH).month:
                            del sp_months[-1]
                        if 'limit' in contract_props:
                            sp_months = sp_months[0:1]
                        for sp_month in sp_months:
                            sorted_keys = sorted(sp_month.keys())
                            month_start = sorted_keys[0]
                            month_finish = sorted_keys[-1]
                            rs = sess.query(RateScript).filter(
                                RateScript.contract == contract,
                                RateScript.start_date == month_start).first()
                            if rs is None:
                                self.log(
                                    "Adding a new rate script starting at " +
                                    hh_format(month_start) + ".")

                                latest_rs = sess.query(RateScript).filter(
                                    RateScript.contract == contract).\
                                    order_by(RateScript.start_date.desc()). \
                                    first()

                                contract.update_rate_script(
                                    sess, latest_rs, latest_rs.start_date,
                                    month_finish, latest_rs.script)
                                rs = contract.insert_rate_script(
                                    sess, month_start, '')
                                sess.flush()
                            script = {
                                'gbp_per_nbp_mwh': dict(
                                    (key_format(k), v)
                                    for k, v in sp_month.items())}
                            self.log(
                                "Updating rate script starting at " +
                                hh_format(month_start) + ".")
                            contract.update_rate_script(
                                sess, rs, rs.start_date, rs.finish_date,
                                json.dumps(
                                    script, indent='    ', sort_keys=True))
                            sess.commit()
                    else:
                        self.log(
                            "The automatic importer is disabled. To "
                            "enable it, edit the contract properties to "
                            "set 'enabled' to True.")

                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    book = sbp_sheet = ssp_sheet = None
                    self.lock.release()
                    self.log("Finished checking System Price rates.")
                    if sess is not None:
                        sess.close()

            self.going.wait(24 * 60 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        sess = None
        try:
            sess = Session()
            self._log(
                "Starting to parse the file with '" + self.parser_name + "'.")
            set_read_write(sess)
            batch = Batch.get_by_id(sess, self.batch_id)
            raw_bills = self.parser.make_raw_bills()
            self._log(
                "Successfully parsed the file, and now I'm starting to "
                "insert the raw bills.")
            for self.bill_num, raw_bill in enumerate(raw_bills):
                try:
                    with sess.begin_nested():
                        sess.execute(
                            "set transaction isolation level serializable "
                            "read write")
                        bill_type = BillType.get_by_code(
                            sess, raw_bill['bill_type_code'])
                        bill = batch.insert_bill(
                            sess, raw_bill['account'], raw_bill['reference'],
                            raw_bill['issue_date'], raw_bill['start_date'],
                            raw_bill['finish_date'], raw_bill['kwh'],
                            raw_bill['net'], raw_bill['vat'],
                            raw_bill['gross'],
                            bill_type, raw_bill['breakdown'])
                        sess.flush()
                        for raw_read in raw_bill['reads']:
                            tpr_code = raw_read['tpr_code']
                            if tpr_code is None:
                                tpr = None
                            else:
                                tpr = Tpr.get_by_code(sess, tpr_code)

                            prev_type = ReadType.get_by_code(
                                sess, raw_read['prev_type_code'])
                            pres_type = ReadType.get_by_code(
                                sess, raw_read['pres_type_code'])
                            bill.insert_read(
                                sess, tpr, raw_read['coefficient'],
                                raw_read['units'], raw_read['msn'],
                                raw_read['mpan'], raw_read['prev_date'],
                                raw_read['prev_value'], prev_type,
                                raw_read['pres_date'], raw_read['pres_value'],
                                pres_type)
                        self.successful_bills.append(raw_bill)
                except BadRequest as e:
                    raw_bill['error'] = str(e.description)
                    self.failed_bills.append(raw_bill)

            if len(self.failed_bills) == 0:
                sess.commit()
                self._log(
                    "All the bills have been successfully loaded and attached "
                    "to the batch.")
            else:
                sess.rollback()
                self._log(
                    "The import has finished, but there were " +
                    str(len(self.failed_bills)) + " failures, and so the "
                    "whole import has been rolled back.")

        except:
            sess.rollback()
            self._log("I've encountered a problem: " + traceback.format_exc())
        finally:
            if sess is not None:
                sess.close()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        while not self.stopped.isSet():
            if self.lock.acquire(False):
                sess = None
                try:
                    sess = Session()
                    self.log("Starting to check RCRCs.")
                    contract = Contract.get_non_core_by_name(sess, 'rcrc')
                    latest_rs = sess.query(RateScript).filter(
                        RateScript.contract_id == contract.id).order_by(
                        RateScript.start_date.desc()).first()
                    latest_rs_id = latest_rs.id
                    latest_rs_start = latest_rs.start_date

                    month_start = latest_rs_start + relativedelta(months=1)
                    month_finish = month_start + relativedelta(months=1) - HH
                    now = Datetime.now(pytz.utc)
                    if now > month_finish:
                        self.log(
                            "Checking to see if data is available from " +
                            str(month_start) + " to " + str(month_finish) +
                            " on Elexon Portal.")
                        config = Contract.get_non_core_by_name(
                            sess, 'configuration')
                        props = config.make_properties()

                        scripting_key = props.get(
                            ELEXON_PORTAL_SCRIPTING_KEY_KEY)
                        if scripting_key is None:
                            raise BadRequest(
                                "The property " +
                                ELEXON_PORTAL_SCRIPTING_KEY_KEY +
                                " cannot be found in the configuration "
                                "properties.")

                        contract_props = contract.make_properties()
                        url_str = ''.join(
                            (
                                contract_props['url'],
                                'file/download/RCRC_FILE?key=',
                                scripting_key))

                        r = requests.get(url_str)
                        parser = csv.reader(
                            (l.decode() for l in r.iter_lines()),
                            delimiter=',', quotechar='"')
                        piterator = iter(parser)
                        values = next(piterator)
                        values = next(piterator)
                        month_rcrcs = {}
                        for values in piterator:
                            hh_date = Datetime.strptime(
                                values[0], "%d/%m/%Y").replace(tzinfo=pytz.utc)
                            hh_date += relativedelta(minutes=30*int(values[2]))
                            if month_start <= hh_date <= month_finish:
                                month_rcrcs[key_format(hh_date)] = values[3]

                        if key_format(month_finish) in month_rcrcs:
                            self.log("The whole month's data is there.")
                            script = "def rates():\n    return {\n" + \
                                ',\n'.join(
                                    "'" + k + "': " + month_rcrcs[k] for k in
                                    sorted(month_rcrcs.keys())) + "}"
                            set_read_write(sess)
                            contract = Contract.get_non_core_by_name(
                                sess, 'rcrc')
                            rs = RateScript.get_by_id(sess, latest_rs_id)
                            contract.update_rate_script(
                                sess, rs, rs.start_date, month_finish,
                                rs.script)
                            contract.insert_rate_script(
                                sess, month_start, script)
                            sess.commit()
                            self.log("Added new rate script.")
                        else:
                            msg = "There isn't a whole month there yet."
                            if len(month_rcrcs) > 0:
                                msg += " The last date is " + \
                                    sorted(month_rcrcs.keys())[-1]
                            self.log(msg)
                except:
                    self.log("Outer problem " + traceback.format_exc())
                    sess.rollback()
                finally:
                    self.lock.release()
                    self.log("Finished checking RCRC rates.")
                    if sess is not None:
                        sess.close()

            self.going.wait(30 * 60)
            self.going.clear()

# 需要导入模块: from chellow.models import Session [as 别名]
# 或者: from chellow.models.Session import rollback [as 别名]
    def run(self):
        sess = None
        try:
            sess = Session()
            self._log(
                "Starting to parse the file with '" + self.parser_name + "'.")

            bill_types = keydefaultdict(
                lambda k: BillType.get_by_code(sess, k))

            tprs = keydefaultdict(
                lambda k: None if k is None else Tpr.get_by_code(sess, k))

            read_types = keydefaultdict(
                lambda k: ReadType.get_by_code(sess, k))

            batch = Batch.get_by_id(sess, self.batch_id)
            contract = batch.contract
            raw_bills = self.parser.make_raw_bills()
            self._log(
                "Successfully parsed the file, and now I'm starting to "
                "insert the raw bills.")
            for self.bill_num, raw_bill in enumerate(raw_bills):
                try:
                    account = raw_bill['account']
                    supply = sess.query(Supply).join(Era).filter(
                        or_(
                            and_(
                                Era.imp_supplier_contract == contract,
                                Era.imp_supplier_account == account),
                            and_(
                                Era.exp_supplier_contract == contract,
                                Era.exp_supplier_account == account),
                            and_(
                                Era.mop_contract == contract,
                                Era.mop_account == account),
                            and_(
                                Era.hhdc_contract == contract,
                                Era.hhdc_account == account))
                        ).distinct().order_by(Supply.id).first()

                    if supply is None:
                        raise BadRequest(
                            "Can't find an era with contract '" +
                            contract.name + "' and account '" + account + "'.")
                    with sess.begin_nested():
                        bill = batch.insert_bill(
                            sess, account, raw_bill['reference'],
                            raw_bill['issue_date'], raw_bill['start_date'],
                            raw_bill['finish_date'], raw_bill['kwh'],
                            raw_bill['net'], raw_bill['vat'],
                            raw_bill['gross'],
                            bill_types[raw_bill['bill_type_code']],
                            raw_bill['breakdown'], supply)
                        for raw_read in raw_bill['reads']:
                            bill.insert_read(
                                sess, tprs[raw_read['tpr_code']],
                                raw_read['coefficient'], raw_read['units'],
                                raw_read['msn'], raw_read['mpan'],
                                raw_read['prev_date'], raw_read['prev_value'],
                                read_types[raw_read['prev_type_code']],
                                raw_read['pres_date'], raw_read['pres_value'],
                                read_types[raw_read['pres_type_code']])
                        self.successful_bills.append(raw_bill)
                except BadRequest as e:
                    raw_bill['error'] = str(e.description)
                    self.failed_bills.append(raw_bill)

            if len(self.failed_bills) == 0:
                sess.commit()
                self._log(
                    "All the bills have been successfully loaded and attached "
                    "to the batch.")
            else:
                sess.rollback()
                self._log(
                    "The import has finished, but there were " +
                    str(len(self.failed_bills)) + " failures, and so the "
                    "whole import has been rolled back.")

        except:
            sess.rollback()
            self._log("I've encountered a problem: " + traceback.format_exc())
        finally:
            if sess is not None:
                sess.close()