Python oqvalidation.OqParam類代碼示例

def get_oqparam(job_ini, pkg=None, calculators=None):
    """
    Parse a dictionary of parameters from one or more INI-style config file.

    :param job_ini:
        Path to configuration file/archive or dictionary of parameters
    :param pkg:
        Python package where to find the configuration file (optional)
    :param calculators:
        Sequence of calculator names (optional) used to restrict the
        valid choices for `calculation_mode`
    :returns:
        An :class:`openquake.commonlib.oqvalidation.OqParam` instance
        containing the validate and casted parameters/values parsed from
        the job.ini file as well as a subdictionary 'inputs' containing
        absolute paths to all of the files referenced in the job.ini, keyed by
        the parameter name.
    """
    # UGLY: this is here to avoid circular imports
    from openquake.commonlib.calculators import base

    OqParam.calculation_mode.validator.choices = tuple(
        calculators or base.calculators)

    if isinstance(job_ini, dict):
        oqparam = OqParam(**job_ini)
    else:
        basedir = os.path.dirname(pkg.__file__) if pkg else ''
        inis = [os.path.join(basedir, ini) for ini in job_ini.split(',')]
        oqparam = OqParam(**get_params(inis))

    oqparam.validate()
    return oqparam

 def test_missing_export_dir(self):
     oq = OqParam(
         calculation_mode='event_based', inputs={},
         sites='0.1 0.2',
         intensity_measure_types='PGA',
         maximum_distance=400)
     oq.validate()
     self.assertEqual(oq.export_dir, os.path.expanduser('~'))

 def test_imts_and_imtls(self):
     oq = OqParam(
         calculation_mode='event_based', inputs={},
         intensity_measure_types_and_levels="{'PGA': [0.1, 0.2]}",
         intensity_measure_types='PGV', sites='0.1 0.2',
         maximum_distance=400)
     oq.validate()
     self.assertEqual(list(oq.imtls.keys()), ['PGA'])

 def test_missing_export_dir(self):
     oq = OqParam(
         calculation_mode='event_based', inputs=GST,
         sites='0.1 0.2',
         intensity_measure_types='PGA',
         reference_vs30_value='200',
         maximum_distance='400')
     oq.validate()
     self.assertEqual(oq.export_dir, os.getcwd())

def show(calc_id, key=None, rlzs=None):
    """
    Show the content of a datastore.

    :param calc_id: numeric calculation ID; if 0, show all calculations
    :param key: key of the datastore
    :param rlzs: flag; if given, print out the realizations in order
    """
    if not calc_id:
        if not os.path.exists(datastore.DATADIR):
            return
        rows = []
        for calc_id in datastore.get_calc_ids(datastore.DATADIR):
            try:
                oq = OqParam.from_(datastore.DataStore(calc_id).attrs)
                cmode, descr = oq.calculation_mode, oq.description
            except:  # invalid datastore directory
                logging.warn('Removed invalid calculation %d', calc_id)
                shutil.rmtree(os.path.join(
                    datastore.DATADIR, 'calc_%s' % calc_id))
            else:
                rows.append((calc_id, cmode, descr))
        for row in sorted(rows, key=lambda row: row[0]):  # by calc_id
            print('#%d %s: %s' % row)
        return
    ds = datastore.DataStore(calc_id)
    if key:
        if key in datastore.view:
            print(datastore.view(key, ds))
            return
        obj = ds[key]
        if hasattr(obj, 'value'):  # an array
            print(write_csv(io.StringIO(), obj.value))
        else:
            print(obj)
        return
    # print all keys
    oq = OqParam.from_(ds.attrs)
    print(oq.calculation_mode, 'calculation (%r) saved in %s contains:' %
          (oq.description, ds.hdf5path))
    for key in ds:
        print(key, humansize(ds.getsize(key)))

    # this part is experimental and not tested on purpose
    if rlzs and 'curves_by_trt_gsim' in ds:
        min_value = 0.01  # used in rmsep
        curves_by_rlz, mean_curves = combined_curves(ds)
        dists = []
        for rlz in sorted(curves_by_rlz):
            curves = curves_by_rlz[rlz]
            dist = sum(rmsep(mean_curves[imt], curves[imt], min_value)
                       for imt in mean_curves.dtype.fields)
            dists.append((dist, rlz))
        for dist, rlz in sorted(dists):
            print('rlz=%s, rmsep=%s' % (rlz, dist))

    def pre_execute(self):
        """
        Check if there is a pre_calculator or a previous calculation ID.
        If yes, read the inputs by invoking the precalculator or by retrieving
        the previous calculation; if not, read the inputs directly.
        """
        if self.pre_calculator is not None:
            # the parameter hazard_calculation_id is only meaningful if
            # there is a precalculator
            precalc_id = self.oqparam.hazard_calculation_id
            if precalc_id is None:  # recompute everything
                precalc = calculators[self.pre_calculator](
                    self.oqparam, self.monitor('precalculator'),
                    self.datastore.calc_id)
                precalc.run(clean_up=False)
                if 'scenario' not in self.oqparam.calculation_mode:
                    self.csm = precalc.csm
            else:  # read previously computed data
                self.datastore.set_parent(datastore.DataStore(precalc_id))
                # update oqparam with the attributes saved in the datastore
                self.oqparam = OqParam.from_(self.datastore.attrs)
                self.read_exposure_sitecol()

        else:  # we are in a basic calculator
            self.read_exposure_sitecol()
            self.read_sources()
        self.datastore.hdf5.flush()

def export_ses_xml(ekey, dstore):
    """
    :param ekey: export key, i.e. a pair (datastore key, fmt)
    :param dstore: datastore object
    """
    fmt = ekey[-1]
    oq = OqParam.from_(dstore.attrs)
    try:
        csm_info = dstore['rlzs_assoc'].csm_info
    except AttributeError:  # for scenario calculators don't export
        return []
    sescollection = dstore['sescollection']
    col_id = 0
    fnames = []
    for sm in csm_info.source_models:
        for trt_model in sm.trt_models:
            sesruptures = list(sescollection[col_id].values())
            col_id += 1
            ses_coll = SESCollection(
                groupby(sesruptures, operator.attrgetter('ses_idx')),
                sm.path, oq.investigation_time)
            smpath = '_'.join(sm.path)
            fname = 'ses-%d-smltp_%s.%s' % (trt_model.id, smpath, fmt)
            dest = os.path.join(dstore.export_dir, fname)
            globals()['_export_ses_' + fmt](dest, ses_coll)
            fnames.append(os.path.join(dstore.export_dir, fname))
    return fnames

def export_gmf(ekey, dstore):
    """
    :param ekey: export key, i.e. a pair (datastore key, fmt)
    :param dstore: datastore object
    """
    sitecol = dstore['sitecol']
    rlzs_assoc = dstore['rlzs_assoc']
    rupture_by_tag = sum(dstore['sescollection'], AccumDict())
    all_tags = dstore['tags'].value
    oq = OqParam.from_(dstore.attrs)
    investigation_time = (None if oq.calculation_mode == 'scenario'
                          else oq.investigation_time)
    samples = oq.number_of_logic_tree_samples
    fmt = ekey[-1]
    gmfs = dstore[ekey[0]]
    nbytes = gmfs.attrs['nbytes']
    logging.info('Internal size of the GMFs: %s', humansize(nbytes))
    if nbytes > GMF_MAX_SIZE:
        logging.warn(GMF_WARNING, dstore.hdf5path)
    fnames = []
    for rlz, gmf_by_idx in zip(
            rlzs_assoc.realizations, rlzs_assoc.combine_gmfs(gmfs)):
        tags = all_tags[list(gmf_by_idx)]
        gmfs = list(gmf_by_idx.values())
        if not gmfs:
            continue
        ruptures = [rupture_by_tag[tag] for tag in tags]
        fname = build_name(dstore, rlz, 'gmf', fmt, samples)
        fnames.append(fname)
        globals()['export_gmf_%s' % fmt](
            ('gmf', fmt), fname, sitecol,
            ruptures, gmfs, rlz, investigation_time)
    return fnames

def export_hcurves_xml_json(ekey, dstore):
    export_type = ekey[1]
    len_ext = len(export_type) + 1
    oq = OqParam.from_(dstore.attrs)
    sitemesh = dstore['sitemesh'].value
    rlzs_assoc = dstore['rlzs_assoc']
    fnames = []
    writercls = (hazard_writers.HazardCurveGeoJSONWriter
                 if export_type == 'geojson' else
                 hazard_writers.HazardCurveXMLWriter)
    rlzs = iter(rlzs_assoc.realizations)
    for kind, curves in dstore[ekey[0]].items():
        rlz = next(rlzs)
        name = hazard_curve_name(
            dstore, ekey, kind, rlzs_assoc, oq.number_of_logic_tree_samples)
        for imt in oq.imtls:
            fname = name[:-len_ext] + '-' + imt + '.' + export_type
            data = [HazardCurve(Location(site), poes[imt])
                    for site, poes in zip(sitemesh, curves)]
            writer = writercls(fname, investigation_time=oq.investigation_time,
                               imls=oq.imtls[imt],
                               smlt_path='_'.join(rlz.sm_lt_path),
                               gsimlt_path=rlz.gsim_rlz.uid)
            writer.serialize(data)
            fnames.append(fname)
    return sorted(fnames)

def export_agg_curve(ekey, dstore):
    oq = OqParam.from_(dstore.attrs)
    cost_types = dstore['cost_types']
    rlzs = dstore['rlzs_assoc'].realizations
    agg_curve = dstore[ekey[0]]
    fnames = []
    L, R = len(cost_types), len(rlzs)
    for ct in cost_types:
        loss_type = ct['name']
        array = agg_curve[loss_type].value
        for ins in range(oq.insured_losses + 1):
            for rlz in rlzs:
                suffix = '' if L == 1 and R == 1 else '-gsimltp_%s_%s' % (
                    rlz.uid, loss_type)
                dest = dstore.export_path('agg_curve%s%s.%s' % (
                    suffix, '_ins' if ins else '', ekey[1]))
                rec = array[rlz.ordinal, ins]
                curve = AggCurve(rec['losses'], rec['poes'], rec['avg'], None)
                risk_writers.AggregateLossCurveXMLWriter(
                    dest, oq.investigation_time, loss_type,
                    source_model_tree_path='_'.join(rlz.sm_lt_path),
                    gsim_tree_path='_'.join(rlz.gsim_lt_path),
                    unit=ct['unit']).serialize(curve)
                fnames.append(dest)
    return sorted(fnames)

def get_data_transfer(dstore):
    """
    Determine the amount of data transferred from the controller node
    to the workers and back in a classical calculation.

    :param dstore: a :class:`openquake.commonlib.datastore.DataStore` instance
    :returns: (block_info, to_send_forward, to_send_back)
    """
    oqparam = OqParam.from_(dstore.attrs)
    sitecol = dstore['sitecol']
    rlzs_assoc = dstore['rlzs_assoc']
    info = dstore['job_info']
    sources = dstore['composite_source_model'].get_sources()
    num_gsims_by_trt = groupby(rlzs_assoc, operator.itemgetter(0),
                               lambda group: sum(1 for row in group))
    gsims_assoc = rlzs_assoc.gsims_by_trt_id
    to_send_forward = 0
    to_send_back = 0
    block_info = []
    for block in split_in_blocks(sources, oqparam.concurrent_tasks or 1,
                                 operator.attrgetter('weight'),
                                 operator.attrgetter('trt_model_id')):
        num_gsims = num_gsims_by_trt.get(block[0].trt_model_id, 0)
        back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
        to_send_back += back * 8  # 8 bytes per float
        args = (block, sitecol, gsims_assoc, PerformanceMonitor(''))
        to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
        block_info.append((len(block), block.weight))
    return numpy.array(block_info, block_dt), to_send_forward, to_send_back

 def get_oqparam(self):
     """
     Return an OqParam object as read from the database
     """
     datadir = os.path.dirname(self.ds_calc_dir)
     dstore = datastore.read(self.id, datadir=datadir)
     oqparam = OqParam.from_(dstore.attrs)
     return oqparam

    def test_missing_maximum_distance(self):
        with self.assertRaises(ValueError):
            OqParam(
                calculation_mode='classical_risk', inputs=dict(site_model=''),
                hazard_calculation_id=None, hazard_output_id=None,
                sites='0.1 0.2').validate()

        with self.assertRaises(ValueError):
            OqParam(
                calculation_mode='classical_risk', inputs=dict(site_model=''),
                hazard_calculation_id=None, hazard_output_id=None,
                sites='0.1 0.2', maximum_distance='0').validate()

        oq = OqParam(
            calculation_mode='event_based', inputs=GST,
            intensity_measure_types_and_levels="{'PGA': [0.1, 0.2]}",
            intensity_measure_types='PGV', sites='0.1 0.2',
            reference_vs30_value='200',
            maximum_distance='{"wrong TRT": 200}')
        oq.inputs['source_model_logic_tree'] = 'something'

        oq._gsims_by_trt = {'Active Shallow Crust': []}
        self.assertFalse(oq.is_valid_maximum_distance())
        self.assertIn('setting the maximum_distance for wrong TRT', oq.error)

        oq._gsims_by_trt = {'Active Shallow Crust': [],
                            'Stable Continental Crust': []}
        oq.maximum_distance = {'Active Shallow Crust': 200}
        self.assertFalse(oq.is_valid_maximum_distance())
        self.assertEqual('missing distance for Stable Continental Crust '
                         'and no default', oq.error)

def view_params(token, dstore):
    oq = OqParam.from_(dstore.attrs)
    params = ('calculation_mode', 'number_of_logic_tree_samples',
              'maximum_distance', 'investigation_time',
              'ses_per_logic_tree_path', 'truncation_level',
              'rupture_mesh_spacing', 'complex_fault_mesh_spacing',
              'width_of_mfd_bin', 'area_source_discretization',
              'random_seed', 'master_seed', 'concurrent_tasks')
    return rst_table([(param, getattr(oq, param)) for param in params])

def view_inputs(token, dstore):
    inputs = OqParam.from_(dstore.attrs).inputs.copy()
    try:
        source_models = [('source', fname) for fname in inputs['source']]
        del inputs['source']
    except KeyError:  # there is no 'source' in scenario calculations
        source_models = []
    return rst_table(
        build_links(list(inputs.items()) + source_models),
        header=['Name', 'File'])