Python utils.read_json函数代码示例

    def setUpClass(self):

        # Directory of sample config files
        self.sample_resource_dir = os.path.dirname(os.path.realpath(__file__))

        # Load sample resource test config
        self.cfg_sample = ru.read_json(os.path.join(self.sample_resource_dir, "sample_resources.json"))
        self.cfg_sample_1 = self.cfg_sample['sample_resource_1']
        self.cfg_sample_2 = self.cfg_sample['sample_resource_2']
        self.cfg_sample_3 = self.cfg_sample['sample_resource_3']

        # Directory of pilot resource config files
        self.pilot_resource_dir = '../../src/radical/pilot/configs'

        # Load xsede pilot resource config
        self.cfg_xsede = ru.read_json(os.path.join(self.pilot_resource_dir, 'resource_xsede.json'))
        self.cfg_xsede_bridges = self.cfg_xsede['bridges']
        self.cfg_xsede_comet_ssh = self.cfg_xsede['comet_ssh']
        self.cfg_xsede_comet_orte = self.cfg_xsede['comet_orte']
        self.cfg_xsede_comet_ortelib = self.cfg_xsede['comet_ortelib']
        self.cfg_xsede_comet_spark = self.cfg_xsede['comet_spark']
        self.cfg_xsede_supermic_ssh = self.cfg_xsede['supermic_ssh']
        self.cfg_xsede_supermic_orte = self.cfg_xsede['supermic_orte']
        self.cfg_xsede_supermic_ortelib = self.cfg_xsede['supermic_ortelib']
        self.cfg_xsede_supermic_spark = self.cfg_xsede['supermic_spark']

        return

def get_session_docs (db, sid, cache=None, cachedir=None) :

    # session docs may have been cached in /tmp/rp_cache_<uid>/<sid>.json -- in that
    # case we pull it from there instead of the database, which will be much
    # quicker.  Also, we do cache any retrieved docs to that place, for later
    # use.  An optional cachdir parameter changes that default location for
    # lookup and storage.
    if  not cachedir :
        cachedir = _CACHE_BASEDIR

    if  not cache :
        cache = "%s/%s.json" % (cachedir, sid)

    try :
        if  os.path.isfile (cache) :
            return ru.read_json (cache)
    except Exception as e :
        # continue w/o cache
        sys.stderr.write ("warning: cannot read session cache at %s (%s)\n" % (cache, e))


    # cache not used or not found -- go to db
    json_data = dict()

    # convert bson to json, i.e. serialize the ObjectIDs into strings.
    json_data['session'] = bson2json (list(db["%s"    % sid].find ()))
    json_data['pmgr'   ] = bson2json (list(db["%s.pm" % sid].find ()))
    json_data['pilot'  ] = bson2json (list(db["%s.p"  % sid].find ()))
    json_data['umgr'   ] = bson2json (list(db["%s.um" % sid].find ()))
    json_data['unit'   ] = bson2json (list(db["%s.cu" % sid].find ()))

    if  len(json_data['session']) == 0 :
        raise ValueError ('no such session %s' % sid)

  # if  len(json_data['session']) > 1 :
  #     print 'more than one session document -- picking first one'

    # there can only be one session, not a list of one
    json_data['session'] = json_data['session'][0]

    # we want to add a list of handled units to each pilot doc
    for pilot in json_data['pilot'] :

        pilot['unit_ids'] = list()

        for unit in json_data['unit'] :

            if  unit['pilot'] == str(pilot['_id']) :
                pilot['unit_ids'].append (str(unit['_id']))

    # if we got here, we did not find a cached version -- thus add this dataset
    # to the cache
    try :
        os.system ('mkdir -p %s' % _CACHE_BASEDIR)
        ru.write_json (json_data, "%s/%s.json" % (_CACHE_BASEDIR, sid))
    except Exception as e :
        # we can live without cache, no problem...
        pass

    return json_data

def test():

    s = None
    try:
        cfg = ru.read_json("%s/session.json" % os.path.dirname(__file__))
        dh  = ru.DebugHelper()
        s   = rp.Session(cfg=cfg)

        ca1 = CompA(s)
        cb1 = CompB(s)
        cb2 = CompB(s)

        ca1.start()
        cb1.start()
        cb2.start()

      # s._controller.add_things([ca1, cb1, cb2])

        time.sleep(3)


    finally:
        if s:
            print 'close'
            s.close()

    def setUpClass(cls):
        """Initialize tests, just creates instance variables needed."""
        super(AcceptanceTests, cls).setUpClass()

        cls.resource = None
        cls.session  = None
        cls.pmgr     = None
        cls.umgr     = None
        cls.n        = 128   # number of units to run
        cls.config   = ru.read_json('%s/config.json' % os.path.dirname(__file__))

        cls.setUp()

def get_session_description(sid, src=None):

    if not src:
        src = os.getcwd()

    if os.path.exists(src):

        # EnTK profiles are always on localhost
        desc = ru.read_json("%s/%s/radical.entk.%s.json" % (src, sid, sid))

    else:
        raise EnTKError('%s/%s does not exist' % (src, sid))

    return desc

def write_workflow(workflow, uid, workflow_fout='entk_workflow', fwrite=True):

    try:
        os.mkdir(uid)
    except:
        pass

    data = list()
    if os.path.isfile('%s/%s.json' % (uid, workflow_fout)):
        data = ru.read_json('%s/%s.json' % (uid, workflow_fout))

    stack = ru.stack()
    data.append({'stack': stack})

    for pipe in workflow:

        p = dict()
        p['uid'] = pipe.uid
        p['name'] = pipe.name
        p['state_history'] = pipe.state_history
        p['stages'] = list()

        for stage in pipe.stages:

            s = dict()
            s['uid'] = stage.uid
            s['name'] = stage.name
            s['state_history'] = stage.state_history
            s['tasks'] = list()

            for task in stage.tasks:
                s['tasks'].append(task.to_dict())

            p['stages'].append(s)

        data.append(p)

    if fwrite:
        ru.write_json(data, '%s/%s.json' % (uid, workflow_fout))
        return 0

    return data

def test_duration_method_with_data_from_run_with_execution_barriers():

    '''
    This function tests if the durations obtained from the analytics function is the same
    as the duration obtained by the utils function. They should both be less than the 'max-min' (of the FINAL 
    and INITIAL states respectively) as the data set in this case consists of profiles when not all units are concurrently
    being executed, i.e. there is an execution barrier between them and thus a 'gap' between their executions.
    '''


    data_loc = '{0}/barrier_data'.format(os.path.dirname(os.path.realpath(__file__)))
    json_files = glob.glob('{0}/*.json'.format(data_loc))
    json_file = json_files[0]
    json      = ru.read_json(json_file)
    sid       = os.path.basename(json_file)[:-5]

    session = ra.Session(sid, 'radical.pilot', src='{0}/'.format(data_loc))       
       

    assert get_duration_using_analytics(session) == get_duration_using_utils(session) 
    assert get_duration_using_analytics(session) < get_duration_using_minmax(session)

    def _read_config(self, config_path, hostname, port, reattempts,
                     resubmit_failed, autoterminate, write_workflow,
                     rts, rmq_cleanup, rts_config):

        if not config_path:
            config_path = os.path.dirname(os.path.abspath(__file__))

        config = ru.read_json(os.path.join(config_path, 'config.json'))

        self._mq_hostname = hostname if hostname else str(config['hostname'])
        self._port = int(port if port else config['port'])
        self._reattempts = reattempts if reattempts else config['reattempts']
        self._resubmit_failed = resubmit_failed if resubmit_failed is not None else config['resubmit_failed']
        self._autoterminate = autoterminate if autoterminate is not None else config['autoterminate']
        self._write_workflow = write_workflow if write_workflow is not None else config['write_workflow']
        self._rts = rts if rts in ['radical.pilot', 'mock'] else str(config['rts'])
        self._rmq_cleanup = rmq_cleanup if rmq_cleanup is not None else config['rmq_cleanup']
        self._rts_config = rts_config if rts_config is not None else config['rts_config']

        self._num_pending_qs = config['pending_qs']
        self._num_completed_qs = config['completed_qs']

def get_config (params) :
    """
    This method attempts to obtain configuration settings from a variety of
    sources, depending on the parameter. it can point to an env var, or to
    a directory containing configuration files, or to a single configuration
    file, or to a list of any above, or it is a config dict already, or a list
    of such dicts.  In all cases, the config is obtained from the respective
    source (which is assumed json formatted in the case of config files), and
    a single merged and expanded dict is returned.
    """


    ret = dict()

    # always make params list for simpler code below
    if  not isinstance(params, list) :
        params = [params]


    for param in params :

        if  not param or None == param : 

            # we silently accept None's, to save some 
            # repetetetetive checks on the calling side
            continue


        elif isinstance (param, dict) :

            # simply merge it into the result
            ru.dict_merge (ret, param, policy='overwrite')


        elif isinstance (param, basestring) :
        
            # check if the string points to an env variable
            if  param in os.environ :
                # assume that the value of the env var is what we really want
                param = os.environ[param]

            # is string, is not env, must be a dir or a file
            if  os.path.isdir (param) :
                # config dir
                cfg_files = glob.glob ("%s/*" % param)
              # print 'is dir %s/*' % param
              # print cfg_files

            elif os.path.isfile (param) :
                # single config file
                cfg_files = [param]

            else :
                troy._logger.warning ("cannot handle config location %s" % param)
                cfg_files = list()

            print 'files: %s' % cfg_files
            # read and merge all config files
            for cfg_file in cfg_files :
                cfg_dict = dict()
                try :
                    cfg_dict = ru.read_json (cfg_file)
                    troy._logger.info ("reading  config in %s" % cfg_file)
                except Exception as e :
                    troy._logger.critical ("skipping config in %s (%s)" % (cfg_file, e))
                    raise

              # import pprint
              # print '================'
              # print cfg_file
              # pprint.pprint (cfg_dict)
              # print '================'

                ru.dict_merge (ret, cfg_dict, policy='overwrite')



        else :
            raise TypeError ("get_config parameter must be (list of) dict or "
                             "string, not %s" % type(param))

  # print '================================'
  # pprint.pprint (ret)
  # print '================================'

    # expand config(s) before returning
    ru.dict_stringexpand (ret)

    return ret

    random.shuffle(sequence)

    return sequence


# =============================================================================
# EXPERIMENT
# =============================================================================
if __name__ == '__main__':

    if len(sys.argv) < 2:
        print "\n\n\tusage: %s <config.json>\n\n" % sys.argv[0]
        sys.exit(-1)

    # read configuration file.
    cfg = ru.read_json(sys.argv[1])

    # TODO: Rename aimes.emgr config keys.
    cfg["skeleton_template"] = cfg["skeleton"]["template"]
    cfg["pct_concurrency"] = cfg["strategy"]["pct_concurrency"]
    cfg["pct_resources"] = cfg["strategy"]["pct_resources"]
    cfg["recipients"] = cfg["log"]["email"]["recipients"]

    # TODO: Override with json skeleton config entries.
    cfg['skeleton_task_duration'] = {
        "max": cfg["skeleton"]["tasks"]["duration"]["max"],
        "min": cfg["skeleton"]["tasks"]["duration"]["min"]}

    # cfg['bundle_resources']   = {'hopper.nersc.gov'          : 'pbs',
    #                              'stampede.tacc.xsede.org'   : 'slurm'}
    #                              'gordon.sdsc.xsede.org'     : 'pbs'}

if __name__ == '__main__':

    if len(sys.argv) != 2:
        print "\n\tusage: %s <dir>\n" % sys.argv[0]
        sys.exit(1)

    loc = sys.argv[1]

    # find json file in dir, and derive session id
    json_files = glob.glob('%s/*.json' % loc)
    
    if len(json_files) < 1: raise ValueError('%s contains no json file!' % loc)
    if len(json_files) > 1: raise ValueError('%s contains more than one json file!' % loc)

    json_file = json_files[0]
    json      = ru.read_json(json_file)
    sid       = os.path.basename(json_file)[:-5]

    print 'sid: %s' % sid

    descr     = rp.utils.get_session_description(sid=sid, src=loc)
    prof      = rp.utils.get_session_profile    (sid=sid, src=loc)

    session = ra.Session(prof, descr)

    # A formatting helper before starting...
    def ppheader(message):
        separator = '\n' + 78 * '-' + '\n'
        print separator + message + separator

    # and here we go. Once we filter our session object so to keep only the

def emulate(command=None, samples=None, src=None):

    if (command and samples) or \
       (command and src    ) or \
       (samples and src    )    :
        raise ValueError ("emulate needs *either* command, sample *or* src")

    if not command and not samples and not src:
        print "warning: emulate needs either command, sample or src"
        return[0, None, None]

    if command or src:

        if command:
            profs = rsu.get_profiles (command, mode='pro')

            # FIXME: average vals over all retrieved profiles
            prof  = profs[0]['profile']

        else:
            prof = ru.read_json(src)

      # pprint.pprint (prof)

        # get time series to emulate (all types of operations are mixed)
        # FIXME: we should also sample walltime for _TIM.  As it is, mixing
        #        time and other samples will yield incorrect results due to
        #        mismatch in granularity.
        # FIXME: add network sample interpretation
        samples  = list()
      # samples += [[_TIM, x[0], {'real'       : x[1].get('real',       0)}]
      #                           for x in prof['time']]

        samples += [[_CPU, x[0], {'time'       : x[1].get('time',       0),
                                  'flops'      : x[1].get('ops',        0),
                                  'efficiency' : x[1].get('efficiency', 0)}]
                                  for x in prof['cpu']['sequence']]

        samples += [[_MEM, x[0], {'size'       : x[1].get('size',       0)}]
                                  for x in prof['mem']['sequence']]

        samples += [[_STO, x[0], {'src'        : x[1].get('src',        None),
                                  'rsize'      : x[1].get('read',       0),
                                  'tgt'        : x[1].get('tgt',        None),
                                  'wsize'      : x[1].get('write',      0)}]
                                  for x in prof['sto']['sequence']]

    # *globally* sort samples by time
    samples = sorted (samples, key=lambda x: x[1])

  # print "samples:\n---"
  # pprint.pprint (samples)
  # print "---"

    watchmode = os.environ.get ('RADICAL_SYNAPSE_WATCHMODE')
    if not watchmode or watchmode.lower in ['none', 'noop']:
        start = time.time()
        _emulator(samples)
        stop  = time.time()

        ret   = None
        info  = dict()
        info['cmd']  = command
        info['time'] = dict()
        info['time']['start'] = start
        info['time']['real']  = stop-start

    else:
        # let the profiler know that we run an emulation, so that the profile is not
        # stored as 'application run'.
        os.environ['_RADICAL_SYNAPSE_EMULATED'] = 'TRUE'
        os.environ['_RADICAL_SYNAPSE_EMULATEE'] = command

        info, ret, _ = profile(_emulator, samples)

        if 'ops' in info['cpu']:
            info['cpu']['efficiency'] = info['cpu']['ops']                       \
                                        / ( info['cpu']['ops']                   \
                                          + info['cpu']['cycles_stalled_front']  \
                                          + info['cpu']['cycles_stalled_back']   \
                                          )

       #print 'efficiency = %s / (%s + %s + %s) = %s' % (
       #          info['cpu']['ops'],
       #          info['cpu']['ops'],
       #          info['cpu']['cycles_stalled_front'],
       #          info['cpu']['cycles_stalled_back'],
       #          info['cpu']['efficiency'])


    return (info, ret, None)

 def __init__(self, uid, cfg, dbs):
     self.uid  = uid
     self._cfg = ru.read_json (credential['config'])
     self._dbs = dbs

#.........这里部分代码省略.........
                                 % (ma, resource))

        # get pilot and global sandbox
        resource_sandbox = self._session._get_resource_sandbox (pilot).path
        session_sandbox  = self._session._get_session_sandbox(pilot).path
        pilot_sandbox    = self._session._get_pilot_sandbox  (pilot).path

        pilot['resource_sandbox'] = str(self._session._get_resource_sandbox(pilot))
        pilot['pilot_sandbox']    = str(self._session._get_pilot_sandbox(pilot))
        pilot['client_sandbox']   = str(self._session._get_client_sandbox())

        # Agent configuration that is not part of the public API.
        # The agent config can either be a config dict, or
        # a string pointing to a configuration name.  If neither
        # is given, check if 'RADICAL_PILOT_AGENT_CONFIG' is
        # set.  The last fallback is 'agent_default'
        agent_config = pilot['description'].get('_config')
        if not agent_config:
            agent_config = os.environ.get('RADICAL_PILOT_AGENT_CONFIG')
        if not agent_config:
            agent_config = rc_agent_config

        if isinstance(agent_config, dict):

            # use dict as is
            agent_cfg = agent_config

        elif isinstance(agent_config, basestring):
            try:
                # interpret as a config name
                agent_cfg_file = os.path.join(self._conf_dir, "agent_%s.json" % agent_config)

                self._log.info("Read agent config file: %s",  agent_cfg_file)
                agent_cfg = ru.read_json(agent_cfg_file)

                # allow for user level overload
                user_cfg_file = '%s/.radical/pilot/config/%s' \
                              % (os.environ['HOME'], os.path.basename(agent_cfg_file))

                if os.path.exists(user_cfg_file):
                    self._log.info("merging user config: %s" % user_cfg_file)
                    user_cfg = ru.read_json(user_cfg_file)
                    ru.dict_merge (agent_cfg, user_cfg, policy='overwrite')

            except Exception as e:
                self._log.exception("Error reading agent config file: %s" % e)
                raise

        else:
            # we can't handle this type
            raise TypeError('agent config must be string (config name) or dict')

        # expand variables in virtenv string
        virtenv = virtenv % {'pilot_sandbox'   : pilot_sandbox,
                             'session_sandbox' : session_sandbox,
                             'resource_sandbox': resource_sandbox}

        # Check for deprecated global_virtenv
        if 'global_virtenv' in rcfg:
            raise RuntimeError("'global_virtenv' is deprecated (%s)" % resource)

        # Create a host:port string for use by the bootstrap_0.
        db_url = rs.Url(agent_dburl)
        if db_url.port:
            db_hostport = "%s:%d" % (db_url.host, db_url.port)
        else:

    def _read_json (filename) :

        data = ru.read_json (filename)
        os.unlink (filename)
        return data