本文整理汇总了Python中radical.utils.read_json函数的典型用法代码示例。如果您正苦于以下问题:Python read_json函数的具体用法?Python read_json怎么用?Python read_json使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了read_json函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: setUpClass
def setUpClass(self):
# Directory of sample config files
self.sample_resource_dir = os.path.dirname(os.path.realpath(__file__))
# Load sample resource test config
self.cfg_sample = ru.read_json(os.path.join(self.sample_resource_dir, "sample_resources.json"))
self.cfg_sample_1 = self.cfg_sample['sample_resource_1']
self.cfg_sample_2 = self.cfg_sample['sample_resource_2']
self.cfg_sample_3 = self.cfg_sample['sample_resource_3']
# Directory of pilot resource config files
self.pilot_resource_dir = '../../src/radical/pilot/configs'
# Load xsede pilot resource config
self.cfg_xsede = ru.read_json(os.path.join(self.pilot_resource_dir, 'resource_xsede.json'))
self.cfg_xsede_bridges = self.cfg_xsede['bridges']
self.cfg_xsede_comet_ssh = self.cfg_xsede['comet_ssh']
self.cfg_xsede_comet_orte = self.cfg_xsede['comet_orte']
self.cfg_xsede_comet_ortelib = self.cfg_xsede['comet_ortelib']
self.cfg_xsede_comet_spark = self.cfg_xsede['comet_spark']
self.cfg_xsede_supermic_ssh = self.cfg_xsede['supermic_ssh']
self.cfg_xsede_supermic_orte = self.cfg_xsede['supermic_orte']
self.cfg_xsede_supermic_ortelib = self.cfg_xsede['supermic_ortelib']
self.cfg_xsede_supermic_spark = self.cfg_xsede['supermic_spark']
return
示例2: get_session_docs
def get_session_docs (db, sid, cache=None, cachedir=None) :
# session docs may have been cached in /tmp/rp_cache_<uid>/<sid>.json -- in that
# case we pull it from there instead of the database, which will be much
# quicker. Also, we do cache any retrieved docs to that place, for later
# use. An optional cachdir parameter changes that default location for
# lookup and storage.
if not cachedir :
cachedir = _CACHE_BASEDIR
if not cache :
cache = "%s/%s.json" % (cachedir, sid)
try :
if os.path.isfile (cache) :
return ru.read_json (cache)
except Exception as e :
# continue w/o cache
sys.stderr.write ("warning: cannot read session cache at %s (%s)\n" % (cache, e))
# cache not used or not found -- go to db
json_data = dict()
# convert bson to json, i.e. serialize the ObjectIDs into strings.
json_data['session'] = bson2json (list(db["%s" % sid].find ()))
json_data['pmgr' ] = bson2json (list(db["%s.pm" % sid].find ()))
json_data['pilot' ] = bson2json (list(db["%s.p" % sid].find ()))
json_data['umgr' ] = bson2json (list(db["%s.um" % sid].find ()))
json_data['unit' ] = bson2json (list(db["%s.cu" % sid].find ()))
if len(json_data['session']) == 0 :
raise ValueError ('no such session %s' % sid)
# if len(json_data['session']) > 1 :
# print 'more than one session document -- picking first one'
# there can only be one session, not a list of one
json_data['session'] = json_data['session'][0]
# we want to add a list of handled units to each pilot doc
for pilot in json_data['pilot'] :
pilot['unit_ids'] = list()
for unit in json_data['unit'] :
if unit['pilot'] == str(pilot['_id']) :
pilot['unit_ids'].append (str(unit['_id']))
# if we got here, we did not find a cached version -- thus add this dataset
# to the cache
try :
os.system ('mkdir -p %s' % _CACHE_BASEDIR)
ru.write_json (json_data, "%s/%s.json" % (_CACHE_BASEDIR, sid))
except Exception as e :
# we can live without cache, no problem...
pass
return json_data
示例3: test
def test():
s = None
try:
cfg = ru.read_json("%s/session.json" % os.path.dirname(__file__))
dh = ru.DebugHelper()
s = rp.Session(cfg=cfg)
ca1 = CompA(s)
cb1 = CompB(s)
cb2 = CompB(s)
ca1.start()
cb1.start()
cb2.start()
# s._controller.add_things([ca1, cb1, cb2])
time.sleep(3)
finally:
if s:
print 'close'
s.close()
示例4: setUpClass
def setUpClass(cls):
"""Initialize tests, just creates instance variables needed."""
super(AcceptanceTests, cls).setUpClass()
cls.resource = None
cls.session = None
cls.pmgr = None
cls.umgr = None
cls.n = 128 # number of units to run
cls.config = ru.read_json('%s/config.json' % os.path.dirname(__file__))
cls.setUp()
示例5: get_session_description
def get_session_description(sid, src=None):
if not src:
src = os.getcwd()
if os.path.exists(src):
# EnTK profiles are always on localhost
desc = ru.read_json("%s/%s/radical.entk.%s.json" % (src, sid, sid))
else:
raise EnTKError('%s/%s does not exist' % (src, sid))
return desc
示例6: write_workflow
def write_workflow(workflow, uid, workflow_fout='entk_workflow', fwrite=True):
try:
os.mkdir(uid)
except:
pass
data = list()
if os.path.isfile('%s/%s.json' % (uid, workflow_fout)):
data = ru.read_json('%s/%s.json' % (uid, workflow_fout))
stack = ru.stack()
data.append({'stack': stack})
for pipe in workflow:
p = dict()
p['uid'] = pipe.uid
p['name'] = pipe.name
p['state_history'] = pipe.state_history
p['stages'] = list()
for stage in pipe.stages:
s = dict()
s['uid'] = stage.uid
s['name'] = stage.name
s['state_history'] = stage.state_history
s['tasks'] = list()
for task in stage.tasks:
s['tasks'].append(task.to_dict())
p['stages'].append(s)
data.append(p)
if fwrite:
ru.write_json(data, '%s/%s.json' % (uid, workflow_fout))
return 0
return data
示例7: test_duration_method_with_data_from_run_with_execution_barriers
def test_duration_method_with_data_from_run_with_execution_barriers():
'''
This function tests if the durations obtained from the analytics function is the same
as the duration obtained by the utils function. They should both be less than the 'max-min' (of the FINAL
and INITIAL states respectively) as the data set in this case consists of profiles when not all units are concurrently
being executed, i.e. there is an execution barrier between them and thus a 'gap' between their executions.
'''
data_loc = '{0}/barrier_data'.format(os.path.dirname(os.path.realpath(__file__)))
json_files = glob.glob('{0}/*.json'.format(data_loc))
json_file = json_files[0]
json = ru.read_json(json_file)
sid = os.path.basename(json_file)[:-5]
session = ra.Session(sid, 'radical.pilot', src='{0}/'.format(data_loc))
assert get_duration_using_analytics(session) == get_duration_using_utils(session)
assert get_duration_using_analytics(session) < get_duration_using_minmax(session)
示例8: _read_config
def _read_config(self, config_path, hostname, port, reattempts,
resubmit_failed, autoterminate, write_workflow,
rts, rmq_cleanup, rts_config):
if not config_path:
config_path = os.path.dirname(os.path.abspath(__file__))
config = ru.read_json(os.path.join(config_path, 'config.json'))
self._mq_hostname = hostname if hostname else str(config['hostname'])
self._port = int(port if port else config['port'])
self._reattempts = reattempts if reattempts else config['reattempts']
self._resubmit_failed = resubmit_failed if resubmit_failed is not None else config['resubmit_failed']
self._autoterminate = autoterminate if autoterminate is not None else config['autoterminate']
self._write_workflow = write_workflow if write_workflow is not None else config['write_workflow']
self._rts = rts if rts in ['radical.pilot', 'mock'] else str(config['rts'])
self._rmq_cleanup = rmq_cleanup if rmq_cleanup is not None else config['rmq_cleanup']
self._rts_config = rts_config if rts_config is not None else config['rts_config']
self._num_pending_qs = config['pending_qs']
self._num_completed_qs = config['completed_qs']
示例9: get_config
def get_config (params) :
"""
This method attempts to obtain configuration settings from a variety of
sources, depending on the parameter. it can point to an env var, or to
a directory containing configuration files, or to a single configuration
file, or to a list of any above, or it is a config dict already, or a list
of such dicts. In all cases, the config is obtained from the respective
source (which is assumed json formatted in the case of config files), and
a single merged and expanded dict is returned.
"""
ret = dict()
# always make params list for simpler code below
if not isinstance(params, list) :
params = [params]
for param in params :
if not param or None == param :
# we silently accept None's, to save some
# repetetetetive checks on the calling side
continue
elif isinstance (param, dict) :
# simply merge it into the result
ru.dict_merge (ret, param, policy='overwrite')
elif isinstance (param, basestring) :
# check if the string points to an env variable
if param in os.environ :
# assume that the value of the env var is what we really want
param = os.environ[param]
# is string, is not env, must be a dir or a file
if os.path.isdir (param) :
# config dir
cfg_files = glob.glob ("%s/*" % param)
# print 'is dir %s/*' % param
# print cfg_files
elif os.path.isfile (param) :
# single config file
cfg_files = [param]
else :
troy._logger.warning ("cannot handle config location %s" % param)
cfg_files = list()
print 'files: %s' % cfg_files
# read and merge all config files
for cfg_file in cfg_files :
cfg_dict = dict()
try :
cfg_dict = ru.read_json (cfg_file)
troy._logger.info ("reading config in %s" % cfg_file)
except Exception as e :
troy._logger.critical ("skipping config in %s (%s)" % (cfg_file, e))
raise
# import pprint
# print '================'
# print cfg_file
# pprint.pprint (cfg_dict)
# print '================'
ru.dict_merge (ret, cfg_dict, policy='overwrite')
else :
raise TypeError ("get_config parameter must be (list of) dict or "
"string, not %s" % type(param))
# print '================================'
# pprint.pprint (ret)
# print '================================'
# expand config(s) before returning
ru.dict_stringexpand (ret)
return ret
示例10: len
random.shuffle(sequence)
return sequence
# =============================================================================
# EXPERIMENT
# =============================================================================
if __name__ == '__main__':
if len(sys.argv) < 2:
print "\n\n\tusage: %s <config.json>\n\n" % sys.argv[0]
sys.exit(-1)
# read configuration file.
cfg = ru.read_json(sys.argv[1])
# TODO: Rename aimes.emgr config keys.
cfg["skeleton_template"] = cfg["skeleton"]["template"]
cfg["pct_concurrency"] = cfg["strategy"]["pct_concurrency"]
cfg["pct_resources"] = cfg["strategy"]["pct_resources"]
cfg["recipients"] = cfg["log"]["email"]["recipients"]
# TODO: Override with json skeleton config entries.
cfg['skeleton_task_duration'] = {
"max": cfg["skeleton"]["tasks"]["duration"]["max"],
"min": cfg["skeleton"]["tasks"]["duration"]["min"]}
# cfg['bundle_resources'] = {'hopper.nersc.gov' : 'pbs',
# 'stampede.tacc.xsede.org' : 'slurm'}
# 'gordon.sdsc.xsede.org' : 'pbs'}
示例11: len
if __name__ == '__main__':
if len(sys.argv) != 2:
print "\n\tusage: %s <dir>\n" % sys.argv[0]
sys.exit(1)
loc = sys.argv[1]
# find json file in dir, and derive session id
json_files = glob.glob('%s/*.json' % loc)
if len(json_files) < 1: raise ValueError('%s contains no json file!' % loc)
if len(json_files) > 1: raise ValueError('%s contains more than one json file!' % loc)
json_file = json_files[0]
json = ru.read_json(json_file)
sid = os.path.basename(json_file)[:-5]
print 'sid: %s' % sid
descr = rp.utils.get_session_description(sid=sid, src=loc)
prof = rp.utils.get_session_profile (sid=sid, src=loc)
session = ra.Session(prof, descr)
# A formatting helper before starting...
def ppheader(message):
separator = '\n' + 78 * '-' + '\n'
print separator + message + separator
# and here we go. Once we filter our session object so to keep only the
示例12: emulate
def emulate(command=None, samples=None, src=None):
if (command and samples) or \
(command and src ) or \
(samples and src ) :
raise ValueError ("emulate needs *either* command, sample *or* src")
if not command and not samples and not src:
print "warning: emulate needs either command, sample or src"
return[0, None, None]
if command or src:
if command:
profs = rsu.get_profiles (command, mode='pro')
# FIXME: average vals over all retrieved profiles
prof = profs[0]['profile']
else:
prof = ru.read_json(src)
# pprint.pprint (prof)
# get time series to emulate (all types of operations are mixed)
# FIXME: we should also sample walltime for _TIM. As it is, mixing
# time and other samples will yield incorrect results due to
# mismatch in granularity.
# FIXME: add network sample interpretation
samples = list()
# samples += [[_TIM, x[0], {'real' : x[1].get('real', 0)}]
# for x in prof['time']]
samples += [[_CPU, x[0], {'time' : x[1].get('time', 0),
'flops' : x[1].get('ops', 0),
'efficiency' : x[1].get('efficiency', 0)}]
for x in prof['cpu']['sequence']]
samples += [[_MEM, x[0], {'size' : x[1].get('size', 0)}]
for x in prof['mem']['sequence']]
samples += [[_STO, x[0], {'src' : x[1].get('src', None),
'rsize' : x[1].get('read', 0),
'tgt' : x[1].get('tgt', None),
'wsize' : x[1].get('write', 0)}]
for x in prof['sto']['sequence']]
# *globally* sort samples by time
samples = sorted (samples, key=lambda x: x[1])
# print "samples:\n---"
# pprint.pprint (samples)
# print "---"
watchmode = os.environ.get ('RADICAL_SYNAPSE_WATCHMODE')
if not watchmode or watchmode.lower in ['none', 'noop']:
start = time.time()
_emulator(samples)
stop = time.time()
ret = None
info = dict()
info['cmd'] = command
info['time'] = dict()
info['time']['start'] = start
info['time']['real'] = stop-start
else:
# let the profiler know that we run an emulation, so that the profile is not
# stored as 'application run'.
os.environ['_RADICAL_SYNAPSE_EMULATED'] = 'TRUE'
os.environ['_RADICAL_SYNAPSE_EMULATEE'] = command
info, ret, _ = profile(_emulator, samples)
if 'ops' in info['cpu']:
info['cpu']['efficiency'] = info['cpu']['ops'] \
/ ( info['cpu']['ops'] \
+ info['cpu']['cycles_stalled_front'] \
+ info['cpu']['cycles_stalled_back'] \
)
#print 'efficiency = %s / (%s + %s + %s) = %s' % (
# info['cpu']['ops'],
# info['cpu']['ops'],
# info['cpu']['cycles_stalled_front'],
# info['cpu']['cycles_stalled_back'],
# info['cpu']['efficiency'])
return (info, ret, None)
示例13: __init__
def __init__(self, uid, cfg, dbs):
self.uid = uid
self._cfg = ru.read_json (credential['config'])
self._dbs = dbs
示例14: _prepare_pilot
#.........这里部分代码省略.........
% (ma, resource))
# get pilot and global sandbox
resource_sandbox = self._session._get_resource_sandbox (pilot).path
session_sandbox = self._session._get_session_sandbox(pilot).path
pilot_sandbox = self._session._get_pilot_sandbox (pilot).path
pilot['resource_sandbox'] = str(self._session._get_resource_sandbox(pilot))
pilot['pilot_sandbox'] = str(self._session._get_pilot_sandbox(pilot))
pilot['client_sandbox'] = str(self._session._get_client_sandbox())
# Agent configuration that is not part of the public API.
# The agent config can either be a config dict, or
# a string pointing to a configuration name. If neither
# is given, check if 'RADICAL_PILOT_AGENT_CONFIG' is
# set. The last fallback is 'agent_default'
agent_config = pilot['description'].get('_config')
if not agent_config:
agent_config = os.environ.get('RADICAL_PILOT_AGENT_CONFIG')
if not agent_config:
agent_config = rc_agent_config
if isinstance(agent_config, dict):
# use dict as is
agent_cfg = agent_config
elif isinstance(agent_config, basestring):
try:
# interpret as a config name
agent_cfg_file = os.path.join(self._conf_dir, "agent_%s.json" % agent_config)
self._log.info("Read agent config file: %s", agent_cfg_file)
agent_cfg = ru.read_json(agent_cfg_file)
# allow for user level overload
user_cfg_file = '%s/.radical/pilot/config/%s' \
% (os.environ['HOME'], os.path.basename(agent_cfg_file))
if os.path.exists(user_cfg_file):
self._log.info("merging user config: %s" % user_cfg_file)
user_cfg = ru.read_json(user_cfg_file)
ru.dict_merge (agent_cfg, user_cfg, policy='overwrite')
except Exception as e:
self._log.exception("Error reading agent config file: %s" % e)
raise
else:
# we can't handle this type
raise TypeError('agent config must be string (config name) or dict')
# expand variables in virtenv string
virtenv = virtenv % {'pilot_sandbox' : pilot_sandbox,
'session_sandbox' : session_sandbox,
'resource_sandbox': resource_sandbox}
# Check for deprecated global_virtenv
if 'global_virtenv' in rcfg:
raise RuntimeError("'global_virtenv' is deprecated (%s)" % resource)
# Create a host:port string for use by the bootstrap_0.
db_url = rs.Url(agent_dburl)
if db_url.port:
db_hostport = "%s:%d" % (db_url.host, db_url.port)
else:
示例15: _read_json
def _read_json (filename) :
data = ru.read_json (filename)
os.unlink (filename)
return data