本文整理匯總了Python中FileDict.FileDict類的典型用法代碼示例。如果您正苦於以下問題：Python FileDict類的具體用法？Python FileDict怎麽用？Python FileDict使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。
在下文中一共展示了FileDict類的8個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: _get_filelist_remote
def _get_filelist_remote(remote_uri, recursive = True):
## If remote_uri ends with '/' then all remote files will have
## the remote_uri prefix removed in the relative path.
## If, on the other hand, the remote_uri ends with something else
## (probably alphanumeric symbol) we'll use the last path part
## in the relative path.
##
## Complicated, eh? See an example:
## _get_filelist_remote("s3://bckt/abc/def") may yield:
## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
## _get_filelist_remote("s3://bckt/abc/def/") will yield:
## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
## Furthermore a prefix-magic can restrict the return list:
## _get_filelist_remote("s3://bckt/abc/def/x") yields:
## { 'xyz/blah.txt' : {} }
info(u"Retrieving list of remote files for %s ..." % remote_uri)
s3 = S3(Config())
response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive)
rem_base_original = rem_base = remote_uri.object()
remote_uri_original = remote_uri
if rem_base != '' and rem_base[-1] != '/':
rem_base = rem_base[:rem_base.rfind('/')+1]
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
rem_base_len = len(rem_base)
rem_list = FileDict(ignore_case = False)
break_now = False
for object in response['list']:
if object['Key'] == rem_base_original and object['Key'][-1] != "/":
## We asked for one file and we got that file :-)
key = os.path.basename(object['Key'])
object_uri_str = remote_uri_original.uri()
break_now = True
rem_list = FileDict(ignore_case = False) ## Remove whatever has already been put to rem_list
else:
key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
object_uri_str = remote_uri.uri() + key
rem_list[key] = {
'size' : int(object['Size']),
'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
'md5' : object['ETag'][1:-1],
'object_key' : object['Key'],
'object_uri_str' : object_uri_str,
'base_uri' : remote_uri,
'dev' : None,
'inode' : None,
}
if rem_list[key]['md5'].find("-") != -1: # always get it for multipart uploads
_get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
md5 = rem_list[key]['md5']
rem_list.record_md5(key, md5)
if break_now:
break
return rem_list示例2: __init__
def __init__(self, homeDir, name, f_hasFailed, interactive=False, newjob=True, quiet=False):
"""
homeDir is where experiments are stored
name is the name of this experiment
if there is another job with the same name, they are assumed
to be the same. delete old jobs that you do not want to be confused with
f_hasFailed is a function that takes a log file and returns a boolean for
whether or not the job has failed
if in interactive mode, will prompt user for input
this should not be used for scripts, which would hang forever in these cases
"""
self.name = name
self.home = os.path.join(homeDir, name)
self.logDir = os.path.join(self.home, 'log')
self.f_hasFailed = f_hasFailed
if os.path.isfile(self.home):
raise Exception('you the home directory you gave must not have any files or folders that match the name you gave')
if newjob:
print 'creating new job:', self.home
if os.path.isdir(self.home):
if interactive:
print "the directory %s already exists" % (self.home)
r = raw_input('would you like me to delete it for you? [y|n] ')
if str2bool(r):
os.system('rm -r ' + self.home)
os.system('mkdir ' + self.home)
else:
raise Exception('cannot proceed')
else:
raise Exception('please give a unique name or delete old jobs: ' + self.home)
elif 0 != os.system('mkdir -p ' + self.home):
raise Exception('cannot make home directory!')
if not os.path.isdir(self.logDir) and 0 != os.system('mkdir -p ' + self.logDir):
raise Exception('cannot make log directory!')
else:
assert os.path.isdir(self.home)
assert os.path.isdir(self.logDir)
if not quiet: print 'loading existing job:', self.home
self.javaOpt = FileDict(os.path.join(self.home, 'java.settings'), exists=not newjob) # start with "-D"
self.metaOpt = FileDict(os.path.join(self.home, 'meta.settings'), exists=not newjob) # xmx, jar, profile, etc
self.qsubOpt = FileDict(os.path.join(self.home, 'qsub.settings'), exists=not newjob) # mem_free, h_rt
self.prepared = False示例3: makeRun
def makeRun(self, baseConfig, runData):
"""Make a new run of config files from the base config and runData.
runData is a list of tuples which contain a label and a dict.
Labels are used to name generated configs and their specified output
files. The dicts are key-value pairs for data to modify in the
base config. Return a list of the names of config files generated.
"""
configNames = []
baseConfigFullPath = os.path.join(self.path, baseConfig)
for label, labelData in runData:
newConfig = FileDict(baseConfigFullPath)
newConfigFullPath = os.path.join(self.path, label + "_config")
labelData.update({"outputLogName" : label + "_out.fd",
"errorLogName" : label + "_error",
"debugLogName" : label + "_debug"})
for key, value in labelData.items():
newConfig.setGlobal(str(key), str(value))
configNames.append(newConfigFullPath)
newConfig.writeToFile(newConfigFullPath)
return configNames示例4: fetch_remote_list
def fetch_remote_list(args, require_attribs = False, recursive = None):
def _get_filelist_remote(remote_uri, recursive = True):
## If remote_uri ends with '/' then all remote files will have
## the remote_uri prefix removed in the relative path.
## If, on the other hand, the remote_uri ends with something else
## (probably alphanumeric symbol) we'll use the last path part
## in the relative path.
##
## Complicated, eh? See an example:
## _get_filelist_remote("s3://bckt/abc/def") may yield:
## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
## _get_filelist_remote("s3://bckt/abc/def/") will yield:
## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
## Furthermore a prefix-magic can restrict the return list:
## _get_filelist_remote("s3://bckt/abc/def/x") yields:
## { 'xyz/blah.txt' : {} }
info(u"Retrieving list of remote files for %s ..." % remote_uri)
s3 = S3(Config())
response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive)
rem_base_original = rem_base = remote_uri.object()
remote_uri_original = remote_uri
if rem_base != '' and rem_base[-1] != '/':
rem_base = rem_base[:rem_base.rfind('/')+1]
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
rem_base_len = len(rem_base)
rem_list = FileDict(ignore_case = False)
break_now = False
for object in response['list']:
if object['Key'] == rem_base_original and object['Key'][-1] != "/":
## We asked for one file and we got that file :-)
key = os.path.basename(object['Key'])
object_uri_str = remote_uri_original.uri()
break_now = True
rem_list = FileDict(ignore_case = False) ## Remove whatever has already been put to rem_list
else:
key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
object_uri_str = remote_uri.uri() + key
rem_list[key] = {
'size' : int(object['Size']),
'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
'md5' : object['ETag'][1:-1],
'object_key' : object['Key'],
'object_uri_str' : object_uri_str,
'base_uri' : remote_uri,
'dev' : None,
'inode' : None,
}
md5 = object['ETag'][1:-1]
rem_list.record_md5(key, md5)
if break_now:
break
return rem_list
cfg = Config()
remote_uris = []
remote_list = FileDict(ignore_case = False)
if type(args) not in (list, tuple):
args = [args]
if recursive == None:
recursive = cfg.recursive
for arg in args:
uri = S3Uri(arg)
if not uri.type == 's3':
raise ParameterError("Expecting S3 URI instead of '%s'" % arg)
remote_uris.append(uri)
if recursive:
for uri in remote_uris:
objectlist = _get_filelist_remote(uri)
for key in objectlist:
remote_list[key] = objectlist[key]
remote_list.record_md5(key, objectlist.get_md5(key))
else:
for uri in remote_uris:
uri_str = str(uri)
## Wildcards used in remote URI?
## If yes we'll need a bucket listing...
if uri_str.find('*') > -1 or uri_str.find('?') > -1:
first_wildcard = uri_str.find('*')
first_questionmark = uri_str.find('?')
if first_questionmark > -1 and first_questionmark < first_wildcard:
first_wildcard = first_questionmark
prefix = uri_str[:first_wildcard]
rest = uri_str[first_wildcard+1:]
## Only request recursive listing if the 'rest' of the URI,
## i.e. the part after first wildcard, contains '/'
need_recursion = rest.find('/') > -1
objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion)
for key in objectlist:
## Check whether the 'key' matches the requested wildcards
if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str):
remote_list[key] = objectlist[key]
else:
## No wildcards - simply append the given URI to the list
#.........這裏部分代碼省略.........
示例5: debug
if src_md5 != dst_md5:
## checksums are different.
attribs_match = False
debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
return attribs_match
# we don't support local->local sync, use 'rsync' or something like that instead ;-)
assert(not(src_remote == False and dst_remote == False))
info(u"Verifying attributes...")
cfg = Config()
## Items left on src_list will be transferred
## Items left on update_list will be transferred after src_list
## Items left on copy_pairs will be copied from dst1 to dst2
update_list = FileDict(ignore_case = False)
## Items left on dst_list will be deleted
copy_pairs = []
debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
for relative_file in src_list.keys():
debug(u"CHECK: %s" % (relative_file))
if dst_list.has_key(relative_file):
## Was --skip-existing requested?
if cfg.skip_existing:
debug(u"IGNR: %s (used --skip-existing)" % (relative_file))
del(src_list[relative_file])
del(dst_list[relative_file])
continue示例6: fetch_remote_list
def fetch_remote_list(args, require_attribs = False, recursive = None, uri_params = {}):
def _get_remote_attribs(uri, remote_item):
response = S3(cfg).object_info(uri)
remote_item.update({
'size': int(response['headers']['content-length']),
'md5': response['headers']['etag'].strip('"\''),
'timestamp' : dateRFC822toUnix(response['headers']['date'])
})
try:
md5 = response['s3cmd-attrs']['md5']
remote_item.update({'md5': md5})
debug(u"retreived md5=%s from headers" % md5)
except KeyError:
pass
def _get_filelist_remote(remote_uri, recursive = True):
## If remote_uri ends with '/' then all remote files will have
## the remote_uri prefix removed in the relative path.
## If, on the other hand, the remote_uri ends with something else
## (probably alphanumeric symbol) we'll use the last path part
## in the relative path.
##
## Complicated, eh? See an example:
## _get_filelist_remote("s3://bckt/abc/def") may yield:
## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
## _get_filelist_remote("s3://bckt/abc/def/") will yield:
## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
## Furthermore a prefix-magic can restrict the return list:
## _get_filelist_remote("s3://bckt/abc/def/x") yields:
## { 'xyz/blah.txt' : {} }
info(u"Retrieving list of remote files for %s ..." % remote_uri)
empty_fname_re = re.compile(r'\A\s*\Z')
s3 = S3(Config())
response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(),
recursive = recursive, uri_params = uri_params)
rem_base_original = rem_base = remote_uri.object()
remote_uri_original = remote_uri
if rem_base != '' and rem_base[-1] != '/':
rem_base = rem_base[:rem_base.rfind('/')+1]
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
rem_base_len = len(rem_base)
rem_list = FileDict(ignore_case = False)
break_now = False
for object in response['list']:
if object['Key'] == rem_base_original and object['Key'][-1] != "/":
## We asked for one file and we got that file :-)
key = os.path.basename(object['Key'])
object_uri_str = remote_uri_original.uri()
break_now = True
rem_list = FileDict(ignore_case = False) ## Remove whatever has already been put to rem_list
else:
key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
object_uri_str = remote_uri.uri() + key
if empty_fname_re.match(key):
# Objects may exist on S3 with empty names (''), which don't map so well to common filesystems.
warning(u"Empty object name on S3 found, ignoring.")
continue
rem_list[key] = {
'size' : int(object['Size']),
'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
'md5' : object['ETag'][1:-1],
'object_key' : object['Key'],
'object_uri_str' : object_uri_str,
'base_uri' : remote_uri,
'dev' : None,
'inode' : None,
}
if rem_list[key]['md5'].find("-") > 0: # always get it for multipart uploads
_get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
md5 = rem_list[key]['md5']
rem_list.record_md5(key, md5)
if break_now:
break
return rem_list
cfg = Config()
remote_uris = []
remote_list = FileDict(ignore_case = False)
if type(args) not in (list, tuple):
args = [args]
if recursive == None:
recursive = cfg.recursive
for arg in args:
uri = S3Uri(arg)
if not uri.type == 's3':
raise ParameterError("Expecting S3 URI instead of '%s'" % arg)
remote_uris.append(uri)
if recursive:
for uri in remote_uris:
objectlist = _get_filelist_remote(uri, recursive = True)
for key in objectlist:
remote_list[key] = objectlist[key]
remote_list.record_md5(key, objectlist.get_md5(key))
#.........這裏部分代碼省略.........
示例7: compare_filelists
def compare_filelists(src_list, dst_list, src_remote, dst_remote):
def __direction_str(is_remote):
return is_remote and "remote" or "local"
def _compare(src_list, dst_lst, src_remote, dst_remote, file):
"""Return True if src_list[file] matches dst_list[file], else False"""
attribs_match = True
if not (src_list.has_key(file) and dst_list.has_key(file)):
info(u"%s: does not exist in one side or the other: src_list=%s, dst_list=%s" % (file, src_list.has_key(file), dst_list.has_key(file)))
return False
## check size first
if 'size' in cfg.sync_checks:
if 'size' in dst_list[file] and 'size' in src_list[file]:
if dst_list[file]['size'] != src_list[file]['size']:
debug(u"xfer: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
attribs_match = False
## check md5
compare_md5 = 'md5' in cfg.sync_checks
# Multipart-uploaded files don't have a valid md5 sum - it ends with "...-nn"
if compare_md5:
if (src_remote == True and '-' in src_list[file]['md5']) or (dst_remote == True and '-' in dst_list[file]['md5']):
compare_md5 = False
info(u"disabled md5 check for %s" % file)
if attribs_match and compare_md5:
try:
src_md5 = src_list.get_md5(file)
dst_md5 = dst_list.get_md5(file)
except (IOError,OSError):
# md5 sum verification failed - ignore that file altogether
debug(u"IGNR: %s (disappeared)" % (file))
warning(u"%s: file disappeared, ignoring." % (file))
raise
if src_md5 != dst_md5:
## checksums are different.
attribs_match = False
debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
return attribs_match
# we don't support local->local sync, use 'rsync' or something like that instead ;-)
assert(not(src_remote == False and dst_remote == False))
info(u"Verifying attributes...")
cfg = Config()
## Items left on src_list will be transferred
## Items left on update_list will be transferred after src_list
## Items left on copy_pairs will be copied from dst1 to dst2
update_list = FileDict(ignore_case = False)
## Items left on dst_list will be deleted
copy_pairs = []
debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
for relative_file in src_list.keys():
debug(u"CHECK: %s" % (relative_file))
if dst_list.has_key(relative_file):
## Was --skip-existing requested?
if cfg.skip_existing:
debug(u"IGNR: %s (used --skip-existing)" % (relative_file))
del(src_list[relative_file])
del(dst_list[relative_file])
continue
try:
same_file = _compare(src_list, dst_list, src_remote, dst_remote, relative_file)
except (IOError,OSError):
debug(u"IGNR: %s (disappeared)" % (relative_file))
warning(u"%s: file disappeared, ignoring." % (relative_file))
del(src_list[relative_file])
del(dst_list[relative_file])
continue
if same_file:
debug(u"IGNR: %s (transfer not needed)" % relative_file)
del(src_list[relative_file])
del(dst_list[relative_file])
else:
# look for matching file in src
try:
md5 = src_list.get_md5(relative_file)
except IOError:
md5 = None
if md5 is not None and dst_list.by_md5.has_key(md5):
# Found one, we want to copy
dst1 = list(dst_list.by_md5[md5])[0]
debug(u"DST COPY src: %s -> %s" % (dst1, relative_file))
copy_pairs.append((src_list[relative_file], dst1, relative_file))
del(src_list[relative_file])
del(dst_list[relative_file])
else:
# record that we will get this file transferred to us (before all the copies), so if we come across it later again,
# we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).
dst_list.record_md5(relative_file, md5)
update_list[relative_file] = src_list[relative_file]
del src_list[relative_file]
#.........這裏部分代碼省略.........
示例8: fromDir
class Job:
@staticmethod
def fromDir(jobDir, interactive=False, quiet=False):
if not os.path.isdir(jobDir):
raise Exception('you must give a directory: ' + jobDir)
home, name = os.path.split(jobDir)
return Job(home, name, interactive, newjob=False, quiet=quiet)
@staticmethod
def jobsRunning():
jobs = set()
lines = subprocess.check_output('qstat').strip()
if len(lines) == 0:
return jobs
ar = lines.split('\n')
assert len(ar) >= 3 # first two lines are formatting
for jstr in ar[2:]:
jobs.add(int(jstr.split()[0]))
return jobs
@staticmethod
def killEverything():
for jid in Job.jobsRunning():
os.system('qdel '+jid)
def __init__(self, homeDir, name, f_hasFailed, interactive=False, newjob=True, quiet=False):
"""
homeDir is where experiments are stored
name is the name of this experiment
if there is another job with the same name, they are assumed
to be the same. delete old jobs that you do not want to be confused with
f_hasFailed is a function that takes a log file and returns a boolean for
whether or not the job has failed
if in interactive mode, will prompt user for input
this should not be used for scripts, which would hang forever in these cases
"""
self.name = name
self.home = os.path.join(homeDir, name)
self.logDir = os.path.join(self.home, 'log')
self.f_hasFailed = f_hasFailed
if os.path.isfile(self.home):
raise Exception('you the home directory you gave must not have any files or folders that match the name you gave')
if newjob:
print 'creating new job:', self.home
if os.path.isdir(self.home):
if interactive:
print "the directory %s already exists" % (self.home)
r = raw_input('would you like me to delete it for you? [y|n] ')
if str2bool(r):
os.system('rm -r ' + self.home)
os.system('mkdir ' + self.home)
else:
raise Exception('cannot proceed')
else:
raise Exception('please give a unique name or delete old jobs: ' + self.home)
elif 0 != os.system('mkdir -p ' + self.home):
raise Exception('cannot make home directory!')
if not os.path.isdir(self.logDir) and 0 != os.system('mkdir -p ' + self.logDir):
raise Exception('cannot make log directory!')
else:
assert os.path.isdir(self.home)
assert os.path.isdir(self.logDir)
if not quiet: print 'loading existing job:', self.home
self.javaOpt = FileDict(os.path.join(self.home, 'java.settings'), exists=not newjob) # start with "-D"
self.metaOpt = FileDict(os.path.join(self.home, 'meta.settings'), exists=not newjob) # xmx, jar, profile, etc
self.qsubOpt = FileDict(os.path.join(self.home, 'qsub.settings'), exists=not newjob) # mem_free, h_rt
self.prepared = False
def getResourceDirectory(self, parentFolderName, childFolderName, overwrite=True):
'''see getResourceFile for details'''
folder = os.path.join(self.home, parentFolderName, childFolderName)
if os.path.isdir(folder):
if not overwrite: raise
else:
os.system('mkdir -p ' + folder)
return folder
def getResourceFile(self, folderName, fileName, overwrite=True):
'''
example usage: folderName='diagnostics', fileName='parameters.txt',
this just returns a path to home/diagnostics/parameters.txt
resources are usefule for job-specific output (avoid job output collision)
'''
folder = os.path.join(self.home, folderName)
if not os.path.isdir(folder):
os.system('mkdir ' + folder)
f = os.path.join(folder, fileName)
if not overwrite and os.path.isfile(f):
raise Exception('this file already exists! ' + f)
return f
def addLib(self, dirOrFile):
if os.path.isfile(dirOrFile):
cp = selt.class_path() + ':' + dirOrFile
#.........這裏部分代碼省略.........