Python ddfs.DDFS類代碼示例

def load_oob(host, name, key):
    from disco.ddfs import DDFS
    ddfs = DDFS(host)
    # NB: this assumes that blobs are listed in LIFO order.
    # We want to return the latest version
    for fd in ddfs.pull(ddfs.job_oob(name), blobfilter=lambda x: x == key):
        return fd.read()

def push_data(self, args):
  path = args['path']
  ds_id = args['ds_id']

  filename = os.path.basename(path)
  tmp_dir = str(int(time.time()))

  # Create temporary files
  os.chdir(config.DISCO_FILES)
  os.makedirs(tmp_dir)
  copy2(filename, "%s/%s" % (tmp_dir, filename))
  os.chdir(tmp_dir)

  command = 'split -n %s %s' % (config.DISCO_NODES, path)
  split_process = Popen(command.split(' '), stdout=PIPE)
  split_process.communicate()

  # Push data to cluster
  command = 'ddfs push data:%s ./xa?' % ds_id
  d = DDFS('disco://localhost')
  files = [("%s/%s/%s" % (config.DISCO_FILES, tmp_dir, filename), filename) for filename in os.listdir(".") if filename.startswith("xa")]
  d.push('data:%s' % ds_id, files)

  r.table('datasets').filter({
      'id': ds_id,
  }).update({
      'state': 'ready_for_crunching'
  }).run(db)

def delete_all():
	'''
	Deletes all tags in DDFS, thus orphaning all blobs and making them subject to eventual removal by the garbage collector.
	'''
	ddfs = DDFS()
	for tag in ddfs.list():
		ddfs.delete(tag)

def ddfs_save(blobs, name, master):
    from disco.ddfs import DDFS
    ddfs = DDFS(master)
    blobs = [(blob, ('discoblob:%s:%s' % (name, os.path.basename(blob))))
             for blob in blobs]
    tag = ddfs_name(name)
    ddfs.push(tag, blobs, retries=600, delayed=True, update=True)
    return "tag://%s" % tag

def inputexpand(input, partition=None, settings=DiscoSettings()):
    from disco.ddfs import DDFS, istag
    if ispartitioned(input) and partition is not False:
        return zip(*(parse_dir(i, partition=partition) for i in iterify(input)))
    if isiterable(input):
        return [inputlist(input, partition=partition, settings=settings)]
    if istag(input):
        ddfs = DDFS(settings=settings)
        return chainify(blobs for name, tags, blobs in ddfs.findtags(input))
    return [input]

def push_by_tag(file_paths,tag=None):
	'''
	'''
	ddfs = DDFS()
	if tag is None:
		for file_path in file_paths:
			tag = file_path.split("/")[-1].split(".")[0]
			ddfs.push(tag,[file_path])
	else:
		ddfs.push(tag,file_paths)

 def setUp(self):
     tag = 'disco:test:authjob'
     self.ddfs = DDFS(self.disco_master_url)
     pushed = self.ddfs.push(tag, [(StringIO('blobdata'), 'blob')])
     self.ddfs.setattr(tag, 'ddfs:read-token', 'r')
     self.input = ['tag://u:[email protected]/' + tag]
     super(AuthJobTestCase, self).setUp()

 def __init__(self, docset_name):
     self.ddfs_tag = docset_name
     self.ddfs_index_tag = docset_name + ':index'
     self.ddfs_link_file_tag = docset_name + ':links'
     self.ddfs = DDFS()
     self.__index = None
     self.dirty = False

def push_by_tag(file_paths, tag=None):
    '''
    '''
    ddfs = DDFS()
    if tag is None:
        for file_path in file_paths:
            tag = os.path.splitext(ntpath.basename(file_path))[0]
            try:
                ddfs.push(tag, [file_path])
            except IOError:
                print("Invalid file path specified.")
    else:
        try:
            ddfs.push(tag, file_paths)
        except IOError:
            print("Invalid file path specified.")

 def setUp(self):
     self.ddfs = DDFS(self.disco_master_url)
     self.ddfs.push('disco:test:authrd', [(StringIO('datablob'), 'blobdata')])
     self.ddfs.push('disco:test:authwr', [(StringIO('datablob'), 'blobdata')])
     self.ddfs.setattr('disco:test:authrd', 'a', 'v')
     self.ddfs.setattr('disco:test:authwr', 'a', 'v')
     self.ddfs.setattr('disco:test:authrd', 'ddfs:read-token', 'rdr')
     self.ddfs.setattr('disco:test:authwr', 'ddfs:write-token', 'wtr')

 def put(self, key, value):
     """
     Stores an out-of-band result *value* with the key *key*. Key must be unique in
     this job. Maximum key length is 256 characters. Only characters in the set
     ``[a-zA-Z_\-:[email protected]]`` are allowed in the key.
     """
     if DDFS.safe_name(key) != key:
         raise DiscoError("OOB key contains invalid characters (%s)" % key)
     util.save_oob(self.master, self.jobname, key, value)

class AuthJobTestCase(DiscoJobTestFixture, DiscoTestCase):
    input = []

    @staticmethod
    def map(e, params):
        return [(e.strip(), '')]

    @property
    def answers(self):
        return [('blobdata', '')]

    def setUp(self):
        tag = 'disco:test:authjob'
        self.ddfs = DDFS(self.disco_master_url)
        pushed = self.ddfs.push(tag, [(StringIO('blobdata'), 'blob')])
        self.ddfs.setattr(tag, 'ddfs:read-token', 'r')
        self.input = ['tag://u:[email protected]/' + tag]
        super(AuthJobTestCase, self).setUp()

    def tearDown(self):
        super(AuthJobTestCase, self).tearDown()
        self.ddfs.delete('disco:test:authjob')

    def __tag_results(self, results):
        from disco.ddfs import DDFS
        ddfs = DDFS()
        results_tag = results[0]
        ddfs.put(self.docset.ddfs_link_file_tag, list(ddfs.blobs(results_tag)))

        # remove old, temporary tag
        ddfs.delete(results_tag)

    def put(self, key, value):
        """
        Stores an out-of-band result *value* (bytes) with the key *key*.

        Key must be unique in this job.
        Maximum key length is 256 characters.
        Only characters in the set ``[a-zA-Z_\-:[email protected]]`` are allowed in the key.
        """
        from disco.ddfs import DDFS
        from disco.util import save_oob
        from disco.error import DiscoError
        if DDFS.safe_name(key) != key:
            raise DiscoError("OOB key contains invalid characters ({0})".format(key))
        save_oob(self.master, self.jobname, key, value)

 def __init__(self,
              host='',
              jobfile='',
              jobname='',
              master=None,
              disco_port=None,
              put_port=None,
              ddfs_data='',
              disco_data='',
              stage=None,
              group=None,
              grouping=None,
              taskid=-1):
     from disco.job import JobPack
     from disco.ddfs import DDFS
     self.host = host
     self.jobfile = jobfile
     self.jobname = jobname
     self.jobpack = JobPack.load(open(jobfile, 'rb'))
     self.jobobjs = dPickle.loads(self.jobpack.jobdata)
     self.master = master
     self.disco_port = disco_port
     self.put_port = put_port
     self.ddfs_data = ddfs_data
     self.disco_data = disco_data
     self.stage = stage
     self.group = '{0[0]}-{0[1]}'.format(group)
     self.group_label, self.group_host = group
     self.grouping = grouping
     self.taskid = taskid
     self.outputs = {}
     self.uid = '{0}:{1}-{2}-{3}-{4}'.format(self.stage,
                                             DDFS.safe_name(self.group),
                                             self.taskid,
                                             hexhash(str((time.time())).encode()),
                                             os.getpid())