Python Disco.purge方法代码示例

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import purge [as 别名]
def data_gen(path):
    return "\n".join([path[1:]] * 10)


def fun_map(e, params):
    return [("=" + e, e)]


def fun_reduce(iter, out, params):
    s = 1
    for k, v in iter:
        if k != "=" + v:
            raise Exception("Corrupted key")
        s *= int(v)
    out.add("result", s)


tserver.run_server(data_gen)

inputs = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31]
job = Disco(sys.argv[1]).new_job(
    name="test_simple", input=tserver.makeurl(inputs), map=fun_map, reduce=fun_reduce, nr_reduces=1, sort=False
)

if list(result_iterator(job.wait())) != [("result", ANS)]:
    raise Exception("Invalid answer")

job.purge()
print "ok"

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import purge [as 别名]
        try:
            self.target         = str(kwargs.pop('target') or '')
            self.mapfilters     = filter(None, kwargs.pop('mapfilters').split('|'))
            self.reducefilters  = filter(None, kwargs.pop('reducefilters').split('}'))
            self.resultsfilters = filter(None, kwargs.pop('resultsfilters').split(']'))
            job = self.job.run()
        except DiscoError, e:
            return HttpResponseServerError("Failed to run DiscoDB job: %s" % e)

        try:
            results = Results(job.results)
        except DiscoError, e:
            return HttpResponseServerError("DiscoDB job failed: %s" % e)
        finally:
            if os.path.exists(purge_file):
                disco_master.purge(job.name)

        return results.response(request)

class KeysResource(DiscoDBResource):
    job_type    = KeyIterator

class ValuesResource(DiscoDBResource):
    job_type    = ValuesIterator

class ItemsResource(DiscoDBResource):
    job_type    = ItemsIterator

class QueryCollection(Collection):
    allowed_methods = ('GET', 'POST')

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import purge [as 别名]
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])

results = disco.new_job(name = "test_chain_0", input = tserver.makeurl([""] * 100),
                map = fun_map, reduce = fun_reduce, nr_reduces = 4,
                sort = False, params = {'suffix': '0'}).wait()

i = 1
while i < 10:
        nresults = disco.new_job(name = "test_chain_%d" % i, input = results,
                map = fun_map, reduce = fun_reduce, nr_reduces = 4,
                map_reader = chain_reader, sort = False,
                params = {'suffix': str(i)}).wait()

        disco.purge(jobname(results[0]))
        results = nresults
        i += 1

for key, value in result_iterator(results):
        if key[:5] not in ani or key[5:] != "0-1-2-3-4-5-6-7-8-9-":
                raise "Corrupted key: %s" % key
        if value != "9":
                raise "Corrupted value: %s" % value

disco.purge(jobname(results[0]))

print "ok"

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import purge [as 别名]
                input = tserver.makeurl([""] * 5),
                map = m))

res = []
while jobs:
        cont = False
        ready, jobs = disco.results(jobs, timeout = 2000)
        res += ready

for n, r in res:
        if n.startswith("test_waitmany_3"):
                if r[0] != "dead":
                        raise Exception("Invalid job status: %s" % n)
        elif r[0] != "ready":
                raise Exception("Invalid job status: %s" % n)
        disco.purge(n)

if len(res) != 4:
        raise Exception("Invalid number of results")






print "ok"
                        
         

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import purge [as 别名]
tserver.run_server(data_gen)

disco = Disco(sys.argv[1])
num = sum(x['max_workers'] for x in disco.nodeinfo()['available'])
print >> sys.stderr, num, "slots available"
inputs = tserver.makeurl(range(num * 10))
random.shuffle(inputs)

jobs = []
for i in range(5):
        jobs.append(disco.new_job(name = "test_async_%d" % i,
                       input = inputs[i * (num * 2):(i + 1) * (num * 2)],
                       map = fun_map, reduce = fun_reduce, nr_reduces = 11,
                       sort = False))
        time.sleep(1)

all = dict(("[%s]" % i, 0) for i in range(num * 10))
while jobs:
        ready, jobs = disco.results(jobs)
        for name, results in ready:
                for k, v in result_iterator(results[1]):
                        all[k] += 1
                disco.purge(name)

for v in all.values():
        if v != 10:
                raise "Invalid results: %s" % all

print "ok"