当前位置: 首页>>代码示例>>Python>>正文


Python Disco.wait方法代码示例

本文整理汇总了Python中disco.core.Disco.wait方法的典型用法代码示例。如果您正苦于以下问题:Python Disco.wait方法的具体用法?Python Disco.wait怎么用?Python Disco.wait使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在disco.core.Disco的用法示例。


在下文中一共展示了Disco.wait方法的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: IndexJob

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
class IndexJob(object):
    def __init__(self, spec, discodex, disco_addr="disco://localhost", profile=False):
        # TODO(sqs): refactoring potential with PagerankJob
        self.spec = spec
        self.discodex = discodex
        self.docset = Docset(spec.docset_name)
        self.disco = Disco(DiscoSettings()["DISCO_MASTER"])
        self.nr_partitions = 8
        self.profile = profile

    def start(self):
        results = self.__run_job(self.__index_job())
        self.__run_discodex_index(results)

    def __run_job(self, job):
        results = job.wait()
        if self.profile:
            self.__profile_job(job)
        return results

    def __index_job(self):
        return self.disco.new_job(
            name="index_tfidf",
            input=["tag://" + self.docset.ddfs_tag],
            map_reader=docparse,
            map=TfIdf.map,
            reduce=TfIdf.reduce,
            sort=True,
            partitions=self.nr_partitions,
            partition=TfIdf.partition,
            merge_partitions=False,
            profile=self.profile,
            params=dict(doc_count=self.docset.doc_count),
        )

    def __run_discodex_index(self, results):
        opts = {
            "parser": "disco.func.chain_reader",
            "demuxer": "freequery.index.tf_idf.TfIdf_demux",
            "nr_ichunks": 1,  # TODO(sqs): after disco#181 fixed, increase this
        }
        ds = DataSet(input=results, options=opts)
        origname = self.discodex.index(ds)
        self.disco.wait(origname)  # origname is also the disco job name
        self.discodex.clone(origname, self.spec.invindex_name)
开发者ID:sqs,项目名称:freequery,代码行数:47,代码来源:job.py

示例2: data_gen

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
def data_gen(path):
    return "\n".join([path[1:]] * 10)


def fun_map(e, params):
    return [("=" + e, e)]


def fun_reduce(iter, out, params):
    s = 1
    for k, v in iter:
        if k != "=" + v:
            raise Exception("Corrupted key")
        s *= int(v)
    out.add("result", s)


tserver.run_server(data_gen)

inputs = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31]
job = Disco(sys.argv[1]).new_job(
    name="test_simple", input=tserver.makeurl(inputs), map=fun_map, reduce=fun_reduce, nr_reduces=1, sort=False
)

if list(result_iterator(job.wait())) != [("result", ANS)]:
    raise Exception("Invalid answer")

job.purge()
print "ok"
开发者ID:jseppanen,项目名称:disco,代码行数:31,代码来源:test_simple.py

示例3: DAMAGES

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from disco.core import Disco, result_iterator
from disco.util import external

ext_map_exec = "java_map.sh"
ext_reduce_exec = "java_reduce.sh"
map_class = "rmaus.disco.external.sample.WordCountMap"
reduce_class = "rmaus.disco.external.sample.WordCountReduce"

job = Disco("http://discomaster-dr-01:8989").new_job(
        name = "java_wordcount",
	input = ["raw://foo", "raw://bar", "raw://foo"],
	ext_params = { "mapFunction" : map_class, "reduceFunction" : reduce_class, "testKey" : "testValue" },
        map = external([ext_map_exec]),
        reduce = external([ext_reduce_exec]))

results = job.wait(show=True)

for result in sorted(result_iterator(results), key=lambda x:x[1]):
	print result
开发者ID:ryan-maus,项目名称:disco-java-ext,代码行数:32,代码来源:RunDiscoJavaExt.py

示例4: Disco

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
                out.add(k, v)

tserver.run_server(data_gen)

job = Disco(sys.argv[1]).new_job(\
        name = "test_profile",\
        input = tserver.makeurl([""] * int(100)),\
        map = really_unique_function_name,\
        reduce = fun_reduce,\
        nr_reduces = 30,\
        sort = False,\
        profile = True)

ANS = {"gutta": int(1e4), "cavat": int(2e4), "capidem": int(1e4)}
i = 0
for key, value in result_iterator(job.wait()):
        i += 1
        if ANS[key] == int(value):
                print "Correct: %s %s" % (key, value)
        else:
                raise "Results don't match (%s): Got %d expected %d" %\
                        (key, int(value), ANS[key])
if i != 3:
        raise "Too few results"

buf = cStringIO.StringIO()
sys.stdout = buf
job.profile_stats().print_stats()
sys.stdout = sys.__stdout__

#stats = job.profile_stats()
开发者ID:chenhao7512,项目名称:disco,代码行数:33,代码来源:test_profile.py

示例5: fun_map

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
import sys
from disco.core import Disco, result_iterator

def fun_map(e, params):
        for i in range(3):
                msg("--special_test_string_%d--" % i)
        return [(e, "")]

inputs = ["raw://discoapi"]

job = Disco(sys.argv[1]).new_job(name = "test_discoapi",
        input = inputs,
        map = fun_map)

r = list(result_iterator(job.wait()))
if [("discoapi", "")] != r:
        raise Exception("Invalid result: <%s> " % r)

n = job.jobspec()["name"]
if not n.startswith("test_discoapi"):
        raise Exception("Invalid jobspec: Expected name prefix test_discoapi, "\
                        "got %s" % n)

events = [ev[2] for offs, ev in job.events()]

for i in range(3):
        m = "--special_test_string_%d--" % i
        if not [x for x in events if m in x]:
                raise Exception("Message '%s' not found in events" % m)

job.purge()
开发者ID:chenhao7512,项目名称:disco,代码行数:33,代码来源:test_discoapi.py

示例6: Disco

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
tserver.run_server(data_gen)

inputs = ["01/11/1965", "14/03/1983", "12/12/2002"]

job = Disco(sys.argv[1]).new_job(name = "test_objectrw",
                input = tserver.makeurl(inputs),
                map = fun_map,
                map_writer = func.object_writer,
                reduce = fun_reduce, 
                reduce_reader = func.object_reader,
                reduce_writer = func.object_writer,
                required_modules = ["math", "datetime", "time"],
                nr_reduces = 1,
                sort = False)

i = 0
for k, v in result_iterator(job.wait(), reader = func.object_reader):
        if k["PI2"] != math.pi:
                raise "Invalid key: %s" % k
        if v.strftime("%d/%m/%Y") not in inputs:
                raise "Invalid value: %s" % v
        i += 1

if i != 30:
        raise "Wrong number of results, got %d, expected 30" % i

job.purge()

print "ok"
开发者ID:chenhao7512,项目名称:disco,代码行数:31,代码来源:test_objectrw.py

示例7: Disco

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
tserver.run_server(data_gen)
inputs = tserver.makeurl([1])
job = Disco(sys.argv[1]).new_job(name = "test_ratelimit",
        input = inputs, map = fun_map)

time.sleep(5)
check_dead(job)

job = Disco(sys.argv[1]).new_job(name = "test_ratelimit2",
        input = inputs, map = fun_map2, status_interval = 1)

time.sleep(5)
check_dead(job)

job = Disco(sys.argv[1]).new_job(name = "test_ratelimit3",
        input = inputs, map = fun_map3, status_interval = 1)

time.sleep(5)
check_dead(job)

job = Disco(sys.argv[1]).new_job(name = "test_ratelimit4",
        input = inputs, map = fun_map2, status_interval = 0)
job.wait()
job.purge()

print "ok"



开发者ID:chenhao7512,项目名称:disco,代码行数:28,代码来源:test_ratelimit.py

示例8: data_gen

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
fail = ["1", "2", "3"]

def data_gen(path):
        lock.acquire()
        e = path[1:]
        if e in fail:
                fail.remove(e)
                lock.release()
                raise tserver.FailedReply()
        else:
                lock.release()
                return str(int(e) * 10) + "\n"

def fun_map(e, params):
        return [(int(e) * 10, "")]

tserver.run_server(data_gen)

job = Disco(sys.argv[1]).new_job(
        name = "test_tempfail",
        input = tserver.makeurl(map(str, range(10))),
        map = fun_map)

res = sum(int(x) for x, y in result_iterator(job.wait()))
if res != 4500:
        raise Exception("Invalid result: Got %d, expected 4500" % res)

job.purge()
print "ok"

开发者ID:chenhao7512,项目名称:disco,代码行数:31,代码来源:test_tempfail.py

示例9: Exception

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
        s = 1
        for k, v in iter:
                if k != "=" + v:
                        raise Exception("Corrupted key")
                s *= int(v)
        out.add("result", s)

tserver.run_server(data_gen)

inputs = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31]

job = Disco(sys.argv[1]).new_job(
                name = "test_writers", 
                input = tserver.makeurl(inputs),
                map = fun_map,
                map_writer = fun_map_writer,
                reduce = fun_reduce, 
                reduce_reader = fun_reduce_reader,
                reduce_writer = fun_reduce_writer,
                nr_reduces = 1,
                sort = False)

res = list(result_iterator(job.wait(), reader = result_reader))

if res != [ANS]:
        raise Exception("Invalid answer: %s" % res)

job.purge()

print "ok"
开发者ID:chenhao7512,项目名称:disco,代码行数:32,代码来源:test_writers.py

示例10: fun_map

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
import sys
from disco.core import Disco, result_iterator

def fun_map(e, params):
        return [("", e + ":map")]

inputs = ["raw://eeny", "raw://meeny", "raw://miny", "raw://moe"]

job = Disco(sys.argv[1]).new_job(name = "test_raw",
        input = inputs,
        map = fun_map)

res = dict((x[6:] + ":map", True) for x in inputs)

for x in result_iterator(job.wait()):
        if x[1] not in res:
                raise "Invalid result: <%s> " % x[1]
        del res[x[1]]

if res:
        raise "Invalid number of results %d" %\
                (len(inputs) - len(res))

job.purge()

print "ok"

开发者ID:chenhao7512,项目名称:disco,代码行数:28,代码来源:test_raw.py

示例11: Disco

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
tserver.run_server(data_gen)

inputs = ["apple", "orange", "pear"]

job = Disco(sys.argv[1]).new_job(
        name="test_streams",
        input=tserver.makeurl(inputs),
        map=fun_map,
        reduce=fun_reduce,
        nr_reduces=1,
        map_reader = map_reader,
        map_input_stream =
                [map_input_stream, map_input1, map_input2, map_input3],
        reduce_output_stream = [reduce_output1, reduce_output2])

for k, v in result_iterator(job.wait(),
                input_stream = [resultiter_input1, map_input_stream]):

        if not k.startswith("red:cba"):
                raise Exception("Invalid prefix in key. Got '%s' "\
                        "expected prefix 'red:cba'" % k)

        if k[7:] not in inputs:
                raise Exception("Invalid result '%s'" % k)
        inputs.remove(k[7:])

if inputs:
        raise Exception("Expected 3 results, got %d" % 3 - len(inputs))

print "ok"
开发者ID:davin,项目名称:disco,代码行数:32,代码来源:test_streams.py

示例12: Disco

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
                out.add("red_" + k, "red_" + v)
        
tserver.run_server(data_gen)

inputs = ["ape", "cat", "dog"]
params = {"test1": "1,2,3",\
          "one two three": "dim\ndam\n",\
          "dummy": "value"}

job = Disco(sys.argv[1]).new_job(
            name = "test_external",
            input = tserver.makeurl(inputs),
            map = external(["ext_test"]), 
            reduce = fun_reduce, 
            ext_params = params,
            nr_reduces = 1,
            sort = False)

results = sorted([(v, k) for k, v in result_iterator(job.wait())])
for i, e in enumerate(results): 
        v, k = e
        if k != "red_dkey" or v != "red_test_%s" % inputs[i / 3]:
                raise Exception("Invalid answer: %s, %s" % (k, v))

if len(results) != 9:
        raise Exception("Wrong number of results: %u vs. 9" % len(results))

job.purge()

print "ok"
开发者ID:chenhao7512,项目名称:disco,代码行数:32,代码来源:test_external.py

示例13: str

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
                raise tserver.FailedReply()
        else:
                return str(int(path[2:]) * 10) + "\n"

def fun_map(e, params):
        return [(e, "")]

def fun_reduce(iter, out, params):
        s = 0
        for k, v in iter:
                s += int(k)
        out.add(s, "")

tserver.run_server(data_gen)

inputs = ["X1", ["2_fail", "2_still_fail", "X200"], "X3", ["4_fail", "X400"]]

job = Disco(sys.argv[1]).new_job(
        name = "test_redundant",
        input = tserver.makeurl(inputs),
        map = fun_map,
        reduce = fun_reduce,
        nr_reduces = 1)

if result_iterator(job.wait()).next()[0] != "6040":
        raise Exception("Invalid result: Got %s, expected 6040" % res)

job.purge()
print "ok"

开发者ID:chenhao7512,项目名称:disco,代码行数:31,代码来源:test_redundant.py

示例14: data_gen

# 需要导入模块: from disco.core import Disco [as 别名]
# 或者: from disco.core.Disco import wait [as 别名]
import sys, tserver
from disco.core import Disco, result_iterator

def data_gen(path):
        return "\n".join([path[1:]])

def fun_map(e, params):
        x = extramodule1.magic(int(e))
        y = extramodule2.kungfu(x)
        return [("", y)]

inputs = ["123"]

tserver.run_server(data_gen)

job = Disco(sys.argv[1]).new_job(name = "test_requiredfiles",
        input = tserver.makeurl(inputs),
        required_files = ["extramodule1.py", "extramodule2.py"],
        required_modules = ["extramodule1", "extramodule2"],
        map = fun_map)

exp = int(inputs[0]) ** 2 + 2
got = int([y for x, y in result_iterator(job.wait())][0])
if exp != got:
        raise "Wrong result! expected %d, got %d" % (exp, got)

job.purge()

print "ok"

开发者ID:chenhao7512,项目名称:disco,代码行数:31,代码来源:test_requiredfiles.py


注:本文中的disco.core.Disco.wait方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。