Python ThreadPool.imap_unordered方法代码示例

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
 def run(self):
     pool = ThreadPool(processes=self._worker_number)
     if self._batch > 1:
         pool.imap_unordered(func=self._func_wrap_batch, iterable=iter(self._forerver_get_batch, None))
     else:
         pool.imap_unordered(func=self._func_wrap, iterable=iter(self._forever_get, None))
     while not self._stop_event.is_set():
         time.sleep(0.01)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
class SiteThreadChecker(SiteChecker):
    pool_size_key = "thread_pool_size"

    def __init__(self, *args, thread_pool_size=1, **kwargs):
        #super(SiteThreadChecker, self).__init__(*args, **kwargs)
        SiteChecker.__init__(self, *args, output_buff_size=thread_pool_size*50, **kwargs)
        self.max_thread = thread_pool_size
        LinkChecker.max_http_connection = self.max_thread
        LinkChecker.max_pool_size = self.max_thread
        #self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1)
        self.pool = ThreadPool(processes=self.max_thread)
        self._set_task_control_max(self.max_thread)
        #self.temp_queue = Queue(self.max_thread * 2)
        #self.temp_queue.put(self.page_list[0])
        #self.pump = OnSiteLinkPump(self.temp_queue, self.page_list)
        #print("init siteThreadChecker finished")

    @staticmethod
    def get_input_parameter(full_link: str, max_page:int, max_level: int, output_queue, pool_size: int):
        temp = SiteChecker.get_input_parameter_base(full_link, max_page, max_level, output_queue)
        temp.update({SiteThreadChecker.pool_size_key: pool_size})
        return temp

    def additional_reset(self):
        if self.pool is not None:
            self.pool.terminate()
            self.pool = ThreadPool(processes=self.max_thread)
            #self.pool = multiprocessing.Pool(processes=self.max_thread, maxtasksperchild=1)

    def addtional_clear(self):
        if self.pool is not None:
            self.pool.terminate()

    def stop(self):
        try:
            self.data_source.set_continue_lock(False)
            self.pool.terminate()
        except:
            pass
        super(SiteThreadChecker, self).stop()

    def begin_crawl(self, level=0):
        #while self.can_continue() and self.data_source.can_continue():
        #print("continue to work, page limit:", self.max_page, " max_level: ", self.max_level)
        #target_func = functools.partial(PageChecker.crawl_page, self)
        try:
            self.pool.imap_unordered(PageChecker.crawl_page_for_iter, self.data_source)
            while self.data_source.can_continue():
                time.sleep(0.1)
            #results = [self.pool.apply_async(PageChecker.crawl_page, args=(self, page))
            #           for page in self.data_source.get_next(OnSiteLink.TypeOnSite, ResponseCode.LinkOK)]
            #[p.get() for p in results]
        except Exception as ex:
            #self.stop()
            msg = "begin_crawl() " + str(self.get_site_info())
            ErrorLogger.log_error("SiteThreadChecker", ex, msg)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def collect_stats(args):
  bots_json = swarming_query('bots', '--limit', '10000')
  if 'error' in bots_json:
    return bots_json
  bots = bots_json.get('items', [])
  stats = {
      'bots_count': len(bots),
      'bots': {},
  }
  logging.info('Found %d bots; collecting tasks.', stats['bots_count'])
  pool = ThreadPool(100)
  count = 0

  def func(bot):
    return process_bot(args, bot)

  try:
    for bot_id, bot_results in pool.imap_unordered(func, bots):
      count += 1
      stats['bots'][bot_id] = bot_results
      logging.info('%4d of %4d (%2.0f%%) of bots processed',
                   count, stats['bots_count'],
                   count * 100 / stats['bots_count'])
  except KeyboardInterrupt as e:
    pass
  return stats

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
        class parallel_map(collections.Iterable):

            def __init__(self, pool_size, function, *iterables):
                if not isinstance(pool_size, numbers.Integral):
                    raise TypeError('pool_size must be an integer, not ' +
                                    repr(pool_size))
                elif not callable(function):
                    raise TypeError('function must be callable, not ' +
                                    repr(function))
                elif not iterables:
                    raise TypeError('missing iterable')
                self.pool = ThreadPool(pool_size)
                self.function = function
                self.results = self.pool.imap_unordered(self.map_function,
                                                        zip(*iterables))

            def map_function(self, args):
                try:
                    value = self.function(*args)
                except Exception:
                    return False, sys.exc_info()
                return True, value

            def __iter__(self):
                errors = []
                for success, value in self.results:
                    if success:
                        yield value
                    else:
                        errors.append(value)
                self.pool.close()
                self.pool.join()
                for error in errors:
                    exec('raise error[1], None, error[2]')

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def get_for_genres(genres):
    genres = set(genres)
    playlists = {}
    new_genres = set()

    for page in xrange(5):
        args = []
        for g in genres:
            args.append((g, page))

        try:
            pool = ThreadPool(PROCESSES)
            pfunc = parse_page
            for i, res in enumerate(pool.imap_unordered(pfunc, args)):
                genre, page, pl, found = res
                print "%d/%d" % (i + 1, len(args))
                playlists.update(pl)
                new_genres |= found
                if not pl:
                    genres.remove(genre)
        except Exception as e:
            print e
            return playlists, []
        finally:
            pool.terminate()
            pool.join()

    return playlists, new_genres

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def get_first_result_from_threads(calls):
    calls = list(enumerate(calls))

    def run_func(call):
        i, call = call
        func = call[0]
        args = call[1] if len(call)>1 else []
        kwargs = call[2] if len(call)>2 else {}
        try:
            return i, func(*args, **kwargs)
        except Exception as e:
            return i, e

    pool = ThreadPool(processes=len(calls))
    result = pool.imap_unordered(run_func, calls).next()

    for thread in pool._pool:
        # via http://stackoverflow.com/a/15274929
        if not thread.isAlive():
            continue
        exc = ctypes.py_object(SystemExit)
        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
            ctypes.c_long(thread.ident), exc)
        if res == 0:
            raise ValueError("nonexistent thread id")
        elif res > 1:
            # """if it returns a number greater than one, you're in trouble,
            # and you should call it again with exc=NULL to revert the effect"""
            ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, None)
            raise SystemError("PyThreadState_SetAsyncExc failed")

    return result

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
    def _fit(self, dataset):
        est = self.getOrDefault(self.estimator)
        epm = self.getOrDefault(self.estimatorParamMaps)
        numModels = len(epm)
        eva = self.getOrDefault(self.evaluator)
        tRatio = self.getOrDefault(self.trainRatio)
        seed = self.getOrDefault(self.seed)
        randCol = self.uid + "_rand"
        df = dataset.select("*", rand(seed).alias(randCol))
        condition = (df[randCol] >= tRatio)
        validation = df.filter(condition).cache()
        train = df.filter(~condition).cache()

        subModels = None
        collectSubModelsParam = self.getCollectSubModels()
        if collectSubModelsParam:
            subModels = [None for i in range(numModels)]

        tasks = _parallelFitTasks(est, train, eva, validation, epm, collectSubModelsParam)
        pool = ThreadPool(processes=min(self.getParallelism(), numModels))
        metrics = [None] * numModels
        for j, metric, subModel in pool.imap_unordered(lambda f: f(), tasks):
            metrics[j] = metric
            if collectSubModelsParam:
                subModels[j] = subModel

        train.unpersist()
        validation.unpersist()

        if eva.isLargerBetter():
            bestIndex = np.argmax(metrics)
        else:
            bestIndex = np.argmin(metrics)
        bestModel = est.fit(dataset, epm[bestIndex])
        return self._copyValues(TrainValidationSplitModel(bestModel, metrics, subModels))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def main():
    assert sys.version_info[0] == 3

    moduleset_versions = get_moduleset_versions()

    pool = ThreadPool(20)
    pool_iter = pool.imap_unordered(_fetch_version, moduleset_versions.keys())

    arch_versions = {}
    for i, some_versions in enumerate(pool_iter):
        arch_versions.update(some_versions)

    for name, version in sorted(moduleset_versions.items()):
        arch_name = fix_name(name)
        if arch_name in arch_versions:
            arch_version, arch_url = arch_versions[arch_name]
            arch_version = arch_version.split("+", 1)[0]
            if arch_name == "readline":
                arch_version = ".".join(arch_version.split(".")[:2])
        else:
            arch_version = "???"
            arch_url = ""

        if is_maybe_newer(arch_version, version):
            print("%-30s %-20s %-20s %s" % (
                name, version, arch_version, arch_url))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def main(argv):
    """Go Main Go"""
    scenario = int(argv[1])
    sdf = load_scenarios()
    queue = realtime_run(sdf.loc[scenario], scenario)
    pool = ThreadPool()  # defaults to cpu-count
    sz = len(queue)
    failures = 0

    def _run(row):
        """ Run ! """
        wr = WeppRun(row[0], row[1], row[2], scenario)
        return wr.run()

    sts00 = datetime.datetime.now()
    sts0 = datetime.datetime.now()
    for i, res in enumerate(pool.imap_unordered(_run, queue), 1):
        if not res:
            failures += 1
        if failures > 100:
            print("ABORT due to more than 100 failures...")
            sys.exit(10)
        if i > 0 and i % 5000 == 0:
            delta00 = datetime.datetime.now() - sts00
            delta0 = datetime.datetime.now() - sts0
            speed00 = i / delta00.total_seconds()
            speed0 = 5000 / delta0.total_seconds()
            remaining = ((sz - i) / speed00) / 3600.
            sts0 = datetime.datetime.now()
            print((
                '%5.2fh Processed %6s/%6s [inst/tot %.2f/%.2f rps] '
                'remaining: %5.2fh'
                ) % (delta00.total_seconds() / 3600., i, sz,
                     speed0, speed00, remaining)
            )

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def test_GIL():
    """tests running of multiple queries in a threadpool"""
    vertices, triangles = triangle_soup(10000, (-5, 5))
    mesh0 = pyopcode.Model(vertices, triangles)
    vertices, triangles = triangle_soup(10000, (-5, 5))
    mesh1 = pyopcode.Model(vertices, triangles)

    col = pyopcode.Collision(mesh0, mesh1)

    identity = np.identity(4).astype(np.float32)

    def transform_generator():
        """generate affine rotation matrices"""
        np.random.seed(42)
        for i in range(100):
            r = np.random.normal(size=(3, 3))
            u, _, v = np.linalg.svd(r)
            r = u.dot(np.eye(*r.shape)).dot(v)
            a = identity.copy()
            a[:3, :3] = r
            yield a

    from multiprocessing.pool import ThreadPool
    pool = ThreadPool(processes=4)
    results = pool.imap_unordered(lambda affine: col.query(affine, identity), transform_generator())

    import time
    start = time.clock()
    for r in results:
        print(len(r))
    print(time.clock() - start)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def UrlMode(corpus, request_parallelism):
    """Finds Wayback Machine URLs and writes them to disk.

  Args:
    corpus: A corpus.
    request_parallelism: The number of concurrent requests.
  """

    for dataset in datasets:
        print "Finding Wayback Machine URLs for the %s set:" % dataset
        old_urls_filename = "%s/%s_urls.txt" % (corpus, dataset)
        new_urls_filename = "%s/wayback_%s_urls.txt" % (corpus, dataset)

        urls = ReadMultipleUrls(old_urls_filename)

        p = ThreadPool(request_parallelism)
        results = p.imap_unordered(WaybackUrl, urls)

        progress_bar = ProgressBar(len(urls))
        new_urls = []
        for result in results:
            if result:
                new_urls.append(result)

            progress_bar.Increment()

        WriteUrls(new_urls_filename, new_urls)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def check_migration(client, stage, framework_slug, draft_bucket, documents_bucket):
    do_check_draft_and_service = functools.partial(check_draft_and_service, client)
    pool = ThreadPool(10)
    drafts = pool.imap_unordered(
        do_check_draft_and_service,
        find_drafts(client, framework_slug))
    for draft in drafts:
        pass

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
 def run_jobs(self, f, jobs):
     if self.usecloud:
         jids = cloud.map(f, jobs, _env=self.cloud_env, _profile=True, _depends_on=self.preprocess_job)
         ires = cloud.iresult(jids)
     else:
         pool = ThreadPool(processes=cv2.getNumberOfCPUs())
         ires = pool.imap_unordered(f, jobs)
     return ires

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def process_threaded(img, filters, threadn = 8):
    accum = np.zeros_like(img)
    def f(kern):
        return cv2.filter2D(img, cv2.CV_8UC3, kern)
    pool = ThreadPool(processes=threadn)
    for fimg in pool.imap_unordered(f, filters):
        np.maximum(accum, fimg, accum)
    return accum

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap_unordered [as 别名]
def main(api_url, api_access_token):
    client = dmapiclient.DataAPIClient(api_url, api_access_token)
    pool = ThreadPool(10)
    count = 1
    for i in pool.imap_unordered(update(client), enumerate(client.find_services_iter())):
        count += i
        if count % 1000 == 0:
            print("** {}".format(count))