Python ThreadPool.imap方法代码示例

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
    def load(self, job_path, deployment_type, listener_port, elb_name_suffix, cf_params):
        env = cf_params['env']
        region = cf_params['region']
        cluster = cf_params['cluster'] if 'cluster' in cf_params else 'lambda'
        has_ecs_service = False if deployment_type != 'ecs_service' else True

        filenames = []
        for subdir, dirs, files in os.walk(job_path):
            for fn in files:
                filenames.append(os.path.join(subdir, fn))

        pool = ThreadPool(32)
        pool.imap(self.process_cf_file, ((cf_params, cluster, elb_name_suffix, env, fn, has_ecs_service, listener_port, region)
                            for idx, fn in enumerate(filenames)), chunksize=1)
        pool.close()
        pool.join()
        logging.info("Completed update of %s" % job_path)

        contains_failure = False
        while not self.q.empty():
            job_result = self.q.get()
            logging.info(job_result)
            if "Failed" in job_result:
                contains_failure = True
        if contains_failure:
            logging.error("One or more CF stacks failed!")
            sys.exit(1)
        else:
            logging.info("All CF stacks deployed successfully!")

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
 def test_synchronize(self):
     demo = LockDemo()
     pool = ThreadPool(2)
     pool.imap(demo.bar, range(2))
     sleep(0.04)
     assert_that(demo.call_count, equal_to(1))
     sleep(0.05)
     assert_that(demo.call_count, equal_to(2))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def find_suppliers_with_details(client, framework_slug):
    pool = ThreadPool(30)

    records = find_suppliers(client, framework_slug)
    records = pool.imap(partial(add_supplier_info, client), records)
    records = pool.imap(partial(add_framework_info, client, framework_slug), records)
    records = pool.imap(partial(add_submitted_draft_counts, client, framework_slug), records)

    return get_csv_rows(records, framework_slug)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
 def test_synchronize_with_same_param(self):
     demo = LockDemo()
     pool = ThreadPool(3)
     pool.imap(demo.foo2, (1, 1))
     pool.apply_async(demo.foo1)
     sleep(0.04)
     assert_that(demo.call_count, equal_to(1))
     sleep(0.05)
     assert_that(demo.call_count, equal_to(2))
     sleep(0.05)
     assert_that(demo.call_count, equal_to(3))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def find_all_labs(client):
    pool = ThreadPool(20)
    records = find_suppliers(client, FRAMEWORK_SLUG)
    records = pool.imap(add_framework_info(client, FRAMEWORK_SLUG), records)
    records = filter(lambda record: record['onFramework'], records)
    records = pool.imap(add_draft_services(client, FRAMEWORK_SLUG), records)
    services = itertools.chain.from_iterable(record['services'] for record in records)
    services = filter(
        lambda record: record['lot'] == 'user-research-studios' and record['status'] == 'submitted',
        services)

    return services

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def find_suppliers_with_details(client, content_loader, framework_slug, supplier_ids=None):
    pool = ThreadPool(30)

    content_loader.load_manifest(framework_slug, 'declaration', 'declaration')
    declaration_content = content_loader.get_manifest(framework_slug, 'declaration')

    records = find_suppliers(client, framework_slug, supplier_ids)
    records = pool.imap(add_supplier_info(client), records)
    records = pool.imap(add_framework_info(client, framework_slug), records)
    records = pool.imap(add_draft_counts(client, framework_slug), records)
    records = map(add_failed_questions(declaration_content), records)

    return records

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def find_services_by_lot(client, framework_slug, lot_slug):
    pool = ThreadPool(30)
    service_adder = add_draft_services(client, framework_slug,
                                       lot=lot_slug,
                                       status="submitted")

    records = find_suppliers(client, framework_slug)
    records = pool.imap(add_supplier_info(client), records)
    records = pool.imap(add_framework_info(client, framework_slug), records)
    records = pool.imap(service_adder, records)
    records = filter(lambda record: len(record["services"]) > 0, records)

    return records

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def run_tidy(sha="HEAD", is_rev_range=False):
    diff_cmdline = ["git", "diff" if is_rev_range else "show", sha]

    # Figure out which paths changed in the given diff.
    changed_paths = subprocess.check_output(diff_cmdline + ["--name-only", "--pretty=format:"]).splitlines()
    changed_paths = [p for p in changed_paths if p]

    # Produce a separate diff for each file and run clang-tidy-diff on it
    # in parallel.
    def tidy_on_path(path):
        patch_file = tempfile.NamedTemporaryFile()
        cmd = diff_cmdline + [
            "--src-prefix=%s/" % ROOT,
            "--dst-prefix=%s/" % ROOT,
            "--",
            path]
        subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT)
        cmdline = [CLANG_TIDY_DIFF,
                   "-clang-tidy-binary", CLANG_TIDY,
                   "-p0",
                   "--",
                   "-DCLANG_TIDY"] + compile_flags.get_flags()
        return subprocess.check_output(
            cmdline,
            stdin=file(patch_file.name),
            cwd=ROOT)
    pool = ThreadPool(multiprocessing.cpu_count())
    try:
        return "".join(pool.imap(tidy_on_path, changed_paths))
    except KeyboardInterrupt as ki:
        sys.exit(1)
    finally:
        pool.terminate()
        pool.join()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
    def get_used_properties(self, set_ids=None, article_ids=None, **filters):
        """
        Returns a sequency of property names in use in the specified set(s) (or setids)
        """
        if set_ids is not None:
            filters["sets"] = set_ids

        if article_ids is not None:
            filters["ids"] = article_ids

        all_properties = self.get_properties()
        flexible_properties = set(all_properties) - set(ALL_FIELDS)

        body = {"query": {"bool": {"must": [
            build_filter(**filters),
            {"exists": {"field": "fakeprop"}}
        ]}}}

        bodies = (copy.deepcopy(body) for _ in range(len(flexible_properties)))
        pool = ThreadPool()
        results = pool.imap(self._get_used_properties, zip(bodies, flexible_properties))

        try:
            for found, prop in zip(results, flexible_properties):
                if found:
                    yield prop
        finally:
            pool.close()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def get_reviews_from_imdb(movie_start_id, movie_end_id):
    """
    save movies reviews in storage.

    Args:
        movie_start_id: the start of the range of the movies.
        movie_end_id:   the end of the range of the movies.
    """

    thread_pool = ThreadPool()
    block_size = round((movie_end_id - movie_start_id) / MAX_THREADS)
    for results in thread_pool.imap(get_reviews_from_single_movie, xrange(movie_start_id, movie_end_id), block_size):
        if results is not None:
            storage.add_reviews(filter(lambda result: result[0] != "", results))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def run_attack(attack, pipe):
    """
    Given an attack function to run, and a Connection object through which to
    communicate, receive a network and set of fractions to remove, and simulate
    attacks on that network for each fraction of nodes. Puts S1/N back through
    the pipe for each fraction.
    """
    network = pipe.recv()
    fractions = pipe.recv()

    N = len(network)
    nodes_to_remove = [int(round(f * N)) for f in fractions]

    thread_pool = ThreadPool(5)
    results = thread_pool.imap(lambda x: attack(network, x), nodes_to_remove)

    for res in results:
        pipe.send(gc_size(res, N))
    pipe.close()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def evaluate_retrieval(model, entity_model, eval_items):
    pool = ThreadPool()

    def entity_rank((entity, word_idxs, doc_len)):
        if word_idxs:
            entity_id = entity_model.entities[entity]
            scores = entity_model.score(model,
                                        entity_model.vectors,
                                        word_idxs)
            rank = np.sum(scores >= scores[entity_id])
        else:
            rank = entity_vecs.shape[0]

        return int(np.log2(doc_len)), np.log2(rank)

    ranks = defaultdict(list)
    for size, rank in pool.imap(entity_rank, eval_items):
        ranks[size].append(rank)

    sorted_ranks = sorted(ranks.iteritems())
    logging.info('%s overall score: %.3f by size: %s',
                 type(entity_model).__name__,
                 np.mean(np.hstack(ranks.values())),
                 ' '.join('%d: %.3f' % (k, np.mean(v)) for k, v in sorted_ranks))

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
def parallel(func, source, chunksize=0, numcpus=multiprocessing.cpu_count()):
    if chunksize:
        source = chunk(source, chunksize)
    p = ThreadPool(numcpus)
    for i in p.imap(func, source):
        yield i

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]

#.........这里部分代码省略.........
                    alias = (title, frag)
                else:
                    alias = title
                aliases.add(alias)

                page = redirect_target.page
                print('%s ==> %s' % (
                    title,
                    page.name + (('#'+frag) if frag else '')))

                if redirect_count >= 10:
                    print('Too many redirect levels: %r' % aliases)
                    break

                title = page.name

            if page.redirect:
                print('Failed to resolve redirect %s', title)
                inc_count('failed_redirect')
                return

            doc = db.get(title)
            if doc:
                current_aliases = set()
                for alias in doc.get('aliases', ()):
                    if isinstance(alias, list):
                        alias = tuple(alias)
                    current_aliases.add(alias)
                if not aliases.issubset(current_aliases):
                    merged_aliases = aliases|current_aliases
                    #remove aliases without fragment if one with fragment is present
                    #this is mostly to cleanup aliases in old scrapes
                    to_remove = set()
                    for alias in merged_aliases:
                        if isinstance(alias, tuple):
                            to_remove.add(alias[0])
                    merged_aliases = merged_aliases - to_remove
                    doc['aliases'] = list(merged_aliases)
                    db[title] = doc
                revid = doc.get('parse', {}).get('revid')
                if page.revision == revid:
                    print('%s is up to date (rev. %s), skipping' %
                          (title, revid))
                    inc_count('up_to_date')
                    return
                else:
                    inc_count('updated')
                    print('New rev. %s is available for %s (have rev. %s)' %
                          (page.revision, title, revid))

            parse = site.api('parse', page=title)
        except KeyboardInterrupt as ki:
            print ('Caught KeyboardInterrupt', ki)
            thread.interrupt_main()
        except couchdb.ResourceConflict:
            print('Update conflict, skipping: %s' % title)
            return
        except Exception:
            print('Failed to process %s:' % title)
            traceback.print_exc()
            inc_count('error')
            return
        if doc:
            doc.update(parse)
        else:
            inc_count('new')
            doc = parse
            if aliases:
                doc['aliases'] = list(aliases)
        try:
            db[title] = doc
        except couchdb.ResourceConflict:
            print('Update conflict, skipping: %s' % title)
            return

    import pylru
    seen = pylru.lrucache(10000)

    def ipages(pages):
        for index, page in enumerate(pages):
            title = page.name
            print('%7s %s' % (index, title))
            if title in seen:
                print('Already saw %s, skipping' % (title,))
                continue
            seen[title] = True
            update_session(title)
            yield page


    with flock(os.path.join(tempfile.gettempdir(),
                            hashlib.sha1(host).hexdigest())):
        if args.speed:
            pool = ThreadPool(processes=args.speed*2)
            for _result in pool.imap(process, ipages(pages)):
                pass

        else:
            for page in ipages(pages):
                process(page)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import imap [as 别名]
class Pipeline(object):
    """
    A pipeline of multiple processors to process S3 objects.
    """

    def __init__(self, access_key=None, secret_key=None, dry_run=False, threads=None):
        self._pipeline = []
        self.access_key = access_key
        self.secret_key = secret_key
        self.dry_run = dry_run
        self.threads = threads
        self.pool = ThreadPool(threads)
        self.thread_local_buckets = threading.local()

    def append(self, analyser, pattern, ignore_case=True):
        if ignore_case:
            pattern = re.compile(pattern, flags=re.IGNORECASE)
        else:
            pattern = re.compile(pattern)
        self._pipeline.append((pattern, analyser))

    def analyse(self, pattern, ignore_case=True):
        def decorator(func):
            self.append(DecoratorAnalyser(func.__name__, func), pattern, ignore_case)
            return func

        return decorator

    def connect_s3(self):
        if self.access_key is not None and self.secret_key is not None:
            return boto.connect_s3(aws_access_key_id=self.access_key,
                                   aws_secret_access_key=self.secret_key)
        else:
            return boto.connect_s3()

    def get_bucket(self, name):
        if getattr(self.thread_local_buckets, name, None) is None:
            logging.debug('Create new connection to S3 from thread %s', threading.currentThread())
            conn = self.connect_s3()
            bucket = conn.get_bucket(name)
            setattr(self.thread_local_buckets, name, bucket)
        return getattr(self.thread_local_buckets, name)

    def run(self, bucket, prefix='', show_progress=True):
        self.pre_run()

        bucket = self.get_bucket(bucket)
        keys = bucket.list(prefix)
        chunk_size = self.threads if self.threads is not None else cpu_count()
        it = self.pool.imap(self.analyse_key, keys, chunksize=chunk_size)
        if show_progress:
            list(progress.dots(it, label='Analysing bucket "%s"' % bucket.name))
        else:
            list(it)

        self.post_run()

    def pre_run(self):
        for _, analyser in self._pipeline:
            analyser.start()

    def post_run(self):
        for _, analyser in self._pipeline:
            analyser.finish()

    def analyse_key(self, key):
        bucket = self.get_bucket(key.bucket.name)
        for pattern, analyser in self._pipeline:
            if pattern.match(key.key):
                # update key metadata since last analyser might already modified it
                key = bucket.get_key(key.key)
                analyser.analyse(key, dry_run=self.dry_run)