Python pool.imap_unordered函数代码示例

 def _push(self, src, dst):
     """
     Push src to dst on the remote.
     """
     force = False
     if src.startswith('+'):
         src = src[1:]
         force = True
     present = [self._refs[name][1] for name in self._refs]
     present.extend(self._pushed.values())
     # before updating the ref, write all objects that are referenced
     objects = git_list_objects(src, present)
     try:
         # upload objects in parallel
         pool = multiprocessing.pool.ThreadPool(processes=self._processes)
         res = pool.imap_unordered(Binder(self, '_put_object'), objects)
         # show progress
         total = len(objects)
         self._trace('', level=Level.INFO, exact=True)
         for done, _ in enumerate(res, 1):
             pct = float(done) / total
             message = '\rWriting objects: {:4.0%} ({}/{})'.format(pct, done, total)
             if done == total:
                 message = '%s, done.\n' % message
             self._trace(message, level=Level.INFO, exact=True)
     except Exception:
         self._fatal('exception while writing objects')
     sha = git_ref_value(src)
     error = self._write_ref(sha, dst, force)
     if error is None:
         self._write('ok %s' % dst)
         self._pushed[dst] = sha
     else:
         self._write('error %s %s' % (dst, error))

def download_junit(db, threads, client_class):
    """Download junit results for builds without them."""
    builds_to_grab = db.get_builds_missing_junit()
    pool = None
    if threads > 1:
        pool = multiprocessing.pool.ThreadPool(
            threads, mp_init_worker, ('', {}, client_class, False))
        test_iterator = pool.imap_unordered(
            get_junits, builds_to_grab)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = client_class('', {})
        test_iterator = (
            get_junits(build_path) for build_path in builds_to_grab)
    for n, (build_id, build_path, junits) in enumerate(test_iterator, 1):
        print('%d/%d' % (n, len(builds_to_grab)),
              build_path, len(junits), len(''.join(junits.values())))
        junits = {k: remove_system_out(v) for k, v in junits.iteritems()}

        db.insert_build_junits(build_id, junits)
        if n % 100 == 0:
            db.commit()
    db.commit()
    if pool:
        pool.close()
        pool.join()

def s3_iter_bucket(bucket, prefix="", accept_key=lambda key: True, key_limit=None, workers=16):
    """
    Iterate and download all S3 files under `bucket/prefix`, yielding out
    `(key, key content)` 2-tuples (generator).

    `accept_key` is a function that accepts a key name (unicode string) and
    returns True/False, signalling whether the given key should be downloaded out or
    not (default: accept all keys).

    If `key_limit` is given, stop after yielding out that many results.

    The keys are processed in parallel, using `workers` processes (default: 16),
    to speed up downloads greatly. If multiprocessing is not available, thus
    NO_MULTIPROCESSING is True, this parameter will be ignored.

    Example::

      >>> mybucket = boto.connect_s3().get_bucket('mybucket')

      >>> # get all JSON files under "mybucket/foo/"
      >>> for key, content in s3_iter_bucket(mybucket, prefix='foo/', accept_key=lambda key: key.endswith('.json')):
      ...     print key, len(content)

      >>> # limit to 10k files, using 32 parallel workers (default is 16)
      >>> for key, content in s3_iter_bucket(mybucket, key_limit=10000, workers=32):
      ...     print key, len(content)

    """
    total_size, key_no = 0, -1
    keys = (key for key in bucket.list(prefix=prefix) if accept_key(key.name))

    if NO_MULTIPROCESSING:
        logger.info("iterating over keys from %s without multiprocessing" % bucket)
        iterator = imap(s3_iter_bucket_process_key, keys)
    else:
        logger.info("iterating over keys from %s with %i workers" % (bucket, workers))
        pool = multiprocessing.pool.Pool(processes=workers)
        iterator = pool.imap_unordered(s3_iter_bucket_process_key, keys)

    for key_no, (key, content) in enumerate(iterator):
        if key_no % 1000 == 0:
            logger.info(
                "yielding key #%i: %s, size %i (total %.1fMB)" % (key_no, key, len(content), total_size / 1024.0 ** 2)
            )

        yield key, content
        key.close()
        total_size += len(content)

        if key_limit is not None and key_no + 1 >= key_limit:
            # we were asked to output only a limited number of keys => we're done
            break

    if not NO_MULTIPROCESSING:
        pool.terminate()

    logger.info("processed %i keys, total size %i" % (key_no + 1, total_size))

def iter_bucket(bucket_name, prefix='', accept_key=lambda key: True,
                key_limit=None, workers=16, retries=3):
    """
    Iterate and download all S3 files under `bucket/prefix`, yielding out
    `(key, key content)` 2-tuples (generator).

    `accept_key` is a function that accepts a key name (unicode string) and
    returns True/False, signalling whether the given key should be downloaded out or
    not (default: accept all keys).

    If `key_limit` is given, stop after yielding out that many results.

    The keys are processed in parallel, using `workers` processes (default: 16),
    to speed up downloads greatly. If multiprocessing is not available, thus
    _MULTIPROCESSING is False, this parameter will be ignored.

    Example::

      >>> # get all JSON files under "mybucket/foo/"
      >>> for key, content in iter_bucket(bucket_name, prefix='foo/', accept_key=lambda key: key.endswith('.json')):
      ...     print key, len(content)

      >>> # limit to 10k files, using 32 parallel workers (default is 16)
      >>> for key, content in iter_bucket(bucket_name, key_limit=10000, workers=32):
      ...     print key, len(content)
    """
    #
    # If people insist on giving us bucket instances, silently extract the name
    # before moving on.  Works for boto3 as well as boto.
    #
    try:
        bucket_name = bucket_name.name
    except AttributeError:
        pass

    total_size, key_no = 0, -1
    key_iterator = _list_bucket(bucket_name, prefix=prefix, accept_key=accept_key)
    download_key = functools.partial(_download_key, bucket_name=bucket_name, retries=retries)

    with _create_process_pool(processes=workers) as pool:
        result_iterator = pool.imap_unordered(download_key, key_iterator)
        for key_no, (key, content) in enumerate(result_iterator):
            if True or key_no % 1000 == 0:
                logger.info(
                    "yielding key #%i: %s, size %i (total %.1fMB)",
                    key_no, key, len(content), total_size / 1024.0 ** 2
                )
            yield key, content
            total_size += len(content)

            if key_limit is not None and key_no + 1 >= key_limit:
                # we were asked to output only a limited number of keys => we're done
                break
    logger.info("processed %i keys, total size %i" % (key_no + 1, total_size))

    def computeSamples(self, work, report_interval=100):
        '''compute samples according to work.

        returns a list of results.
        '''
        n = len(work)

        E.debug('sampling will work on %i items' % n)

        results = []

        if self.num_threads == 0:
            for i, w in enumerate(work):
                r = computeSample(
                    (w, self.samples_outfile, self.outfile_sample_metrics,
                     None))
                if i % report_interval == 0:
                    E.info("%i/%i done (%5.2f)" % (i, n, 100.0 * i / n))
                results.append(r)
        else:
            E.info("generating processpool with %i threads for %i items" %
                   (self.num_threads, len(work)))

            manager = multiprocessing.Manager()

            lock = manager.Lock()

            pool = multiprocessing.Pool(self.num_threads)

            # use file names - not files when multiprocessing
            samples_outfile, metrics_outfile = None, None
            if self.samples_outfile:
                samples_outfile = self.samples_outfile.name
                self.samples_outfile.flush()
            if self.outfile_sample_metrics:
                metrics_outfile = self.outfile_sample_metrics.name
                self.outfile_sample_metrics.flush()

            ww = [(w, samples_outfile, metrics_outfile, lock) for w in work]

            for i, r in enumerate(pool.imap_unordered(computeSample, ww)):
                if i % report_interval == 0:
                    E.info("%i/%i done (%5.2f)" % (i, n, 100.0 * i / n))
                results.append(r)

            pool.close()
            pool.join()

        return results

def get_builds(db, jobs_dir, metadata, threads, client_class):
    """
    Adds information about tests to a dictionary.

    Args:
        jobs_dir: the GCS path containing jobs.
        metadata: a dict of metadata about the jobs_dir.
        threads: how many threads to use to download build information.
        client_class: a constructor for a GCSClient (or a subclass).
    """
    gcs = client_class(jobs_dir, metadata)

    print('Loading builds from %s' % jobs_dir)
    sys.stdout.flush()

    builds_have = db.get_existing_builds(jobs_dir)
    print('already have %d builds' % len(builds_have))
    sys.stdout.flush()

    jobs_and_builds = gcs.get_builds(builds_have)
    pool = None
    if threads > 1:
        pool = multiprocessing.Pool(threads, mp_init_worker,
                                    (jobs_dir, metadata, client_class))
        builds_iterator = pool.imap_unordered(
            get_started_finished, jobs_and_builds)
    else:
        global WORKER_CLIENT  # pylint: disable=global-statement
        WORKER_CLIENT = gcs
        builds_iterator = (
            get_started_finished(job_build) for job_build in jobs_and_builds)

    try:
        for n, (build_dir, started, finished) in enumerate(builds_iterator):
            print(build_dir)
            if started or finished:
                db.insert_build(build_dir, started, finished)
            if n % 200 == 0:
                db.commit()
    except KeyboardInterrupt:
        if pool:
            pool.terminate()
        raise
    else:
        if pool:
            pool.close()
            pool.join()
    db.commit()

def get_started_finished(gcs_client, db, todo):
    """Download started/finished.json from build dirs in todo."""
    acks = []
    build_dirs = []
    pool = multiprocessing.pool.ThreadPool(16)
    try:
        for ack_id, (build_dir, started, finished) in pool.imap_unordered(
                lambda (ack_id, job, build): (ack_id, gcs_client.get_started_finished(job, build)),
                todo):
            if finished:
                if not db.insert_build(build_dir, started, finished):
                    print('already present??')
                start = time.localtime(started.get('timestamp', 0) if started else 0)
                print(build_dir, bool(started), bool(finished),
                      time.strftime('%F %T %Z', start),
                      finished and finished.get('result'))
                build_dirs.append(build_dir)
                acks.append(ack_id)
            else:
                print('finished.json missing?', build_dir, started, finished)
    finally:
        pool.close()
    db.commit()
    return acks, build_dirs

def run_iptestall(options):
    """Run the entire IPython test suite by calling nose and trial.

    This function constructs :class:`IPTester` instances for all IPython
    modules and package and then runs each of them.  This causes the modules
    and packages of IPython to be tested each in their own subprocess using
    nose.

    Parameters
    ----------

    All parameters are passed as attributes of the options object.

    testgroups : list of str
      Run only these sections of the test suite. If empty, run all the available
      sections.

    fast : int or None
      Run the test suite in parallel, using n simultaneous processes. If None
      is passed, one process is used per CPU core. Default 1 (i.e. sequential)

    inc_slow : bool
      Include slow tests, like IPython.parallel. By default, these tests aren't
      run.

    xunit : bool
      Produce Xunit XML output. This is written to multiple foo.xunit.xml files.

    coverage : bool or str
      Measure code coverage from tests. True will store the raw coverage data,
      or pass 'html' or 'xml' to get reports.

    extra_args : list
      Extra arguments to pass to the test subprocesses, e.g. '-v'
    """
    if options.fast != 1:
        # If running in parallel, capture output so it doesn't get interleaved
        TestController.buffer_output = True

    to_run, not_run = prepare_controllers(options)

    def justify(ltext, rtext, width=70, fill='-'):
        ltext += ' '
        rtext = (' ' + rtext).rjust(width - len(ltext), fill)
        return ltext + rtext

    # Run all test runners, tracking execution time
    failed = []
    t_start = time.time()

    print()
    if options.fast == 1:
        # This actually means sequential, i.e. with 1 job
        for controller in to_run:
            print('IPython test group:', controller.section)
            sys.stdout.flush()  # Show in correct order when output is piped
            controller, res = do_run(controller)
            if res:
                failed.append(controller)
                if res == -signal.SIGINT:
                    print("Interrupted")
                    break
            print()

    else:
        # Run tests concurrently
        try:
            pool = multiprocessing.pool.ThreadPool(options.fast)
            for (controller, res) in pool.imap_unordered(do_run, to_run):
                res_string = 'OK' if res == 0 else 'FAILED'
                print(justify('IPython test group: ' + controller.section, res_string))
                if res:
                    print(bytes_to_str(controller.stdout))
                    failed.append(controller)
                    if res == -signal.SIGINT:
                        print("Interrupted")
                        break
        except KeyboardInterrupt:
            return

    for controller in not_run:
        print(justify('IPython test group: ' + controller.section, 'NOT RUN'))

    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(to_run)
    nfail = len(failed)
    # summarize results
    print('_'*70)
    print('Test suite completed for system with the following information:')
    print(report())
    took = "Took %.3fs." % t_tests
    print('Status: ', end='')
    if not failed:
        print('OK (%d test groups).' % nrunners, took)
    else:
        # If anything went wrong, point out what command to rerun manually to
        # see the actual errors and individual summary
        failed_sections = [c.section for c in failed]
        print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
#.........这里部分代码省略.........

def run_jstestall(options):
    """Run the entire Javascript test suite.
    
    This function constructs TestControllers and runs them in subprocesses.

    Parameters
    ----------

    All parameters are passed as attributes of the options object.

    testgroups : list of str
      Run only these sections of the test suite. If empty, run all the available
      sections.

    fast : int or None
      Run the test suite in parallel, using n simultaneous processes. If None
      is passed, one process is used per CPU core. Default 1 (i.e. sequential)

    inc_slow : bool
      Include slow tests. By default, these tests aren't run.

    slimerjs : bool
      Use slimerjs if it's installed instead of phantomjs for casperjs tests.

    url : unicode
      Address:port to use when running the JS tests.

    xunit : bool
      Produce Xunit XML output. This is written to multiple foo.xunit.xml files.

    extra_args : list
      Extra arguments to pass to the test subprocesses, e.g. '-v'
    """
    to_run, not_run = prepare_controllers(options)

    def justify(ltext, rtext, width=70, fill='-'):
        ltext += ' '
        rtext = (' ' + rtext).rjust(width - len(ltext), fill)
        return ltext + rtext

    # Run all test runners, tracking execution time
    failed = []
    t_start = time.time()

    print()
    if options.fast == 1:
        # This actually means sequential, i.e. with 1 job
        for controller in to_run:
            print('Test group:', controller.section)
            sys.stdout.flush()  # Show in correct order when output is piped
            controller, res = do_run(controller, buffer_output=False)
            if res:
                failed.append(controller)
                if res == -signal.SIGINT:
                    print("Interrupted")
                    break
            print()

    else:
        # Run tests concurrently
        try:
            pool = multiprocessing.pool.ThreadPool(options.fast)
            for (controller, res) in pool.imap_unordered(do_run, to_run):
                res_string = 'OK' if res == 0 else 'FAILED'
                print(justify('Test group: ' + controller.section, res_string))
                if res:
                    controller.print_extra_info()
                    print(bytes_to_str(controller.stdout))
                    failed.append(controller)
                    if res == -signal.SIGINT:
                        print("Interrupted")
                        break
        except KeyboardInterrupt:
            return

    for controller in not_run:
        print(justify('Test group: ' + controller.section, 'NOT RUN'))

    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(to_run)
    nfail = len(failed)
    # summarize results
    print('_'*70)
    print('Test suite completed for system with the following information:')
    print(report())
    took = "Took %.3fs." % t_tests
    print('Status: ', end='')
    if not failed:
        print('OK (%d test groups).' % nrunners, took)
    else:
        # If anything went wrong, point out what command to rerun manually to
        # see the actual errors and individual summary
        failed_sections = [c.section for c in failed]
        print('ERROR - {} out of {} test groups failed ({}).'.format(nfail,
                                  nrunners, ', '.join(failed_sections)), took)
        print()
        print('You may wish to rerun these, with:')
        print('  python -m notebook.jstest', *failed_sections)
        print()
#.........这里部分代码省略.........

def generate_apply_order(all_patches, skip_checks=False):
    """Resolve dependencies, and afterwards check if everything applies properly."""
    depends     = sorted([i for i, patch in all_patches.iteritems() if not patch.disabled])
    resolved    = resolve_dependencies(all_patches, depends=depends)
    max_patches = max(resolved) + 1

    if skip_checks:
        return resolved

    # Generate timestamps based on dependencies, still required for binary patches
    # Find out which files are modified by multiple patches
    modified_files = {}
    for i, patch in [(i, all_patches[i]) for i in resolved]:
        patch.verify_time = [0]*max_patches
        patch.verify_time[i] += 1
        for j in patch.depends:
            patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time)

        for f in patch.modified_files:
            if f not in modified_files:
                modified_files[f] = []
            modified_files[f].append(i)

    # Check dependencies
    dependency_cache = _load_dict(config.path_cache)
    pool = multiprocessing.pool.ThreadPool(processes=4)
    try:
        for filename, indices in modified_files.iteritems():

            # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case
            if contains_binary_patch(all_patches, indices, filename):
                if not causal_time_relation_any(all_patches, indices):
                    raise PatchUpdaterError("Because of binary patch modifying file %s the following patches need explicit dependencies: %s" %
                                            (filename, ", ".join([all_patches[i].name for i in indices])))
                continue

            original_content = get_wine_file(filename)
            original_hash    = _sha256(original_content)
            selected_patches = select_patches(all_patches, indices, filename)

            # Generate a unique id based on the original content, the selected patches
            # and the dependency information. Since this information only has to be compared
            # we can throw it into a single hash.
            m = hashlib.sha256()
            m.update(original_hash)
            for i in indices:
                m.update("P%s" % selected_patches[i][0])
                for j in indices:
                    if causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time):
                        m.update("D%s" % selected_patches[j][0])
            unique_hash = m.digest()

            # Skip checks if it matches the information from the cache
            # For backwards compatibility, convert string entries to list
            if dependency_cache.has_key(filename):
                if not isinstance(dependency_cache[filename], list):
                    dependency_cache[filename] = [dependency_cache[filename]]
                if unique_hash in dependency_cache[filename]:
                    dependency_cache[filename].append(unique_hash)
                    dependency_cache[filename].remove(unique_hash)
                    continue

            # Show a progress bar while applying the patches - this task might take some time
            chunk_size = 20
            with progressbar.ProgressBar(desc=filename, total=2 ** len(indices) / chunk_size) as progress:

                def test_apply(current):
                    set_apply = [(i, all_patches[i]) for i in current]
                    set_skip  = [(i, all_patches[i]) for i in indices if i not in current]

                    # Check if there is any patch2 which depends directly or indirectly on patch1.
                    # If this is the case we found an impossible situation, we can be skipped in this test.
                    for i, patch1 in set_apply:
                        for j, patch2 in set_skip:
                            if causal_time_smaller(patch2.verify_time, patch1.verify_time):
                                return True # we can skip this test

                    try:
                        original = original_content
                        for i, _ in set_apply:
                            original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0)
                    except patchutils.PatchApplyError:
                        return False

                    return True # everything is fine

                def test_apply_seq(current_list):
                    for current in current_list:
                        if not test_apply(current):
                            return current
                    return None

                iterables = []
                for i in xrange(0, len(indices) + 1):
                    iterables.append(itertools.combinations(indices, i))
                it = _split_seq(itertools.chain(*iterables), chunk_size)
                for k, failed in enumerate(pool.imap_unordered(test_apply_seq, it)):
                    if failed is not None:
                        progress.finish("<failed to apply>")
                        raise PatchUpdaterError("Changes to file %s don't apply: %s" %
#.........这里部分代码省略.........

def generate_script(all_patches):
    """Resolve dependencies, and afterwards check if everything applies properly."""
    depends     = sorted([i for i, patch in all_patches.iteritems() if not patch.disabled])
    resolved    = resolve_dependencies(all_patches, depends=depends)
    max_patches = max(resolved) + 1

    # Generate timestamps based on dependencies, still required for binary patches
    # Find out which files are modified by multiple patches
    modified_files = {}
    for i, patch in [(i, all_patches[i]) for i in resolved]:
        patch.verify_time = [0]*max_patches
        patch.verify_time[i] += 1
        for j in patch.depends:
            patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time)

        for f in patch.modified_files:
            if f not in modified_files:
                modified_files[f] = []
            modified_files[f].append(i)

    # Check dependencies
    dependency_cache = _load_dict(config.path_cache)
    pool = multiprocessing.pool.ThreadPool(processes=4)
    try:
        for filename, indices in modified_files.iteritems():

            # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case
            if contains_binary_patch(all_patches, indices, filename):
                if not causal_time_relation_any(all_patches, indices):
                    raise PatchUpdaterError("Because of binary patch modifying file %s the following patches need explicit dependencies: %s" %
                                            (filename, ", ".join([all_patches[i].name for i in indices])))
                continue

            original_content = get_wine_file(filename)
            original_hash    = _sha256(original_content)
            selected_patches = select_patches(all_patches, indices, filename)

            # Generate a unique id based on the original content, the selected patches
            # and the dependency information. Since this information only has to be compared
            # we can throw it into a single hash.
            m = hashlib.sha256()
            m.update(original_hash)
            for i in indices:
                m.update("P%s" % selected_patches[i][0])
                for j in indices:
                    if causal_time_smaller(all_patches[j].verify_time, all_patches[i].verify_time):
                        m.update("D%s" % selected_patches[j][0])
            unique_hash = m.digest()

            # Skip checks if it matches the information from the cache
            try:
                if dependency_cache[filename] == unique_hash:
                    continue
            except KeyError:
                pass

            # Show a progress bar while applying the patches - this task might take some time
            chunk_size = 20
            with progressbar.ProgressBar(desc=filename, total=2 ** len(indices) / chunk_size) as progress:

                def test_apply(current):
                    set_apply = [(i, all_patches[i]) for i in current]
                    set_skip  = [(i, all_patches[i]) for i in indices if i not in current]

                    # Check if there is any patch2 which depends directly or indirectly on patch1.
                    # If this is the case we found an impossible situation, we can be skipped in this test.
                    for i, patch1 in set_apply:
                        for j, patch2 in set_skip:
                            if causal_time_smaller(patch2.verify_time, patch1.verify_time):
                                return None # we can skip this test

                    try:
                        original = original_content
                        for i, _ in set_apply:
                            original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0)
                    except patchutils.PatchApplyError:
                        return current

                    return None # everything is fine

                def test_apply_seq(current_list):
                    for current in current_list:
                        failed = test_apply(current)
                        if failed is not None:
                            return failed
                    return None

                iterables = []
                for i in xrange(0, len(indices) + 1):
                    iterables.append(itertools.combinations(indices, i))
                it = _split_seq(itertools.chain(*iterables), chunk_size)
                for k, failed in enumerate(pool.imap_unordered(test_apply_seq, it)):
                    if failed is not None:
                        progress.finish("<failed to apply>")
                        raise PatchUpdaterError("Changes to file %s don't apply: %s" %
                                                (filename, ", ".join([all_patches[i].name for i in failed])))
                    progress.update(k)

            # Update the dependency cache
            dependency_cache[filename] = unique_hash
#.........这里部分代码省略.........

def run_iptestall(options):
    """Run the entire IPython test suite by calling nose and trial.

    This function constructs :class:`IPTester` instances for all IPython
    modules and package and then runs each of them.  This causes the modules
    and packages of IPython to be tested each in their own subprocess using
    nose.
    
    Parameters
    ----------

    All parameters are passed as attributes of the options object.

    testgroups : list of str
      Run only these sections of the test suite. If empty, run all the available
      sections.

    fast : int or None
      Run the test suite in parallel, using n simultaneous processes. If None
      is passed, one process is used per CPU core. Default 1 (i.e. sequential)

    inc_slow : bool
      Include slow tests, like IPython.parallel. By default, these tests aren't
      run.

    xunit : bool
      Produce Xunit XML output. This is written to multiple foo.xunit.xml files.

    coverage : bool or str
      Measure code coverage from tests. True will store the raw coverage data,
      or pass 'html' or 'xml' to get reports.
    """
    if options.fast != 1:
        # If running in parallel, capture output so it doesn't get interleaved
        TestController.buffer_output = True

    if options.testgroups:
        to_run = [PyTestController(name) for name in options.testgroups]
        not_run = []
    else:
        to_run, not_run = prepare_py_test_controllers(inc_slow=options.all)

    configure_controllers(to_run, xunit=options.xunit, coverage=options.coverage)

    def justify(ltext, rtext, width=70, fill="-"):
        ltext += " "
        rtext = (" " + rtext).rjust(width - len(ltext), fill)
        return ltext + rtext

    # Run all test runners, tracking execution time
    failed = []
    t_start = time.time()

    print()
    if options.fast == 1:
        # This actually means sequential, i.e. with 1 job
        for controller in to_run:
            print("IPython test group:", controller.section)
            controller, res = do_run(controller)
            if res:
                failed.append(controller)
                if res == -signal.SIGINT:
                    print("Interrupted")
                    break
            print()

    else:
        # Run tests concurrently
        try:
            pool = multiprocessing.pool.ThreadPool(options.fast)
            for (controller, res) in pool.imap_unordered(do_run, to_run):
                res_string = "OK" if res == 0 else "FAILED"
                print(justify("IPython test group: " + controller.section, res_string))
                if res:
                    print(bytes_to_str(controller.stdout))
                    failed.append(controller)
                    if res == -signal.SIGINT:
                        print("Interrupted")
                        break
        except KeyboardInterrupt:
            return

    for controller in not_run:
        print(justify("IPython test group: " + controller.section, "NOT RUN"))

    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(to_run)
    nfail = len(failed)
    # summarize results
    print("_" * 70)
    print("Test suite completed for system with the following information:")
    print(report())
    print("Ran %s test groups in %.3fs" % (nrunners, t_tests))
    print()
    print("Status: ", end="")
    if not failed:
        print("OK")
    else:
        # If anything went wrong, point out what command to rerun manually to
#.........这里部分代码省略.........

def generate_script(all_patches):
    """Resolve dependencies, and afterwards check if everything applies properly."""
    depends     = sorted([i for i, patch in all_patches.iteritems() if not patch.disabled])
    resolved    = resolve_dependencies(all_patches, depends=depends)
    max_patches = max(resolved) + 1

    # Generate timestamps based on dependencies, still required for binary patches
    # Find out which files are modified by multiple patches
    modified_files = {}
    for i, patch in [(i, all_patches[i]) for i in resolved]:
        patch.verify_time = [0]*max_patches
        patch.verify_time[i] += 1
        for j in patch.depends:
            patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time)

        for f in patch.modified_files:
            if f not in modified_files:
                modified_files[f] = []
            modified_files[f].append(i)

    # Check dependencies
    pool = multiprocessing.pool.ThreadPool(processes=4)
    try:

        # Checking all dependencies takes a very long time, so to improve development speed,
        # run a first quick check with all patches enabled.
        with progressbar.ProgressBar(desc="pre-check ...", total=len(modified_files)) as progress:
            for k, (filename, indices) in enumerate(modified_files.iteritems()):

                # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case
                if contains_binary_patch(all_patches, indices, filename):
                    if not causal_time_relation_any(all_patches, indices):
                        raise PatchUpdaterError("Because of binary patch modifying file %s the following patches need explicit dependencies: %s" %
                                                (filename, ", ".join([all_patches[i].name for i in indices])))
                    continue

                original         = get_wine_file(filename)
                selected_patches = select_patches(all_patches, indices, filename)
                set_apply        = [(i, all_patches[i]) for i in indices]

                try:
                    for i, patch in set_apply:
                        original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0)
                except patchutils.PatchApplyError:
                    progress.finish("<failed to apply>")
                    raise PatchUpdaterError("Changes to file %s don't apply: %s" %
                                            (filename, ", ".join([all_patches[i].name for i in indices])))
                progress.update(k)

        # More detailed checks, required to make sure that dependencies are set correctly
        for filename, indices in modified_files.iteritems():

            if contains_binary_patch(all_patches, indices, filename):
                continue

            original_content = get_wine_file(filename)
            selected_patches = select_patches(all_patches, indices, filename)

            # Show a progress bar while applying the patches - this task might take some time
            chunk_size = 20
            with progressbar.ProgressBar(desc=filename, total=2 ** len(indices) / chunk_size) as progress:

                def test_apply(bitstring):
                    set_apply = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if u]
                    set_skip  = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if not u]

                    # Check if there is any patch2 which depends directly or indirectly on patch1.
                    # If this is the case we found an impossible situation, we can be skipped in this test.
                    for i, patch1 in set_apply:
                        for j, patch2 in set_skip:
                            if causal_time_smaller(patch2.verify_time, patch1.verify_time):
                                return True # we can skip this test

                    try:
                        original = original_content
                        for i, patch in set_apply:
                            original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0)
                    except patchutils.PatchApplyError:
                        return False

                    return True # everything is fine

                def test_apply_seq(bitstrings):
                    for bitstring in bitstrings:
                        if not test_apply(bitstring):
                            return False
                    return True

                it = _split_seq(itertools.product([0,1], repeat=len(indices)), chunk_size)
                for k, res in enumerate(pool.imap_unordered(test_apply_seq, it)):
                    if not res:
                        progress.finish("<failed to apply>")
                        raise PatchUpdaterError("Changes to file %s don't apply: %s" %
                                                (filename, ", ".join([all_patches[i].name for i in indices])))
                    progress.update(k)
    finally:
        pool.close()

    # Generate code for helper functions
    lines = []
#.........这里部分代码省略.........

def iter_bucket(bucket_name, prefix='', accept_key=None,
                key_limit=None, workers=16, retries=3):
    """
    Iterate and download all S3 objects under `s3://bucket_name/prefix`.

    Parameters
    ----------
    bucket_name: str
        The name of the bucket.
    prefix: str, optional
        Limits the iteration to keys starting wit the prefix.
    accept_key: callable, optional
        This is a function that accepts a key name (unicode string) and
        returns True/False, signalling whether the given key should be downloaded.
        The default behavior is to accept all keys.
    key_limit: int, optional
        If specified, the iterator will stop after yielding this many results.
    workers: int, optional
        The number of subprocesses to use.
    retries: int, optional
        The number of time to retry a failed download.

    Yields
    ------
    str
        The full key name (does not include the bucket name).
    bytes
        The full contents of the key.

    Notes
    -----
    The keys are processed in parallel, using `workers` processes (default: 16),
    to speed up downloads greatly. If multiprocessing is not available, thus
    _MULTIPROCESSING is False, this parameter will be ignored.

    Examples
    --------

      >>> # get all JSON files under "mybucket/foo/"
      >>> for key, content in iter_bucket(bucket_name, prefix='foo/', accept_key=lambda key: key.endswith('.json')):
      ...     print key, len(content)

      >>> # limit to 10k files, using 32 parallel workers (default is 16)
      >>> for key, content in iter_bucket(bucket_name, key_limit=10000, workers=32):
      ...     print key, len(content)
    """
    if accept_key is None:
        accept_key = lambda key: True

    #
    # If people insist on giving us bucket instances, silently extract the name
    # before moving on.  Works for boto3 as well as boto.
    #
    try:
        bucket_name = bucket_name.name
    except AttributeError:
        pass

    total_size, key_no = 0, -1
    key_iterator = _list_bucket(bucket_name, prefix=prefix, accept_key=accept_key)
    download_key = functools.partial(_download_key, bucket_name=bucket_name, retries=retries)

    with _create_process_pool(processes=workers) as pool:
        result_iterator = pool.imap_unordered(download_key, key_iterator)
        for key_no, (key, content) in enumerate(result_iterator):
            if True or key_no % 1000 == 0:
                logger.info(
                    "yielding key #%i: %s, size %i (total %.1fMB)",
                    key_no, key, len(content), total_size / 1024.0 ** 2
                )
            yield key, content
            total_size += len(content)

            if key_limit is not None and key_no + 1 >= key_limit:
                # we were asked to output only a limited number of keys => we're done
                break
    logger.info("processed %i keys, total size %i" % (key_no + 1, total_size))

    if len(gold) > 0:
        matthews_score = matthews_corrcoef(gold, pred)
    if within_tree == 0 or nb_paths == 0:
        return (root, -5, -5, -5, -5)
    return (root, outside_edges/len(tree_nodes), one_neg_edges/within_tree,
            matthews_score, total_path_length/nb_paths)


def tree_score(inside_edges, outside_edges):
    return inside_edges - outside_edges


def merge_trees(list_of_tree):
    list_of_tree = sorted(list_of_tree, key=lambda x: x[1])


def cbfs(root):
    return consistent_bfs(ADJACENCY, EDGE_SIGNS, root)

if __name__ == '__main__':
    # pylint: disable=C0103
    import persistent as p
    from multiprocessing import pool
    import random
    pool = pool.Pool(13)
    roots = random.sample(list(ADJACENCY.keys()), 10000)
    res = pool.imap_unordered(cbfs, roots, chunksize=len(roots)//13)
    pool.close()
    pool.join()
    p.save_var('cbfs_val.my', list(res))