Python ThreadPool.terminate方法代码示例

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def run_using_threadpool(fn_to_execute, inputs, pool_size):
  """For internal use only; no backwards-compatibility guarantees.

  Runs the given function on given inputs using a thread pool.

  Args:
    fn_to_execute: Function to execute
    inputs: Inputs on which given function will be executed in parallel.
    pool_size: Size of thread pool.
  Returns:
    Results retrieved after executing the given function on given inputs.
  """

  # ThreadPool crashes in old versions of Python (< 2.7.5) if created
  # from a child thread. (http://bugs.python.org/issue10015)
  if not hasattr(threading.current_thread(), '_children'):
    threading.current_thread()._children = weakref.WeakKeyDictionary()
  pool = ThreadPool(min(pool_size, len(inputs)))
  try:
    # We record and reset logging level here since 'apitools' library Beam
    # depends on updates the logging level when used with a threadpool -
    # https://github.com/google/apitools/issues/141
    # TODO: Remove this once above issue in 'apitools' is fixed.
    old_level = logging.getLogger().level
    return pool.map(fn_to_execute, inputs)
  finally:
    pool.terminate()
    logging.getLogger().setLevel(old_level)

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
    def launch_parallel_tests(self):
        image_name = "django_parallel_tests/%s" % self.project_name
        if len(self.docker.images(name=image_name)) == 0:
            self.build_image()

        req_hash = hashlib.sha224(str(sorted(self.requirements))).hexdigest()
        try:
            last_req_hash = open(".last_requirements").read().strip()
        except:
            last_req_hash = None

        if req_hash != last_req_hash:
            self.build_image()
            with open(".last_requirements", "w") as f:
                f.write(req_hash)

        pool = ThreadPool()
        tests = [[test] for test in self.tests]
        run_tests = partial(run_tests_for_project, self.project_name)

        result = pool.map_async(run_tests, tests)
        try:
            while True:
                time.sleep(0.1)
                if result.ready():
                    print "got result", result.get()
                    return
        except KeyboardInterrupt:
            pool.terminate()
            pool.join()
        else:
            pool.close()
            pool.join()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def download(country=None, processes=16):
    if not os.path.exists(COUNTRIES_VALIDATION_DATA_DIR):
        os.mkdir(COUNTRIES_VALIDATION_DATA_DIR)
    data = manager.dict()
    countries = get_countries()
    if country:
        country = country.upper()
        if country not in countries:
            raise ValueError(
                    '%s is not supported country code' % country)
        countries = [country]
    for country in countries:
        work_queue.put(country)
    workers = ThreadPool(processes, worker, initargs=(data,))
    work_queue.join()
    workers.terminate()
    logger.debug('Queue finished')
    with io.open(COUNTRY_PATH % 'all', 'w', encoding='utf8') as all_output:
        all_output.write(u'{')
        for country in countries:
            country_dict = {}
            for key, address_data in data.items():
                if key[:2] == country:
                    country_dict[key] = address_data
            logger.debug('Saving %s', country)
            country_json = serialize(country_dict, COUNTRY_PATH % country.lower())
            all_output.write(country_json[1:-1])
            if country != countries[-1]:
                all_output.write(u',')
        all_output.write(u'}')

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
    def _run_tests(self):
        "Runs the tests, produces no report."
        run_alone = []

        tests = self._tests
        pool = ThreadPool(self._worker_count)
        try:
            for cmd, options in tests:
                options = options or {}
                if matches(self._configured_run_alone_tests, cmd):
                    run_alone.append((cmd, options))
                else:
                    self._spawn(pool, cmd, options)
            pool.close()
            pool.join()

            if run_alone:
                util.log("Running tests marked standalone")
                for cmd, options in run_alone:
                    self._run_one(cmd, **options)
        except KeyboardInterrupt:
            try:
                util.log('Waiting for currently running to finish...')
                self._reap_all()
            except KeyboardInterrupt:
                pool.terminate()
                raise
        except:
            pool.terminate()
            raise

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def download(processes=16):
    if not os.path.exists(COUNTRIES_VALIDATION_DATA_DIR):
        os.mkdir(COUNTRIES_VALIDATION_DATA_DIR)
    data = manager.dict()
    countries = fetch(MAIN_URL)['countries'].split('~')
    countries = ['PL']
    for country in countries:
        work_queue.put(country)
    workers = ThreadPool(processes, worker, initargs=(data,))
    work_queue.join()
    workers.terminate()
    logger.debug('Queue finished')
    with io.open(COUNTRY_PATH % 'all', 'w', encoding='utf8') as all_output:
        all_output.write(u'{')
        for country in countries:
            country_dict = {}
            for key, address_data in data.items():
                if key[:2] == country:
                    country_dict[key] = address_data
            logger.debug('Saving %s', country)
            country_json = serialize(country_dict, COUNTRY_PATH % country.lower())
            all_output.write(country_json[1:-1])
            if country != countries[-1]:
                all_output.write(u',')
        all_output.write(u'}')

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
 def run(self, max_number_of_live_tokens=None, group=None):
     group = Pool()
     try:
         stages = []
         
         in_q = _DummyQueue()
         end_in = Event()
         if self._filters[0].is_serial:
             serial = Lock()
         else:
             serial = _DummyLock()
         if self._filters[0].is_ordered:
             out_q = PriorityQueue()
         else:
             out_q = Queue()
         
         
         for i, f in enumerate(self._filters):
             pass
                 
         send_q, recv_q = Queue(), Queue()
         
         group.close()
     except:
         group.terminate()
     finally:
         group.join()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
  def collect_logs(self):
    """Collect all the microservice log files."""
    log_dir = os.path.join(self.options.log_dir, 'service_logs')
    if not os.path.exists(log_dir):
      os.makedirs(log_dir)

    def fetch_service_log(service):
      try:
        logging.debug('Fetching logs for "%s"...', service)
        deployer = (self if service in HALYARD_SERVICES
                    else self.__spinnaker_deployer)
        deployer.do_fetch_service_log_file(service, log_dir)
      except Exception as ex:
        message = 'Error fetching log for service "{service}": {ex}'.format(
            service=service, ex=ex)
        if ex.message.find('No such file') >= 0:
          message += '\n    Perhaps the service never started.'
          # dont log since the error was already captured.
        else:
          logging.error(message)
          message += '\n{trace}'.format(
              trace=traceback.format_exc())

        write_data_to_secure_path(
            message, os.path.join(log_dir, service + '.log'))

    logging.info('Collecting server log files into "%s"', log_dir)
    all_services = list(SPINNAKER_SERVICES)
    all_services.extend(HALYARD_SERVICES)
    thread_pool = ThreadPool(len(all_services))
    thread_pool.map(fetch_service_log, all_services)
    thread_pool.terminate()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def get_for_genres(genres):
    genres = set(genres)
    playlists = {}
    new_genres = set()

    for page in xrange(5):
        args = []
        for g in genres:
            args.append((g, page))

        try:
            pool = ThreadPool(PROCESSES)
            pfunc = parse_page
            for i, res in enumerate(pool.imap_unordered(pfunc, args)):
                genre, page, pl, found = res
                print "%d/%d" % (i + 1, len(args))
                playlists.update(pl)
                new_genres |= found
                if not pl:
                    genres.remove(genre)
        except Exception as e:
            print e
            return playlists, []
        finally:
            pool.terminate()
            pool.join()

    return playlists, new_genres

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
    def getMessagesBySource(self, source, batch_mode=False):
        """
        Returns the messages for the given source, including messages
        from the configured builder (if available) and static checks
        Extra arguments are
        """
        self._setupEnvIfNeeded()

        if self._USE_THREADS:
            records = []
            pool = ThreadPool()

            static_check = pool.apply_async(
                getStaticMessages, args=(source.getSourceContent().split('\n'), ))

            if self._isBuilderCallable():
                builder_check = pool.apply_async(self._getBuilderMessages,
                                                 args=[source, batch_mode])
                records += builder_check.get()

            records += static_check.get()

            pool.terminate()
            pool.join()
        else:
            records = getStaticMessages(source.getSourceContent().split('\n'))
            if self._isBuilderCallable():
                records += self._getBuilderMessages(source, batch_mode)

        self._saveCache()
        return records

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def run_tidy(sha="HEAD", is_rev_range=False):
    diff_cmdline = ["git", "diff" if is_rev_range else "show", sha]

    # Figure out which paths changed in the given diff.
    changed_paths = subprocess.check_output(diff_cmdline + ["--name-only", "--pretty=format:"]).splitlines()
    changed_paths = [p for p in changed_paths if p]

    # Produce a separate diff for each file and run clang-tidy-diff on it
    # in parallel.
    def tidy_on_path(path):
        patch_file = tempfile.NamedTemporaryFile()
        cmd = diff_cmdline + [
            "--src-prefix=%s/" % ROOT,
            "--dst-prefix=%s/" % ROOT,
            "--",
            path]
        subprocess.check_call(cmd, stdout=patch_file, cwd=ROOT)
        cmdline = [CLANG_TIDY_DIFF,
                   "-clang-tidy-binary", CLANG_TIDY,
                   "-p0",
                   "--",
                   "-DCLANG_TIDY"] + compile_flags.get_flags()
        return subprocess.check_output(
            cmdline,
            stdin=file(patch_file.name),
            cwd=ROOT)
    pool = ThreadPool(multiprocessing.cpu_count())
    try:
        return "".join(pool.imap(tidy_on_path, changed_paths))
    except KeyboardInterrupt as ki:
        sys.exit(1)
    finally:
        pool.terminate()
        pool.join()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
 def wrapper(*args, **kwargs):
     pool = ThreadPool(processes=1)
     async = pool.apply_async(callback, args, kwargs)
     try:
         return async.get(self.get_timeout_sec(route))
     except TimeoutError:
         pool.terminate()
         raise bottle.HTTPError(503, 'Service Unavailable, process timeout')

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
 def __do_proc(self, buf):
     pool = ThreadPool(processes=1)
     result = pool.apply_async(self.proc, args=(buf,))
     try:
         result.get(timeout=self.__timeout)
     except TimeoutError:
         self._log('timeout (%ss)' % str(self.__timeout))
     finally:
         pool.terminate()

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def pmap(f, iterable):
    """Map `f` over `iterable` in parallel using a ``ThreadPool``.
    """
    p = ThreadPool()
    try:
        result = p.map(f, iterable)
    finally:
        p.terminate()
    return result

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
    def get_data(self, targets, start="", stop="", interval_str=""):
        if isinstance(targets, basestring):
            targets = [targets]
        current_handlers = {}
        for target in targets:
            handler = self.get_handler(target)
            if handler not in current_handlers:
                current_handlers[handler] = []
            current_handlers[handler].append(target)
        max_targets = 5  # max targets per http request
        started_at = time.time()
        run_args = []
        for handler, targets in current_handlers.items():
            while targets:
                run_args.append((handler.get_data, targets[:max_targets]))
                targets = targets[max_targets:]

        def _run((func, targets)):
            try:
                return func(targets, start=start, stop=stop,
                            interval_str=interval_str)
            except Exception as exc:
                log.warning("Multihandler got response: %r", exc)
                return []

        pool = ThreadPool(10)
        parts = pool.map(_run, run_args)
        data = reduce(lambda x, y: x + y, parts)
        pool.terminate()
        log.info("Multihandler get_data completed in: %.2f secs",
                 time.time() - started_at)

        # align start/stop
        starts = set()
        stops = set()
        for item in data:
            starts.add(item['datapoints'][0][1])
            stops.add(item['datapoints'][-1][1])
        start = max(starts) if len(starts) > 1 else 0
        stop = min(stops) if len(stops) > 1 else 0
        if start or stop:
            log.debug("%s %s %s %s", starts, start, stops, stop)
            for item in data:
                if start:
                    for i in range(len(item['datapoints'])):
                        if item['datapoints'][i][1] >= start:
                            if i:
                                item['datapoints'] = item['datapoints'][i:]
                            break
                if stop:
                    for i in range(len(item['datapoints'])):
                        if item['datapoints'][-(i+1)][1] <= stop:
                            if i:
                                item['datapoints'] = item['datapoints'][:-i]
                            break
        return data

# 需要导入模块: from multiprocessing.pool import ThreadPool [as 别名]
# 或者: from multiprocessing.pool.ThreadPool import terminate [as 别名]
def main(compilation_db_path, source_files, verbose, formatter, iwyu_args):
    """ Entry point. """
    # Canonicalize compilation database path
    if os.path.isdir(compilation_db_path):
        compilation_db_path = os.path.join(compilation_db_path,
                                           'compile_commands.json')

    compilation_db_path = os.path.realpath(compilation_db_path)
    if not os.path.isfile(compilation_db_path):
        print('ERROR: No such file or directory: \'%s\'' % compilation_db_path)
        return 1

    # Read compilation db from disk
    with open(compilation_db_path, 'r') as fileobj:
        compilation_db = json.load(fileobj)

    # expand symlinks
    for entry in compilation_db:
        entry['file'] = os.path.realpath(entry['file'])

    # Cross-reference source files with compilation database
    source_files = [os.path.realpath(s) for s in source_files]
    if not source_files:
        # No source files specified, analyze entire compilation database
        entries = compilation_db
    else:
        # Source files specified, analyze the ones appearing in compilation db,
        # warn for the rest.
        entries = []
        for source in source_files:
            matches = [e for e in compilation_db if e['file'] == source]
            if matches:
                entries.extend(matches)
            else:
                print('WARNING: \'%s\' not found in compilation database.' %
                      source)

    # Run analysis
    def run_iwyu_task(entry):
        cwd, compile_command = entry['directory'], entry['command']
        compile_command = workaround_parent_dir_relative_includes(
            cwd, compile_command)
        return run_iwyu(cwd, compile_command, iwyu_args, verbose)
    pool = ThreadPool(multiprocessing.cpu_count())
    try:
        for iwyu_output in pool.imap_unordered(run_iwyu_task, entries):
            formatter(iwyu_output)
    except KeyboardInterrupt as ki:
        sys.exit(1)
    except OSError as why:
        print('ERROR: Failed to launch include-what-you-use: %s' % why)
        return 1
    finally:
        pool.terminate()
        pool.join()
    return 0