Python SubprocPool.background方法代码示例

# 需要导入模块: from pants.base.worker_pool import SubprocPool [as 别名]
# 或者: from pants.base.worker_pool.SubprocPool import background [as 别名]
  def exec_on_subproc(self, f, args):
    """Send work to a subprocess and block on it.

       This can be used by existing background Work in a ThreadPool to sidestep the GIL:
       The Thread calls this method and still blocks until the work is complete, so
       existing reporting and accounting is unchanged, but the actual work is executed
       in a subprocess, avoiding lock contention.

       :param f: A multiproc-friendly (importable) work function.
       :param args: Multiproc-friendly (pickleable) arguments to f.
    """
    return SubprocPool.background().apply(f, args)

# 需要导入模块: from pants.base.worker_pool import SubprocPool [as 别名]
# 或者: from pants.base.worker_pool.SubprocPool import background [as 别名]
  def __init__(self,
               info_dir,
               stats_upload_url=None,
               stats_upload_timeout=2,
               num_foreground_workers=8,
               num_background_workers=8):
    self.run_timestamp = time.time()  # A double, so we get subsecond precision for ids.
    cmd_line = ' '.join(['./pants'] + sys.argv[1:])

    # run_id is safe for use in paths.
    millis = (self.run_timestamp * 1000) % 1000
    run_id = 'pants_run_%s_%d' % \
             (time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(self.run_timestamp)), millis)

    self.info_dir = os.path.join(info_dir, run_id)
    self.run_info = RunInfo(os.path.join(self.info_dir, 'info'))
    self.run_info.add_basic_info(run_id, self.run_timestamp)
    self.run_info.add_info('cmd_line', cmd_line)
    self.stats_url = stats_upload_url
    self.stats_timeout = stats_upload_timeout

    # Create a 'latest' symlink, after we add_infos, so we're guaranteed that the file exists.
    link_to_latest = os.path.join(os.path.dirname(self.info_dir), 'latest')
    if os.path.lexists(link_to_latest):
      os.unlink(link_to_latest)
    os.symlink(self.info_dir, link_to_latest)

    # Time spent in a workunit, including its children.
    self.cumulative_timings = AggregatedTimings(os.path.join(self.info_dir, 'cumulative_timings'))

    # Time spent in a workunit, not including its children.
    self.self_timings = AggregatedTimings(os.path.join(self.info_dir, 'self_timings'))

    # Hit/miss stats for the artifact cache.
    self.artifact_cache_stats = \
      ArtifactCacheStats(os.path.join(self.info_dir, 'artifact_cache_stats'))

    # Number of threads for foreground work.
    self._num_foreground_workers = num_foreground_workers

    # Number of threads for background work.
    self._num_background_workers = num_background_workers

    # We report to this Report.
    self.report = None

    # self._threadlocal.current_workunit contains the current workunit for the calling thread.
    # Note that multiple threads may share a name (e.g., all the threads in a pool).
    self._threadlocal = threading.local()

    # For main thread work. Created on start().
    self._main_root_workunit = None

    # For concurrent foreground work.  Created lazily if needed.
    # Associated with the main thread's root workunit.
    self._foreground_worker_pool = None

    # For background work.  Created lazily if needed.
    self._background_worker_pool = None
    self._background_root_workunit = None

    # Trigger subproc pool init while our memory image is still clean (see SubprocPool docstring)
    SubprocPool.foreground()
    SubprocPool.background()

    self._aborted = False