Python storage.Storage类代码示例

def _process_initializer(node_builder, storage):
  """Another picklable top-level function that provides multi-processes' initial states.

  States are returned as a tuple. States are `Closable` so they can be cleaned up once
  processes are done.
  """
  return (node_builder, Storage.clone(storage))

 def __init__(self):
   # Objects indexed by their keys, i.e, content digests
   self._objects = Storage.create()
   # Memoized object Ids.
   self._id_to_key = dict()
   self._key_to_id = dict()
   self._next_id = 0

  def setup_legacy_graph(path_ignore_patterns):
    """Construct and return the components necessary for LegacyBuildGraph construction.

    :param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
                                      usually taken from the `--pants-ignore` global option.
    :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
    """

    build_root = get_buildroot()
    project_tree = FileSystemProjectTree(build_root, path_ignore_patterns)
    symbol_table_cls = LegacySymbolTable

    # Register "literal" subjects required for these tasks.
    # TODO: Replace with `Subsystems`.
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=LegacyPythonCallbacksParser)

    # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
    # LegacyBuildGraph will explicitly request the products it needs.
    tasks = (
      create_legacy_graph_tasks() +
      create_fs_tasks() +
      create_graph_tasks(address_mapper, symbol_table_cls)
    )

    scheduler = LocalScheduler(dict(), tasks, project_tree)
    engine = LocalSerialEngine(scheduler, Storage.create(debug=False))

    return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)

  def __init__(self, scheduler, storage=None, cache=None, pool_size=None, debug=True):
    """
    :param scheduler: The local scheduler for creating execution graphs.
    :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
    :param storage: The storage instance for serializables keyed by their hashes.
    :type storage: :class:`pants.engine.storage.Storage`
    :param cache: The cache instance for storing execution results, by default it uses the same
      Storage instance if not specified.
    :type cache: :class:`pants.engine.storage.Cache`
    :param int pool_size: The number of worker processes to use; by default 2 processes per core will
                          be used.
    :param bool debug: `True` to turn on pickling error debug mode (slower); True by default.
    """
    # This is the only place where non in-memory storage is needed, create one if not specified.
    storage = storage or Storage.create(in_memory=False)
    super(LocalMultiprocessEngine, self).__init__(scheduler, storage, cache)
    self._pool_size = pool_size if pool_size and pool_size > 0 else 2 * multiprocessing.cpu_count()

    execute_step = functools.partial(_execute_step, debug)

    self._processed_queue = Queue()
    self.node_builder = scheduler.node_builder
    process_initializer = functools.partial(_process_initializer, self._storage)
    self._pool = StatefulPool(self._pool_size, process_initializer, execute_step)
    self._debug = debug
    self._pool.start()

 def hybrid_engine(self, pool_size=None):
   async_nodes = (FilesystemNode,)
   storage = Storage.create(in_memory=True)
   cache = Cache.create(storage=storage)
   with closing(ThreadHybridEngine(self.scheduler, storage,
                                   threaded_node_types=async_nodes, cache=cache,
                                   pool_size=pool_size, debug=True)) as e:
     yield e

def _process_initializer(storage):
  """Another picklable top-level function that provides multi-processes' initial states.

  States are returned as a tuple. States are `Closable` so they can be cleaned up once
  processes are done.
  """
  storage = Storage.clone(storage)
  return (storage, Cache.create(storage=storage))

 def setUp(self):
   self.storage = Storage.create()
   self.result = StepResult(state='something')
   self.request = StepRequest(step_id=123,
                              node='some node',
                              dependencies={'some dep': 'some state',
                                            'another dep': 'another state'},
                              inline_nodes=False,
                              project_tree='some project tree')

 def setUp(self):
     self.storage = Storage.create(in_memory=True)
     self.result = StepResult(state="something")
     self.request = StepRequest(
         step_id=123,
         node="some node",
         dependencies={"some dep": "some state", "another dep": "another state"},
         project_tree="some project tree",
     )

def visualize_build_request(build_root, goals, subjects):
  with subsystem_instance(Native.Factory) as native_factory:
    scheduler = setup_json_scheduler(build_root, native_factory.create())

    execution_request = scheduler.build_request(goals, subjects)
    # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
    engine = LocalSerialEngine(scheduler, Storage.create())
    engine.reduce(execution_request)
    visualize_execution_graph(scheduler)

def visualize_build_request(build_root, goals, subjects):
  scheduler = setup_json_scheduler(build_root)

  execution_request = scheduler.build_request(goals, subjects)
  # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
  engine = LocalSerialEngine(scheduler, Storage.create())
  try:
    engine.reduce(execution_request)
    visualize_execution_graph(scheduler, execution_request)
  finally:
    engine.close()

 def __init__(self, scheduler, storage=None, cache=None):
   """
   :param scheduler: The local scheduler for creating execution graphs.
   :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
   :param storage: The storage instance for serializables keyed by their hashes.
   :type storage: :class:`pants.engine.storage.Storage`
   :param cache: The cache instance for storing execution results, by default it uses the same
     Storage instance if not specified.
   :type cache: :class:`pants.engine.storage.Cache`
   """
   self._scheduler = scheduler
   self._storage = storage or Storage.create()
   self._cache = cache or Cache.create(storage)

  def mk_scheduler(self,
                   tasks=None,
                   goals=None,
                   storage=None,
                   project_tree=None,
                   symbol_table_cls=EmptyTable):
    """Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
    goals = goals or dict()
    tasks = tasks or []
    storage = storage or Storage.create(in_memory=True)
    project_tree = project_tree or self.mk_fs_tree()

    tasks = list(tasks) + create_fs_tasks()
    scheduler = LocalScheduler(goals, tasks, storage, project_tree)
    return scheduler, storage

 def __init__(self, scheduler, storage=None, cache=None, use_cache=True):
   """
   :param scheduler: The local scheduler for creating execution graphs.
   :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
   :param storage: The storage instance for serializables keyed by their hashes.
   :type storage: :class:`pants.engine.storage.Storage`
   :param cache: The cache instance for storing execution results, by default it uses the same
     Storage instance if not specified.
   :type cache: :class:`pants.engine.storage.Cache`
   :param use_cache: True to enable usage of the cache. The cache incurs a large amount of
     overhead for small tasks, and needs TODO: further improvement.
   :type use_cache: bool
   """
   self._scheduler = scheduler
   self._storage = storage or Storage.create()
   self._cache = cache or Cache.create(storage)
   self._use_cache = use_cache

  def setup_legacy_graph(pants_ignore_patterns,
                         build_root=None,
                         symbol_table_cls=None,
                         build_ignore_patterns=None,
                         exclude_target_regexps=None):
    """Construct and return the components necessary for LegacyBuildGraph construction.

    :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
                                       usually taken from the '--pants-ignore' global option.
    :param str build_root: A path to be used as the build root. If None, then default is used.
    :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
                                         None to use the default.
    :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
                                       files, usually taken from the '--build-ignore' global option.
    :param list exclude_target_regexps: A list of regular expressions for excluding targets.
    :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
    """

    build_root = build_root or get_buildroot()
    scm = get_scm()
    symbol_table_cls = symbol_table_cls or LegacySymbolTable

    project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)

    # Register "literal" subjects required for these tasks.
    # TODO: Replace with `Subsystems`.
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=LegacyPythonCallbacksParser,
                                   build_ignore_patterns=build_ignore_patterns,
                                   exclude_target_regexps=exclude_target_regexps)

    # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
    # LegacyBuildGraph will explicitly request the products it needs.
    tasks = (
      create_legacy_graph_tasks(symbol_table_cls) +
      create_fs_tasks() +
      create_graph_tasks(address_mapper, symbol_table_cls)
    )

    scheduler = LocalScheduler(dict(), tasks, project_tree)
    # TODO: Do not use the cache yet, as it incurs a high overhead.
    engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False)
    change_calculator = EngineChangeCalculator(engine, scm) if scm else None

    return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)

  def setUp(self):
    # Set up a scheduler that supports address mapping.
    symbol_table_cls = TargetTable
    self.storage = Storage.create(in_memory=True)
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=JsonParser,
                                   build_pattern=r'.+\.BUILD.json$')
    tasks = create_graph_tasks(address_mapper, symbol_table_cls)

    project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
    self.build_root = project_tree.build_root
    self.scheduler, _ = self.mk_scheduler(tasks=tasks,
                                          project_tree=project_tree,
                                          storage=self.storage,
                                          symbol_table_cls=symbol_table_cls)

    self.a_b = Address.parse('a/b')
    self.a_b_target = Target(name='b',
                             dependencies=['//d:e'],
                             configurations=['//a', Struct(embedded='yes')],
                             type_alias='target')