Python queue_runner.start_queue_runners方法代码示例

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def create_session(self):
      """Creates a coordinated session."""
      # Keep the tf_sess for unit testing.
      self.tf_sess = self._session_creator.create_session()
      # We don't want coordinator to suppress any exception.
      self.coord = coordinator.Coordinator(clean_stop_exception_types=[])
      queue_runner.start_queue_runners(sess=self.tf_sess, coord=self.coord)
      # Inform the hooks that a new session has been created.
      for hook in self._hooks:
        hook.after_create_session(self.tf_sess, self.coord)
      return _CoordinatedSession(
          _HookedSession(self.tf_sess, self._hooks), self.coord,
          self._stop_grace_period_secs) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def create_session(self):
      """Creates a coordinated session."""
      # Keep the tf_sess for unit testing.
      self.tf_sess = self._session_creator.create_session()
      # We don't want coordinator to suppress any exception.
      self.coord = coordinator.Coordinator(clean_stop_exception_types=[])
      queue_runner.start_queue_runners(sess=self.tf_sess, coord=self.coord)
      # Inform the hooks that a new session has been created.
      for hook in self._hooks:
        hook.after_create_session(self.tf_sess, self.coord)
      return _CoordinatedSession(
          _HookedSession(self.tf_sess, self._hooks), self.coord) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def create_session(self):
      """Creates a coordinated session."""
      # Keep the tf_sess for unit testing.
      self.tf_sess = self._session_creator.create_session()
      # We don't want coordinator to suppress any exception.
      self.coord = coordinator.Coordinator(clean_stop_exception_types=[])
      queue_runner.start_queue_runners(sess=self.tf_sess, coord=self.coord)
      return _CoordinatedSession(
          _HookedSession(self.tf_sess, self._hooks), self.coord) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def run(self,
          num_batches=None,
          graph=None,
          session=None,
          start_queues=True,
          initialize_variables=True,
          **kwargs):
    """Builds and runs the columns of the `DataFrame` and yields batches.

    This is a generator that yields a dictionary mapping column names to
    evaluated columns.

    Args:
      num_batches: the maximum number of batches to produce. If none specified,
        the returned value will iterate through infinite batches.
      graph: the `Graph` in which the `DataFrame` should be built.
      session: the `Session` in which to run the columns of the `DataFrame`.
      start_queues: if true, queues will be started before running and halted
        after producting `n` batches.
      initialize_variables: if true, variables will be initialized.
      **kwargs: Additional keyword arguments e.g. `num_epochs`.

    Yields:
      A dictionary, mapping column names to the values resulting from running
      each column for a single batch.
    """
    if graph is None:
      graph = ops.get_default_graph()
    with graph.as_default():
      if session is None:
        session = sess.Session()
      self_built = self.build(**kwargs)
      keys = list(self_built.keys())
      cols = list(self_built.values())
      if initialize_variables:
        if variables.local_variables():
          session.run(variables.local_variables_initializer())
        if variables.global_variables():
          session.run(variables.global_variables_initializer())
      if start_queues:
        coord = coordinator.Coordinator()
        threads = qr.start_queue_runners(sess=session, coord=coord)
      i = 0
      while num_batches is None or i < num_batches:
        i += 1
        try:
          values = session.run(cols)
          yield collections.OrderedDict(zip(keys, values))
        except errors.OutOfRangeError:
          break
      if start_queues:
        coord.request_stop()
        coord.join(threads) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None):
  """Run `output_dict` tensors with each input in `feed_dicts`.

  If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise,
  init all variables.

  Args:
    output_dict: A `dict` mapping string names to `Tensor` objects to run.
      Tensors must all be from the same graph.
    feed_dicts: Iterable of `dict` objects of input values to feed.
    restore_checkpoint_path: A string containing the path to a checkpoint to
      restore.

  Yields:
    A sequence of dicts of values read from `output_dict` tensors, one item
    yielded for each item in `feed_dicts`. Keys are the same as `output_dict`,
    values are the results read from the corresponding `Tensor` in
    `output_dict`.

  Raises:
    ValueError: if `output_dict` or `feed_dicts` is None or empty.
  """
  if not output_dict:
    raise ValueError('output_dict is invalid: %s.' % output_dict)
  if not feed_dicts:
    raise ValueError('feed_dicts is invalid: %s.' % feed_dicts)

  graph = contrib_ops.get_graph_from_inputs(output_dict.values())
  with graph.as_default() as g:
    with tf_session.Session('') as session:
      session.run(
          resources.initialize_resources(resources.shared_resources() +
                                         resources.local_resources()))
      if restore_checkpoint_path:
        _restore_from_checkpoint(session, g, restore_checkpoint_path)
      else:
        session.run(variables.global_variables_initializer())
      session.run(variables.local_variables_initializer())
      session.run(lookup_ops.tables_initializer())
      coord = coordinator.Coordinator()
      threads = None
      try:
        threads = queue_runner.start_queue_runners(session, coord=coord)
        for f in feed_dicts:
          yield session.run(output_dict, f)
      finally:
        coord.request_stop()
        if threads:
          coord.join(threads, stop_grace_period_secs=120) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def run_feeding_forever(self,
                          sess_callback,
                          outer_coordinator=None,
                          tolerator=None,
                          start_queue_runners=True):
    """Runs feeding forever.

    This method exits only if `outer_coordinator` has a stop requested
    or if a remote feed encounters an un-tolerated error. The most
    likely cause of `outer_coordinator` stopping besides a manual call
    to `request_stop()` is a `QueueRunner` thread reaching the end of
    its queue or encountering an error.

    Returns only after joining `outer_coordinator`.

    Args:
      sess_callback: A function which, when called, returns a Session
        to use for feeding. Can be called multiple times due to retries.
      outer_coordinator: If present, a `Coordinator` which the feeding
        process will respect. Will be created if omitted.
      tolerator: If present, a `failure_tolerator.FailureTolerator` which is
        used to manage retries of feeding the remote devices.
      start_queue_runners: Whether to start queue runners before
        beginning to feed the remote devices. Defaults to True. If
        False and no other mechanism is used to start queue runners, this
        method will hang forever without doing work.

    """
    # We use /two/ coordinators: one which runs normal queue
    # runners (outer_coordinator), and one which runs the remote
    # enqueues (using an inner coordinator) with retries and failure
    # tolerance. By using two coordinators, errors
    # encountered while running the remote enqueue ops don't cause the
    # outer_coordinator to be shut down.
    if outer_coordinator is None:
      outer_coordinator = coordinator.Coordinator()

    # Start the outer queue runners:
    if start_queue_runners:
      session = sess_callback()
      # Work around b/32749157 by running an operation before proceeding --
      # this way the session used for queue runners will be fully established
      # before we create another session with the same target.
      session.run(self._fake_op)
      queue_runner.start_queue_runners(sess=session,
                                       coord=outer_coordinator)

    if self._num_remote_feeds == 0:
      self._feeding_event.set()
      outer_coordinator.join()
      return
    else:
      try:
        self._feed_remote_queues_forever(
            sess_callback, outer_coordinator, tolerator)
      finally:
        self._feeding_event.set()
        outer_coordinator.join() 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def _feed_remote_queues_forever(
      self, sess_callback, outer_coordinator, tolerator):
    if tolerator is None:
      tolerator = failure_tolerator.FailureTolerator(limit=5)

    # In a retry loop, keep the remote queue runners going:
    while True:
      if outer_coordinator.should_stop():
        return

      inner_coordinator = coordinator.Coordinator()

      # Make sure inner_coordinator stops when outer_coordinator does:
      _link_coordinators(inner_coordinator, outer_coordinator)

      # Create a fresh session to use for remote queues:
      inner_session = sess_callback()
      inner_session.run(self._fake_op)  # Work around b/32749157, as above

      queue_runner.start_queue_runners(sess=inner_session,
                                       coord=inner_coordinator,
                                       collection=Feeder.REMOTE_QUEUE_RUNNERS)

      self._feeding_event.set()  # Notify that feeding has begun

      try:
        with tolerator.forgive():
          # Wait for a stop to be requested.
          inner_coordinator.wait_for_stop()

          # TODO(shoutis): If outer_coordinator.should_stop(), it
          # would be nice to interrupt the remote queue runners (which
          # may be blocked if their remote queue is full) -- but
          # there's no way currently; see b/32774422.

          # Cause any exceptions from the remote queue runners to be
          # reraised immediately, without waiting for their associated
          # threads to terminate like join() would. This means a retry
          # can begin immediately after any remote device fails,
          # rather than having to wait for any pending enqueues to
          # other remote devices to finish first.
          inner_coordinator.raise_requested_exception()

          # If this line is reached, there was a graceful shutdown
          # requested.

          # Request the outer coordinator to stop. Since
          # outer_coordinator.request_stop() is the currently only way
          # for inner_coordinator() to finish without failure, this is
          # redundant, but it's harmless and defends against infinite
          # hangs should code changes make it possible for
          # inner_coordinator to finish in other ways.
          outer_coordinator.request_stop()

          return
      except Exception as e:
        # Pass non-forgiven errors along to outer_coordinator:
        outer_coordinator.request_stop(e)
        raise 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None):
  """Run `output_dict` tensors with each input in `feed_dicts`.

  If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise,
  init all variables.

  Args:
    output_dict: A `dict` mapping string names to `Tensor` objects to run.
      Tensors must all be from the same graph.
    feed_dicts: Iterable of `dict` objects of input values to feed.
    restore_checkpoint_path: A string containing the path to a checkpoint to
      restore.

  Yields:
    A sequence of dicts of values read from `output_dict` tensors, one item
    yielded for each item in `feed_dicts`. Keys are the same as `output_dict`,
    values are the results read from the corresponding `Tensor` in
    `output_dict`.

  Raises:
    ValueError: if `output_dict` or `feed_dicts` is None or empty.
  """
  if not output_dict:
    raise ValueError('output_dict is invalid: %s.' % output_dict)
  if not feed_dicts:
    raise ValueError('feed_dicts is invalid: %s.' % feed_dicts)

  graph = contrib_ops.get_graph_from_inputs(output_dict.values())
  with graph.as_default() as g:
    with tf_session.Session('') as session:
      session.run(
          resources.initialize_resources(resources.shared_resources() +
                                         resources.local_resources()))
      if restore_checkpoint_path:
        _restore_from_checkpoint(session, g, restore_checkpoint_path)
      else:
        session.run(variables.global_variables_initializer())
      session.run(variables.local_variables_initializer())
      session.run(data_flow_ops.tables_initializer())
      coord = coordinator.Coordinator()
      threads = None
      try:
        threads = queue_runner.start_queue_runners(session, coord=coord)
        for f in feed_dicts:
          yield session.run(output_dict, f)
      finally:
        coord.request_stop()
        if threads:
          coord.join(threads, stop_grace_period_secs=120) 

# 需要导入模块: from tensorflow.python.training import queue_runner [as 别名]
# 或者: from tensorflow.python.training.queue_runner import start_queue_runners [as 别名]
def run_feeds_iter(output_dict, feed_dicts, restore_checkpoint_path=None):
  """Run `output_dict` tensors with each input in `feed_dicts`.

  If `restore_checkpoint_path` is supplied, restore from checkpoint. Otherwise,
  init all variables.

  Args:
    output_dict: A `dict` mapping string names to `Tensor` objects to run.
      Tensors must all be from the same graph.
    feed_dicts: Iterable of `dict` objects of input values to feed.
    restore_checkpoint_path: A string containing the path to a checkpoint to
      restore.

  Yields:
    A sequence of dicts of values read from `output_dict` tensors, one item
    yielded for each item in `feed_dicts`. Keys are the same as `output_dict`,
    values are the results read from the corresponding `Tensor` in
    `output_dict`.

  Raises:
    ValueError: if `output_dict` or `feed_dicts` is None or empty.
  """
  if not output_dict:
    raise ValueError('output_dict is invalid: %s.' % output_dict)
  if not feed_dicts:
    raise ValueError('feed_dicts is invalid: %s.' % feed_dicts)

  graph = contrib_ops.get_graph_from_inputs(output_dict.values())
  with graph.as_default() as g:
    with tf_session.Session('') as session:
      session.run(
          resources.initialize_resources(resources.shared_resources() +
                                         resources.local_resources()))
      if restore_checkpoint_path:
        _restore_from_checkpoint(session, g, restore_checkpoint_path)
      else:
        session.run(variables.global_variables_initializer())
      session.run(variables.local_variables_initializer())
      session.run(data_flow_ops.initialize_all_tables())
      coord = coordinator.Coordinator()
      threads = None
      try:
        threads = queue_runner.start_queue_runners(session, coord=coord)
        for f in feed_dicts:
          yield session.run(output_dict, f)
      finally:
        coord.request_stop()
        if threads:
          coord.join(threads, stop_grace_period_secs=120)