Python execution.MetricsContainer类代码示例

  def call(self):
    self._call_count += 1
    assert self._call_count <= (1 + len(self._applied_ptransform.side_inputs))
    metrics_container = MetricsContainer(self._applied_ptransform.full_label)
    scoped_metrics_container = ScopedMetricsContainer(metrics_container)

    for side_input in self._applied_ptransform.side_inputs:
      if side_input not in self._side_input_values:
        has_result, value = (
            self._evaluation_context.get_value_or_schedule_after_output(
                side_input, self))
        if not has_result:
          # Monitor task will reschedule this executor once the side input is
          # available.
          return
        self._side_input_values[side_input] = value

    side_input_values = [self._side_input_values[side_input]
                         for side_input in self._applied_ptransform.side_inputs]

    try:
      evaluator = self._transform_evaluator_registry.get_evaluator(
          self._applied_ptransform, self._input_bundle,
          side_input_values, scoped_metrics_container)

      if self._fired_timers:
        for timer_firing in self._fired_timers:
          evaluator.process_timer_wrapper(timer_firing)

      if self._input_bundle:
        for value in self._input_bundle.get_elements_iterable():
          evaluator.process_element(value)

      with scoped_metrics_container:
        result = evaluator.finish_bundle()
        result.logical_metric_updates = metrics_container.get_cumulative()

      if self._evaluation_context.has_cache:
        for uncommitted_bundle in result.uncommitted_output_bundles:
          self._evaluation_context.append_to_cache(
              self._applied_ptransform, uncommitted_bundle.tag,
              uncommitted_bundle.get_elements_iterable())
        undeclared_tag_values = result.undeclared_tag_values
        if undeclared_tag_values:
          for tag, value in undeclared_tag_values.iteritems():
            self._evaluation_context.append_to_cache(
                self._applied_ptransform, tag, value)

      self._completion_callback.handle_result(self, self._input_bundle, result)
      return result
    except Exception as e:  # pylint: disable=broad-except
      self._completion_callback.handle_exception(self, e)
    finally:
      self._evaluation_context.metrics().commit_physical(
          self._input_bundle,
          metrics_container.get_cumulative())
      self._transform_evaluation_state.complete(self)

  def __init__(self, operation_name, spec, counter_factory, state_sampler):
    """Initializes a worker operation instance.

    Args:
      operation_name: The system name assigned by the runner for this
        operation.
      spec: A operation_specs.Worker* instance.
      counter_factory: The CounterFactory to use for our counters.
      state_sampler: The StateSampler for the current operation.
    """
    self.operation_name = operation_name
    self.spec = spec
    self.counter_factory = counter_factory
    self.consumers = collections.defaultdict(list)

    # These are overwritten in the legacy harness.
    self.step_name = operation_name
    self.metrics_container = MetricsContainer(self.step_name)
    self.scoped_metrics_container = ScopedMetricsContainer(
        self.metrics_container)

    self.state_sampler = state_sampler
    self.scoped_start_state = self.state_sampler.scoped_state(
        self.operation_name, 'start')
    self.scoped_process_state = self.state_sampler.scoped_state(
        self.operation_name, 'process')
    self.scoped_finish_state = self.state_sampler.scoped_state(
        self.operation_name, 'finish')
    # TODO(ccy): the '-abort' state can be added when the abort is supported in
    # Operations.
    self.receivers = []

  def __init__(self, name_context, spec, counter_factory, state_sampler):
    """Initializes a worker operation instance.

    Args:
      name_context: A NameContext instance or string(deprecated), with the
        name information for this operation.
      spec: A operation_specs.Worker* instance.
      counter_factory: The CounterFactory to use for our counters.
      state_sampler: The StateSampler for the current operation.
    """
    if isinstance(name_context, common.NameContext):
      # TODO(BEAM-4028): Clean this up once it's completely migrated.
      # We use the specific operation name that is used for metrics and state
      # sampling.
      self.name_context = name_context
    else:
      self.name_context = common.NameContext(name_context)

    self.spec = spec
    self.counter_factory = counter_factory
    self.consumers = collections.defaultdict(list)

    # These are overwritten in the legacy harness.
    self.metrics_container = MetricsContainer(self.name_context.metrics_name())

    self.state_sampler = state_sampler
    self.scoped_start_state = self.state_sampler.scoped_state(
        self.name_context, 'start', metrics_container=self.metrics_container)
    self.scoped_process_state = self.state_sampler.scoped_state(
        self.name_context, 'process', metrics_container=self.metrics_container)
    self.scoped_finish_state = self.state_sampler.scoped_state(
        self.name_context, 'finish', metrics_container=self.metrics_container)
    # TODO(ccy): the '-abort' state can be added when the abort is supported in
    # Operations.
    self.receivers = []

  def test_uses_right_container(self):
    c1 = MetricsContainer('step1')
    c2 = MetricsContainer('step2')
    counter = Metrics.counter('ns', 'name')
    MetricsEnvironment.set_current_container(c1)
    counter.inc()
    MetricsEnvironment.set_current_container(c2)
    counter.inc(3)
    MetricsEnvironment.unset_current_container()

    self.assertEqual(
        c1.get_cumulative().counters.items(),
        [(MetricKey('step1', MetricName('ns', 'name')), 1)])

    self.assertEqual(
        c2.get_cumulative().counters.items(),
        [(MetricKey('step2', MetricName('ns', 'name')), 3)])

  def call(self):
    self._call_count += 1
    assert self._call_count <= (1 + len(self._applied_ptransform.side_inputs))
    metrics_container = MetricsContainer(self._applied_ptransform.full_label)
    scoped_metrics_container = ScopedMetricsContainer(metrics_container)

    for side_input in self._applied_ptransform.side_inputs:
      if side_input not in self._side_input_values:
        has_result, value = (
            self._evaluation_context.get_value_or_schedule_after_output(
                side_input, self))
        if not has_result:
          # Monitor task will reschedule this executor once the side input is
          # available.
          return
        self._side_input_values[side_input] = value
    side_input_values = [self._side_input_values[side_input]
                         for side_input in self._applied_ptransform.side_inputs]

    while self._retry_count < self._max_retries_per_bundle:
      try:
        self.attempt_call(metrics_container,
                          scoped_metrics_container,
                          side_input_values)
        break
      except Exception as e:
        self._retry_count += 1
        logging.error(
            'Exception at bundle %r, due to an exception.\n %s',
            self._input_bundle, traceback.format_exc())
        if self._retry_count == self._max_retries_per_bundle:
          logging.error('Giving up after %s attempts.',
                        self._max_retries_per_bundle)
          if self._retry_count == 1:
            logging.info(
                'Use the experimental flag --direct_runner_bundle_retry'
                ' to retry failed bundles (up to %d times).',
                TransformExecutor._MAX_RETRY_PER_BUNDLE)
          self._completion_callback.handle_exception(self, e)

    self._evaluation_context.metrics().commit_physical(
        self._input_bundle,
        metrics_container.get_cumulative())
    self._transform_evaluation_state.complete(self)

  def call(self):
    self._call_count += 1
    assert self._call_count <= (1 + len(self._applied_ptransform.side_inputs))
    metrics_container = MetricsContainer(self._applied_ptransform.full_label)
    scoped_metrics_container = ScopedMetricsContainer(metrics_container)

    for side_input in self._applied_ptransform.side_inputs:
      # Find the projection of main's window onto the side input's window.
      window_mapping_fn = side_input._view_options().get(
          'window_mapping_fn', sideinputs._global_window_mapping_fn)
      main_onto_side_window = window_mapping_fn(self._latest_main_input_window)
      block_until = main_onto_side_window.end

      if side_input not in self._side_input_values:
        value = self._evaluation_context.get_value_or_block_until_ready(
            side_input, self, block_until)
        if not value:
          # Monitor task will reschedule this executor once the side input is
          # available.
          return
        self._side_input_values[side_input] = value
    side_input_values = [self._side_input_values[side_input]
                         for side_input in self._applied_ptransform.side_inputs]

    while self._retry_count < self._max_retries_per_bundle:
      try:
        self.attempt_call(metrics_container,
                          scoped_metrics_container,
                          side_input_values)
        break
      except Exception as e:
        self._retry_count += 1
        logging.error(
            'Exception at bundle %r, due to an exception.\n %s',
            self._input_bundle, traceback.format_exc())
        if self._retry_count == self._max_retries_per_bundle:
          logging.error('Giving up after %s attempts.',
                        self._max_retries_per_bundle)
          self._completion_callback.handle_exception(self, e)

    self._evaluation_context.metrics().commit_physical(
        self._input_bundle,
        metrics_container.get_cumulative())
    self._transform_evaluation_state.complete(self)

  def test_scoped_container(self):
    c1 = MetricsContainer('mystep')
    c2 = MetricsContainer('myinternalstep')
    with ScopedMetricsContainer(c1):
      self.assertEqual(c1, MetricsEnvironment.current_container())
      counter = Metrics.counter('ns', 'name')
      counter.inc(2)

      with ScopedMetricsContainer(c2):
        self.assertEqual(c2, MetricsEnvironment.current_container())
        counter = Metrics.counter('ns', 'name')
        counter.inc(3)
        self.assertEqual(
            c2.get_cumulative().counters.items(),
            [(MetricKey('myinternalstep', MetricName('ns', 'name')), 3)])

      self.assertEqual(c1, MetricsEnvironment.current_container())
      counter = Metrics.counter('ns', 'name')
      counter.inc(4)
      self.assertEqual(
          c1.get_cumulative().counters.items(),
          [(MetricKey('mystep', MetricName('ns', 'name')), 6)])

  def test_get_cumulative_or_updates(self):
    mc = MetricsContainer('astep')

    clean_values = []
    dirty_values = []
    for i in range(1, 11):
      counter = mc.get_counter(MetricName('namespace', 'name{}'.format(i)))
      distribution = mc.get_distribution(
          MetricName('namespace', 'name{}'.format(i)))
      counter.inc(i)
      distribution.update(i)
      if i % 2 == 0:
        # Some are left to be DIRTY (i.e. not yet committed).
        # Some are left to be CLEAN (i.e. already committed).
        dirty_values.append(i)
        continue
      # Assert: Counter/Distribution is DIRTY or COMMITTING (not CLEAN)
      self.assertEqual(distribution.commit.before_commit(), True)
      self.assertEqual(counter.commit.before_commit(), True)
      distribution.commit.after_commit()
      counter.commit.after_commit()
      # Assert: Counter/Distribution has been committed, therefore it's CLEAN
      self.assertEqual(counter.commit.state, CellCommitState.CLEAN)
      self.assertEqual(distribution.commit.state, CellCommitState.CLEAN)
      clean_values.append(i)

    # Retrieve NON-COMMITTED updates.
    logical = mc.get_updates()
    self.assertEqual(len(logical.counters), 5)
    self.assertEqual(len(logical.distributions), 5)
    self.assertEqual(set(dirty_values),
                     set([v for _, v in logical.counters.items()]))
    # Retrieve ALL updates.
    cumulative = mc.get_cumulative()
    self.assertEqual(len(cumulative.counters), 10)
    self.assertEqual(len(cumulative.distributions), 10)
    self.assertEqual(set(dirty_values + clean_values),
                     set([v for _, v in cumulative.counters.items()]))

class Operation(object):
  """An operation representing the live version of a work item specification.

  An operation can have one or more outputs and for each output it can have
  one or more receiver operations that will take that as input.
  """

  def __init__(self, operation_name, spec, counter_factory, state_sampler):
    """Initializes a worker operation instance.

    Args:
      operation_name: The system name assigned by the runner for this
        operation.
      spec: A operation_specs.Worker* instance.
      counter_factory: The CounterFactory to use for our counters.
      state_sampler: The StateSampler for the current operation.
    """
    self.operation_name = operation_name
    self.spec = spec
    self.counter_factory = counter_factory
    self.consumers = collections.defaultdict(list)

    # These are overwritten in the legacy harness.
    self.step_name = operation_name
    self.metrics_container = MetricsContainer(self.step_name)
    self.scoped_metrics_container = ScopedMetricsContainer(
        self.metrics_container)

    self.state_sampler = state_sampler
    self.scoped_start_state = self.state_sampler.scoped_state(
        self.operation_name, 'start')
    self.scoped_process_state = self.state_sampler.scoped_state(
        self.operation_name, 'process')
    self.scoped_finish_state = self.state_sampler.scoped_state(
        self.operation_name, 'finish')
    # TODO(ccy): the '-abort' state can be added when the abort is supported in
    # Operations.
    self.receivers = []

  def start(self):
    """Start operation."""
    self.debug_logging_enabled = logging.getLogger().isEnabledFor(
        logging.DEBUG)
    # Everything except WorkerSideInputSource, which is not a
    # top-level operation, should have output_coders
    if getattr(self.spec, 'output_coders', None):
      self.receivers = [ConsumerSet(self.counter_factory, self.step_name,
                                    i, self.consumers[i], coder)
                        for i, coder in enumerate(self.spec.output_coders)]

  def finish(self):
    """Finish operation."""
    pass

  def process(self, o):
    """Process element in operation."""
    pass

  def output(self, windowed_value, output_index=0):
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)

  def add_receiver(self, operation, output_index=0):
    """Adds a receiver operation for the specified output."""
    self.consumers[output_index].append(operation)

  def progress_metrics(self):
    return beam_fn_api_pb2.Metrics.PTransform(
        processed_elements=beam_fn_api_pb2.Metrics.PTransform.ProcessedElements(
            measured=beam_fn_api_pb2.Metrics.PTransform.Measured(
                total_time_spent=(
                    self.scoped_start_state.sampled_seconds()
                    + self.scoped_process_state.sampled_seconds()
                    + self.scoped_finish_state.sampled_seconds()),
                # Multi-output operations should override this.
                output_element_counts=(
                    # If there is exactly one output, we can unambiguously
                    # fix its name later, which we do.
                    # TODO(robertwb): Plumb the actual name here.
                    {'ONLY_OUTPUT': self.receivers[0].opcounter
                                    .element_counter.value()}
                    if len(self.receivers) == 1
                    else None))),
        user=self.metrics_container.to_runner_api())

  def __str__(self):
    """Generates a useful string for this object.

    Compactly displays interesting fields.  In particular, pickled
    fields are not displayed.  Note that we collapse the fields of the
    contained Worker* object into this object, since there is a 1-1
    mapping between Operation and operation_specs.Worker*.

    Returns:
      Compact string representing this object.
    """
    return self.str_internal()

  def str_internal(self, is_recursive=False):
    """Internal helper for __str__ that supports recursion.

#.........这里部分代码省略.........

 def test_add_to_counter(self):
   mc = MetricsContainer('astep')
   counter = mc.get_counter(MetricName('namespace', 'name'))
   counter.inc()
   counter = mc.get_counter(MetricName('namespace', 'name'))
   self.assertEqual(counter.value, 1)

 def test_create_new_counter(self):
   mc = MetricsContainer('astep')
   self.assertFalse(MetricName('namespace', 'name') in mc.counters)
   mc.get_counter(MetricName('namespace', 'name'))
   self.assertTrue(MetricName('namespace', 'name') in mc.counters)

class Operation(object):
  """An operation representing the live version of a work item specification.

  An operation can have one or more outputs and for each output it can have
  one or more receiver operations that will take that as input.
  """

  def __init__(self, name_context, spec, counter_factory, state_sampler):
    """Initializes a worker operation instance.

    Args:
      name_context: A NameContext instance or string(deprecated), with the
        name information for this operation.
      spec: A operation_specs.Worker* instance.
      counter_factory: The CounterFactory to use for our counters.
      state_sampler: The StateSampler for the current operation.
    """
    if isinstance(name_context, common.NameContext):
      # TODO(BEAM-4028): Clean this up once it's completely migrated.
      # We use the specific operation name that is used for metrics and state
      # sampling.
      self.name_context = name_context
    else:
      self.name_context = common.NameContext(name_context)

    self.spec = spec
    self.counter_factory = counter_factory
    self.execution_context = None
    self.consumers = collections.defaultdict(list)

    # These are overwritten in the legacy harness.
    self.metrics_container = MetricsContainer(self.name_context.metrics_name())

    self.state_sampler = state_sampler
    self.scoped_start_state = self.state_sampler.scoped_state(
        self.name_context, 'start', metrics_container=self.metrics_container)
    self.scoped_process_state = self.state_sampler.scoped_state(
        self.name_context, 'process', metrics_container=self.metrics_container)
    self.scoped_finish_state = self.state_sampler.scoped_state(
        self.name_context, 'finish', metrics_container=self.metrics_container)
    # TODO(ccy): the '-abort' state can be added when the abort is supported in
    # Operations.
    self.receivers = []
    # Legacy workers cannot call setup() until after setting additional state
    # on the operation.
    self.setup_done = False

  def setup(self):
    with self.scoped_start_state:
      self.debug_logging_enabled = logging.getLogger().isEnabledFor(
          logging.DEBUG)
      # Everything except WorkerSideInputSource, which is not a
      # top-level operation, should have output_coders
      #TODO(pabloem): Define better what step name is used here.
      if getattr(self.spec, 'output_coders', None):
        self.receivers = [
            ConsumerSet.create(
                self.counter_factory,
                self.name_context.logging_name(),
                i,
                self.consumers[i], coder)
            for i, coder in enumerate(self.spec.output_coders)]
    self.setup_done = True

  def start(self):
    """Start operation."""
    if not self.setup_done:
      # For legacy workers.
      self.setup()

  def process(self, o):
    """Process element in operation."""
    pass

  def try_split(self, fraction_of_remainder):
    return None

  def finish(self):
    """Finish operation."""
    pass

  def reset(self):
    self.metrics_container.reset()

  def output(self, windowed_value, output_index=0):
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)

  def add_receiver(self, operation, output_index=0):
    """Adds a receiver operation for the specified output."""
    self.consumers[output_index].append(operation)

  def progress_metrics(self):
    return beam_fn_api_pb2.Metrics.PTransform(
        processed_elements=beam_fn_api_pb2.Metrics.PTransform.ProcessedElements(
            measured=beam_fn_api_pb2.Metrics.PTransform.Measured(
                total_time_spent=(
                    self.scoped_start_state.sampled_seconds()
                    + self.scoped_process_state.sampled_seconds()
                    + self.scoped_finish_state.sampled_seconds()),
                # Multi-output operations should override this.
#.........这里部分代码省略.........