Python round_engine.RoundEngine类代码示例

  def _do_run(self):
    # Update the reporting settings, now that we have flags etc.
    def is_quiet_task():
      for goal in self.goals:
        if goal.has_task_of_type(QuietTaskMixin):
          return True
      return False

    is_explain = self.global_options.explain
    update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)

    context = Context(
      config=self.config,
      options=self.options,
      run_tracker=self.run_tracker,
      target_roots=self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      address_mapper=self.address_mapper,
      spec_excludes=self.spec_excludes
    )

    unknown = []
    for goal in self.goals:
      if not goal.ordered_task_names():
        unknown.append(goal)

    if unknown:
      context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
      return 1

    engine = RoundEngine()
    return engine.execute(context, self.goals)

  def _execute_engine(self):
    engine = RoundEngine()
    sorted_goal_infos = engine.sort_goals(self._context, self._goals)
    RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
    result = engine.execute(self._context, self._goals)

    if self._context.invalidation_report:
      self._context.invalidation_report.report()

    return result

  def _execute_engine(self):
    unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
    if unknown_goals:
      self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
      return 1

    engine = RoundEngine()
    result = engine.execute(self._context, self._goals)

    if self._invalidation_report:
      self._invalidation_report.report()

    return result

  def run(self, lock):
    # TODO(John Sirois): Consider moving to straight python logging.  The divide between the
    # context/work-unit logging and standard python logging doesn't buy us anything.

    # Enable standard python logging for code with no handle to a context/work-unit.
    if self.options.log_level:
      LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
      logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)
        log.init('goals')
      else:
        log.init()

    # Update the reporting settings, now that we have flags etc.
    def is_console_task():
      for phase in self.phases:
        for goal in phase.goals():
          if issubclass(goal.task_type, ConsoleTask):
            return True
      return False

    is_explain = self.options.explain
    update_reporting(self.options, is_console_task() or is_explain, self.run_tracker)

    context = Context(
      self.config,
      self.options,
      self.run_tracker,
      self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      lock=lock)

    unknown = []
    for phase in self.phases:
      if not phase.goals():
        unknown.append(phase)

    if unknown:
      context.log.error('Unknown goal(s): %s\n' % ' '.join(phase.name for phase in unknown))
      return 1

    engine = RoundEngine()
    return engine.execute(context, self.phases)

  def setUp(self):
    super(RoundEngineTest, self).setUp()

    self._context = self.context()
    self.assertTrue(self._context.is_unlocked())

    self.engine = RoundEngine()
    self.actions = []

  def setUp(self):
    super(RoundEngineTest, self).setUp()

    self.set_new_options_for_scope('', explain=False)
    self._context = self.context()
    self.assertTrue(self._context.is_unlocked())

    self.engine = RoundEngine()
    self.actions = []

  def _do_run(self):
    # Update the reporting settings, now that we have flags etc.
    def is_quiet_task():
      for goal in self.goals:
        if goal.has_task_of_type(QuietTaskMixin):
          return True
      return False

    is_explain = self.global_options.explain
    if self.reporting.global_instance().get_options().invalidation_report:
      invalidation_report = InvalidationReport()
    else:
      invalidation_report = None
    self.reporting.update_reporting(self.global_options,
                                    is_quiet_task() or is_explain,
                                    self.run_tracker,
                                    invalidation_report=invalidation_report)

    context = Context(
      options=self.options,
      run_tracker=self.run_tracker,
      target_roots=self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      address_mapper=self.address_mapper,
      spec_excludes=self.spec_excludes,
      invalidation_report=invalidation_report
    )

    unknown = []
    for goal in self.goals:
      if not goal.ordered_task_names():
        unknown.append(goal)

    if unknown:
      context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown)))
      return 1

    engine = RoundEngine()
    result = engine.execute(context, self.goals)
    if invalidation_report:
      invalidation_report.report()
    return result

  def _execute_engine(self):
    workdir = self._context.options.for_global_scope().pants_workdir
    if not workdir.endswith('.pants.d'):
      self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n'
                              .format(workdir))
      return 1

    unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
    if unknown_goals:
      self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
      return 1

    engine = RoundEngine()
    result = engine.execute(self._context, self._goals)

    if self._invalidation_report:
      self._invalidation_report.report()

    return result

  def setUp(self):
    super(RoundEngineTest, self).setUp()

    self.set_options_for_scope('', explain=False)
    for outer in ['goal1', 'goal2', 'goal3', 'goal4', 'goal5']:
      for inner in ['task1', 'task2', 'task3', 'task4', 'task5']:
        self.set_options_for_scope('{}.{}'.format(outer, inner),
                                   level='info', colors=False)

    self.engine = RoundEngine()
    self.actions = []

class RoundEngineTest(EngineTestBase, TestBase):
  def setUp(self):
    super(RoundEngineTest, self).setUp()

    self.set_options_for_scope('', explain=False)
    for outer in ['goal1', 'goal2', 'goal3', 'goal4', 'goal5']:
      for inner in ['task1', 'task2', 'task3', 'task4', 'task5']:
        self.set_options_for_scope('{}.{}'.format(outer, inner),
                                   level='info', colors=False)

    self.engine = RoundEngine()
    self.actions = []
    self._context = None

  def tearDown(self):
    if self._context is not None:
      self.assertTrue(not self._context or self._context.is_unlocked())
    super(RoundEngineTest, self).tearDown()

  def alternate_target_roots_action(self, tag):
    return 'alternate_target_roots', tag, self._context

  def prepare_action(self, tag):
    return 'prepare', tag, self._context

  def execute_action(self, tag):
    return 'execute', tag, self._context

  def construct_action(self, tag):
    return 'construct', tag, self._context

  def record(self, tag, product_types=None, required_data=None, optional_data=None,
             alternate_target_roots=None):

    class RecordingTask(Task):
      options_scope = tag

      @classmethod
      def product_types(cls):
        return product_types or []

      @classmethod
      def alternate_target_roots(cls, options, address_mapper, build_graph):
        self.actions.append(self.alternate_target_roots_action(tag))
        return alternate_target_roots

      @classmethod
      def prepare(cls, options, round_manager):
        for product in (required_data or ()):
          round_manager.require_data(product)
        for product in (optional_data or ()):
          round_manager.optional_data(product)
        self.actions.append(self.prepare_action(tag))

      def __init__(me, *args, **kwargs):
        super(RecordingTask, me).__init__(*args, **kwargs)
        self.actions.append(self.construct_action(tag))

      def execute(me):
        self.actions.append(self.execute_action(tag))

    return RecordingTask

  def install_task(self, name, product_types=None, goal=None, required_data=None,
                   optional_data=None, alternate_target_roots=None):
    """Install a task to goal and return all installed tasks of the goal.

    This is needed to initialize tasks' context.
    """
    task_type = self.record(name, product_types, required_data, optional_data,
                            alternate_target_roots)
    return super(RoundEngineTest,
                 self).install_task(name=name, action=task_type, goal=goal).task_types()

  def create_context(self, for_task_types=None, target_roots=None):
    self._context = self.context(for_task_types=for_task_types, target_roots=target_roots)
    self.assertTrue(self._context.is_unlocked())

  def assert_actions(self, *expected_execute_ordering):
    expected_pre_execute_actions = set()
    expected_execute_actions = []
    for action in expected_execute_ordering:
      expected_pre_execute_actions.add(self.alternate_target_roots_action(action))
      expected_pre_execute_actions.add(self.prepare_action(action))
      expected_execute_actions.append(self.construct_action(action))
      expected_execute_actions.append(self.execute_action(action))

    expeceted_execute_actions_length = len(expected_execute_ordering) * 2
    self.assertEqual(expected_pre_execute_actions,
                     set(self.actions[:-expeceted_execute_actions_length]))
    self.assertEqual(expected_execute_actions, self.actions[-expeceted_execute_actions_length:])

  def test_lifecycle_ordering(self):
    task1 = self.install_task('task1', goal='goal1', product_types=['1'])
    task2 = self.install_task('task2', goal='goal1', product_types=['2'], required_data=['1'])
    task3 = self.install_task('task3', goal='goal3', product_types=['3'], required_data=['2'])
    task4 = self.install_task('task4', goal='goal4', required_data=['1', '2', '3'])
    self.create_context(for_task_types=task1+task2+task3+task4)
    self.engine.attempt(self._context, self.as_goals('goal4'))
    self.assert_actions('task1', 'task2', 'task3', 'task4')
#.........这里部分代码省略.........

class RoundEngineTest(EngineTestBase, BaseTest):
  def setUp(self):
    super(RoundEngineTest, self).setUp()

    self.set_new_options_for_scope('', explain=False)
    self._context = self.context()
    self.assertTrue(self._context.is_unlocked())

    self.engine = RoundEngine()
    self.actions = []

  def tearDown(self):
    self.assertTrue(self._context.is_unlocked())
    super(RoundEngineTest, self).tearDown()

  def construct_action(self, tag):
    return 'construct', tag, self._context

  def prepare_action(self, tag):
    return 'prepare', tag, self._context

  def execute_action(self, tag):
    return 'execute', tag, self._context

  def record(self, tag, product_types=None, required_data=None):
    class RecordingTask(Task):
      def __init__(me, *args, **kwargs):
        super(RecordingTask, me).__init__(*args, **kwargs)
        self.actions.append(self.construct_action(tag))

      @classmethod
      def product_types(cls):
        return product_types or []

      def prepare(me, round_manager):
        for requirement in (required_data or ()):
          round_manager.require_data(requirement)
        self.actions.append(self.prepare_action(tag))

      def execute(me):
        self.actions.append(self.execute_action(tag))

    return RecordingTask

  def install_task(self, name, product_types=None, goal=None, required_data=None):
    task = self.record(name, product_types, required_data)
    return super(RoundEngineTest, self).install_task(name=name, action=task, goal=goal)

  def assert_actions(self, *expected_execute_ordering):
    expected_pre_execute_actions = set()
    expected_execute_actions = []
    for action in expected_execute_ordering:
      expected_pre_execute_actions.add(self.construct_action(action))
      expected_pre_execute_actions.add(self.prepare_action(action))
      expected_execute_actions.append(self.execute_action(action))

    self.assertEqual(expected_pre_execute_actions,
                     set(self.actions[:-len(expected_execute_ordering)]))
    self.assertEqual(expected_execute_actions, self.actions[-len(expected_execute_ordering):])

  def test_lifecycle_ordering(self):
    self.install_task('task1', goal='goal1', product_types=['1'])
    self.install_task('task2', goal='goal1', product_types=['2'], required_data=['1'])
    self.install_task('task3', goal='goal3', product_types=['3'], required_data=['2'])
    self.install_task('task4', goal='goal4', required_data=['1', '2', '3'])

    self.engine.attempt(self._context, self.as_goals('goal4'))

    self.assert_actions('task1', 'task2', 'task3', 'task4')

  def test_lifecycle_ordering_install_order_invariant(self):
    # Here we swap the order of goal3 and goal4 task installation from the order in
    # `test_lifecycle_ordering` above.  We can't swap task1 and task2 since they purposefully
    # do have an implicit order dependence with a dep inside the same goal.
    self.install_task('task1', goal='goal1', product_types=['1'])
    self.install_task('task2', goal='goal1', product_types=['2'], required_data=['1'])
    self.install_task('task4', goal='goal4', required_data=['1', '2', '3'])
    self.install_task('task3', goal='goal3', product_types=['3'], required_data=['2'])

    self.engine.attempt(self._context, self.as_goals('goal4'))

    self.assert_actions('task1', 'task2', 'task3', 'task4')

  def test_inter_goal_dep(self):
    self.install_task('task1', goal='goal1', product_types=['1'])
    self.install_task('task2', goal='goal1', required_data=['1'])

    self.engine.attempt(self._context, self.as_goals('goal1'))

    self.assert_actions('task1', 'task2')

  def test_inter_goal_dep_self_cycle(self):
    self.install_task('task1', goal='goal1', product_types=['1'], required_data=['1'])

    with self.assertRaises(self.engine.TaskOrderError):
      self.engine.attempt(self._context, self.as_goals('goal1'))

  def test_inter_goal_dep_downstream(self):
    self.install_task('task1', goal='goal1', required_data=['1'])
    self.install_task('task2', goal='goal1', product_types=['1'])
#.........这里部分代码省略.........

    def run(self, lock):
        # TODO(John Sirois): Consider moving to straight python logging.  The divide between the
        # context/work-unit logging and standard python logging doesn't buy us anything.

        # Enable standard python logging for code with no handle to a context/work-unit.
        if self.options.log_level:
            LogOptions.set_stderr_log_level((self.options.log_level or "info").upper())
            logdir = self.options.logdir or self.config.get("goals", "logdir", default=None)
            if logdir:
                safe_mkdir(logdir)
                LogOptions.set_log_dir(logdir)
                log.init("goals")
            else:
                log.init()

        # Update the reporting settings, now that we have flags etc.
        def is_quiet_task():
            for goal in self.goals:
                if goal.has_task_of_type(QuietTaskMixin):
                    return True
            return False

        # Target specs are mapped to the patterns which match them, if any. This variable is a key for
        # specs which don't match any exclusion regexes. We know it won't already be in the list of
        # patterns, because the asterisks in its name make it an invalid regex.
        _UNMATCHED_KEY = "** unmatched **"

        def targets_by_pattern(targets, patterns):
            mapping = defaultdict(list)
            for target in targets:
                matched_pattern = None
                for pattern in patterns:
                    if re.search(pattern, target.address.spec) is not None:
                        matched_pattern = pattern
                        break
                if matched_pattern is None:
                    mapping[_UNMATCHED_KEY].append(target)
                else:
                    mapping[matched_pattern].append(target)
            return mapping

        is_explain = self.options.explain
        update_reporting(self.options, is_quiet_task() or is_explain, self.run_tracker)

        if self.options.target_excludes:
            excludes = self.options.target_excludes
            log.debug("excludes:\n  {excludes}".format(excludes="\n  ".join(excludes)))
            by_pattern = targets_by_pattern(self.targets, excludes)
            self.targets = by_pattern[_UNMATCHED_KEY]
            # The rest of this if-statement is just for debug logging.
            log.debug(
                "Targets after excludes: {targets}".format(targets=", ".join(t.address.spec for t in self.targets))
            )
            excluded_count = sum(len(by_pattern[p]) for p in excludes)
            log.debug(
                "Excluded {count} target{plural}.".format(
                    count=excluded_count, plural=("s" if excluded_count != 1 else "")
                )
            )
            for pattern in excludes:
                log.debug(
                    "Targets excluded by pattern {pattern}\n  {targets}".format(
                        pattern=pattern, targets="\n  ".join(t.address.spec for t in by_pattern[pattern])
                    )
                )

        context = Context(
            config=self.config,
            options=self.options,
            run_tracker=self.run_tracker,
            target_roots=self.targets,
            requested_goals=self.requested_goals,
            build_graph=self.build_graph,
            build_file_parser=self.build_file_parser,
            address_mapper=self.address_mapper,
            lock=lock,
        )

        unknown = []
        for goal in self.goals:
            if not goal.ordered_task_names():
                unknown.append(goal)

        if unknown:
            context.log.error("Unknown goal(s): %s\n" % " ".join(goal.name for goal in unknown))
            return 1

        engine = RoundEngine()
        return engine.execute(context, self.goals)

  def _do_run(self):
    # TODO(John Sirois): Consider moving to straight python logging.  The divide between the
    # context/work-unit logging and standard python logging doesn't buy us anything.

    # TODO(Eric Ayers) We are missing log messages. Set the log level earlier
    # Enable standard python logging for code with no handle to a context/work-unit.
    if self.global_options.level:
      LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper())
      logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)

        prev_log_level = None
        # If quiet, temporarily change stderr log level to kill init's output.
        if self.global_options.quiet:
          prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level())
          # loglevel_name can fail, so only change level if we were able to get the current one.
          if prev_log_level is not None:
            LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY)

        log.init('goals')

        if prev_log_level is not None:
          LogOptions.set_stderr_log_level(prev_log_level)
      else:
        log.init()

    # Update the reporting settings, now that we have flags etc.
    def is_quiet_task():
      for goal in self.goals:
        if goal.has_task_of_type(QuietTaskMixin):
          return True
      return False

    is_explain = self.global_options.explain
    update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)

    context = Context(
      config=self.config,
      options=self.options,
      run_tracker=self.run_tracker,
      target_roots=self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      address_mapper=self.address_mapper,
      spec_excludes=self.get_spec_excludes()
    )

    unknown = []
    for goal in self.goals:
      if not goal.ordered_task_names():
        unknown.append(goal)

    if unknown:
      context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
      return 1

    engine = RoundEngine()
    return engine.execute(context, self.goals)

  def run(self):
    # TODO(John Sirois): Consider moving to straight python logging.  The divide between the
    # context/work-unit logging and standard python logging doesn't buy us anything.

    # Enable standard python logging for code with no handle to a context/work-unit.
    if self.global_options.level:
      LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper())
      logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)

        prev_log_level = None
        # If quiet, temporarily change stderr log level to kill init's output.
        if self.global_options.quiet:
          prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level())
          # loglevel_name can fail, so only change level if we were able to get the current one.
          if prev_log_level is not None:
            LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY)

        log.init('goals')

        if prev_log_level is not None:
          LogOptions.set_stderr_log_level(prev_log_level)
      else:
        log.init()

    # Update the reporting settings, now that we have flags etc.
    def is_quiet_task():
      for goal in self.goals:
        if goal.has_task_of_type(QuietTaskMixin):
          return True
      return False

    # Target specs are mapped to the patterns which match them, if any. This variable is a key for
    # specs which don't match any exclusion regexes. We know it won't already be in the list of
    # patterns, because the asterisks in its name make it an invalid regex.
    _UNMATCHED_KEY = '** unmatched **'

    def targets_by_pattern(targets, patterns):
      mapping = defaultdict(list)
      for target in targets:
        matched_pattern = None
        for pattern in patterns:
          if re.search(pattern, target.address.spec) is not None:
            matched_pattern = pattern
            break
        if matched_pattern is None:
          mapping[_UNMATCHED_KEY].append(target)
        else:
          mapping[matched_pattern].append(target)
      return mapping

    is_explain = self.global_options.explain
    update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)

    if self.global_options.exclude_target_regexp:
      excludes = self.global_options.exclude_target_regexp
      log.debug('excludes:\n  {excludes}'.format(excludes='\n  '.join(excludes)))
      by_pattern = targets_by_pattern(self.targets, excludes)
      self.targets = by_pattern[_UNMATCHED_KEY]
      # The rest of this if-statement is just for debug logging.
      log.debug('Targets after excludes: {targets}'.format(
          targets=', '.join(t.address.spec for t in self.targets)))
      excluded_count = sum(len(by_pattern[p]) for p in excludes)
      log.debug('Excluded {count} target{plural}.'.format(count=excluded_count,
          plural=('s' if excluded_count != 1 else '')))
      for pattern in excludes:
        log.debug('Targets excluded by pattern {pattern}\n  {targets}'.format(pattern=pattern,
            targets='\n  '.join(t.address.spec for t in by_pattern[pattern])))

    context = Context(
      config=self.config,
      new_options=self.new_options,
      run_tracker=self.run_tracker,
      target_roots=self.targets,
      requested_goals=self.requested_goals,
      build_graph=self.build_graph,
      build_file_parser=self.build_file_parser,
      address_mapper=self.address_mapper,
      spec_excludes=self.get_spec_excludes()
    )

    unknown = []
    for goal in self.goals:
      if not goal.ordered_task_names():
        unknown.append(goal)

    if unknown:
      context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
      return 1

    engine = RoundEngine()
    return engine.execute(context, self.goals)