Python run_tracker.RunTracker类代码示例

def clean_global_runtime_state():
  """Resets the global runtime state of a pants runtime for cleaner forking."""
  # Reset RunTracker state.
  RunTracker.global_instance().reset(reset_options=False)

  # Reset Goals and Tasks.
  Goal.clear()

  def pre_fork(self):
    """Pre-fork() callback for ProcessManager.daemonize()."""
    for service in self._services:
      service.pre_fork()

    # Teardown the RunTracker's SubprocPool pre-fork.
    RunTracker.global_instance().shutdown_worker_pool()

  def test_write_stats_to_json_file(self):
    # Set up
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    # Execute & verify
    with temporary_file_path() as file_name:
      RunTracker.write_stats_to_json(file_name, stats)
      with open(file_name, 'r') as f:
        result = json.load(f)
        self.assertEqual(stats, result)

  def _clean_runtime_state(self):
    """Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
    # TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
    # pants runs to improve testability and avoid potential bitrot.

    # Reset RunTracker state.
    RunTracker.global_instance().reset(reset_options=False)

    # Reset Goals and Tasks.
    Goal.clear()

  def _execute_engine(self):
    engine = RoundEngine()
    sorted_goal_infos = engine.sort_goals(self._context, self._goals)
    RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)
    result = engine.execute(self._context, self._goals)

    if self._context.invalidation_report:
      self._context.invalidation_report.report()

    return result

  def setup(self):
    options_bootstrapper = OptionsBootstrapper()

    # Force config into the cache so we (and plugin/backend loading code) can use it.
    # TODO: Plumb options in explicitly.
    options_bootstrapper.get_bootstrap_options()
    self.config = Config.from_cache()

    # Add any extra paths to python path (eg for loading extra source backends)
    extra_paths = self.config.getlist('backends', 'python-path', [])
    if extra_paths:
      sys.path.extend(extra_paths)

    # Load plugins and backends.
    backend_packages = self.config.getlist('backends', 'packages', [])
    plugins = self.config.getlist('backends', 'plugins', [])
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []
    known_scopes = ['']
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
    self.register_options()

    self.run_tracker = RunTracker.from_config(self.config)
    report = initial_reporting(self.config, self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      # construct base parameters to be filled in for BuildGraph
      for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
        build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
        # TODO(pl): This is an unfortunate interface leak, but I don't think
        # in the long run that we should be relying on "bootstrap" BUILD files
        # that do nothing except modify global state.  That type of behavior
        # (e.g. source roots, goal registration) should instead happen in
        # project plugins, or specialized configuration files.
        self.build_file_parser.parse_build_file_family(build_file)

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()

    self._expand_goals_and_specs()

  def test_upload_stats(self):
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
      def do_POST(handler):
        try:
          self.assertEquals('/upload', handler.path)
          self.assertEquals('application/x-www-form-urlencoded', handler.headers['Content-type'])
          length = int(handler.headers['Content-Length'])
          post_data = urlparse.parse_qs(handler.rfile.read(length).decode('utf-8'))
          decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
          self.assertEquals(stats, decoded_post_data)
          handler.send_response(200)
        except Exception:
          handler.send_response(400)  # Ensure the main thread knows the test failed.
          raise


    server_address = ('', 0)
    server = BaseHTTPServer.HTTPServer(server_address, Handler)
    host, port = server.server_address

    server_thread = threading.Thread(target=server.serve_forever)
    server_thread.daemon = True
    server_thread.start()

    self.assertTrue(RunTracker.post_stats('http://{}:{}/upload'.format(host, port), stats))

    server.shutdown()
    server.server_close()

  def _run(self):
    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      engine_result = self._maybe_run_v2()
      goal_runner_result = self._maybe_run_v1(run_tracker, reporting)

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    final_exit_code = self._compute_final_exit_code(
      engine_result,
      goal_runner_result,
      run_tracker_result
    )
    self._exiter.exit(final_exit_code)

  def _run(self):
    # Bootstrap options and logging.
    options_bootstrapper = self._options_bootstrapper or OptionsBootstrapper(env=self._env,
                                                                             args=self._args)
    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
    setup_logging_from_options(bootstrap_options)
    build_config = BuildConfigInitializer.get(options_bootstrapper)
    options = OptionsInitializer.create(options_bootstrapper, build_config)
    global_options = options.for_global_scope()

    # Apply exiter options.
    self._exiter.apply_options(options)

    # Option values are usually computed lazily on demand,
    # but command line options are eagerly computed for validation.
    for scope in options.scope_to_flags.keys():
      options.for_scope(scope)

    # Verify the configs here.
    if global_options.verify_config:
      options_bootstrapper.verify_configs_against_options(options)

    # Launch RunTracker as early as possible (just after Subsystem options are initialized).
    run_tracker = RunTracker.global_instance()
    reporting = Reporting.global_instance()
    reporting.initialize(run_tracker, self._run_start_time)

    try:
      # Determine the build root dir.
      root_dir = get_buildroot()

      # Capture a repro of the 'before' state for this build, if needed.
      repro = Reproducer.global_instance().create_repro()
      if repro:
        repro.capture(run_tracker.run_info.get_as_dict())

      # Setup and run GoalRunner.
      goal_runner = GoalRunner.Factory(root_dir,
                                       options,
                                       build_config,
                                       run_tracker,
                                       reporting,
                                       self._target_roots,
                                       self._daemon_build_graph,
                                       self._exiter).setup()

      goal_runner_result = goal_runner.run()

      if repro:
        # TODO: Have Repro capture the 'after' state (as a diff) as well?
        repro.log_location_of_repro_file()
    finally:
      run_tracker_result = run_tracker.end()

    # Take the exit code with higher abs value in case of negative values.
    final_exit_code = goal_runner_result if abs(goal_runner_result) > abs(run_tracker_result) else run_tracker_result
    self._exiter.exit(final_exit_code)

  def register_options(self):
    # Add a 'bootstrap' attribute to the register function, so that register_global can
    # access the bootstrap option values.
    def register_global(*args, **kwargs):
      return self.options.register_global(*args, **kwargs)
    register_global.bootstrap = self.options.bootstrap_option_values()
    register_global_options(register_global)

    # This is the first case we have of non-task, non-global options.
    # The current implementation special-cases RunTracker, and is temporary.
    # In the near future it will be replaced with a 'Subsystem' abstraction.
    # But for now this is useful for kicking the tires.
    def register_run_tracker(*args, **kwargs):
      self.options.register('run-tracker', *args, **kwargs)
    RunTracker.register_options(register_run_tracker)

    for goal in Goal.all():
      goal.register_options(self.options)

def clean_global_runtime_state(reset_runtracker=True, reset_subsystem=False):
  """Resets the global runtime state of a pants runtime for cleaner forking.

  :param bool reset_runtracker: Whether or not to clean RunTracker global state.
  :param bool reset_subsystem: Whether or not to clean Subsystem global state.
  """
  if reset_runtracker:
    # Reset RunTracker state.
    RunTracker.global_instance().reset(reset_options=False)

  if reset_subsystem:
    # Reset subsystem state.
    Subsystem.reset()

  # Reset Goals and Tasks.
  Goal.clear()

  # Reset backend/plugins state.
  OptionsInitializer.reset()

  def test_upload_stats(self):
    stats = {'stats': {'foo': 'bar', 'baz': 42}}

    class Handler(http.server.BaseHTTPRequestHandler):
      def do_POST(handler):
        try:
          if handler.path.startswith('/redirect'):
            code = int(handler.path[-3:])
            handler.send_response(code)
            handler.send_header('location', mk_url('/upload'))
            handler.end_headers()
          else:
            self.assertEqual('/upload', handler.path)
            self.assertEqual('application/x-www-form-urlencoded', handler.headers['Content-type'])
            length = int(handler.headers['Content-Length'])
            post_data = parse_qs(handler.rfile.read(length).decode('utf-8'))
            decoded_post_data = {k: json.loads(v[0]) for k, v in post_data.items()}
            self.assertEqual(stats, decoded_post_data)
            handler.send_response(200)
            handler.end_headers()
        except Exception:
          handler.send_response(400)  # Ensure the main thread knows the test failed.
          raise

    server_address = ('', 0)
    server = http.server.HTTPServer(server_address, Handler)
    host, port = server.server_address

    def mk_url(path):
      return 'http://{}:{}{}'.format(host, port, path)

    server_thread = threading.Thread(target=server.serve_forever)
    server_thread.daemon = True
    server_thread.start()

    self.context(for_subsystems=[Cookies])
    self.assertTrue(RunTracker.post_stats(mk_url('/upload'), stats))
    self.assertTrue(RunTracker.post_stats(mk_url('/redirect307'), stats))
    self.assertFalse(RunTracker.post_stats(mk_url('/redirect302'), stats))

    server.shutdown()
    server.server_close()

def create_run_tracker(info_dir=None):
  """Creates a ``RunTracker`` and starts it.

  :param string info_dir: An optional director for the run tracker to store state; defaults to a
    new temp dir that will be be cleaned up on interpreter exit.
  """
  # TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
  # disciplined manner
  info_dir = info_dir or safe_mkdtemp()
  run_tracker = RunTracker(info_dir)
  report = Report()
  settings = PlainTextReporter.Settings(outfile=sys.stdout,
                                        log_level=Report.INFO,
                                        color=False,
                                        indent=True,
                                        timing=False,
                                        cache_stats=False)
  report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
  run_tracker.start(report)
  return run_tracker

  def test_create_dict_with_nested_keys_and_val(self):
    keys = []

    with self.assertRaises(ValueError):
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something')

    keys += ['one']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': 'something'}
    )

    keys += ['two']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': 'something'}}
    )

    keys += ['three']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': 'something'}}}
    )

    keys += ['four']
    self.assertEquals(
      RunTracker._create_dict_with_nested_keys_and_val(keys, 'something'),
      {'one': {'two': {'three': {'four': 'something'}}}}
    )

  def _execute_engine(self):
    workdir = self._context.options.for_global_scope().pants_workdir
    if not workdir.endswith('.pants.d'):
      self._context.log.error('Pants working directory should end with \'.pants.d\', currently it is {}\n'
                              .format(workdir))
      return 1

    unknown_goals = [goal.name for goal in self._goals if not goal.ordered_task_names()]
    if unknown_goals:
      self._context.log.error('Unknown goal(s): {}\n'.format(' '.join(unknown_goals)))
      return 1

    engine = RoundEngine()

    sorted_goal_infos = engine.sort_goals(self._context, self._goals)
    RunTracker.global_instance().set_sorted_goal_infos(sorted_goal_infos)

    result = engine.execute(self._context, self._goals)

    if self._context.invalidation_report:
      self._context.invalidation_report.report()

    return result