Python options_bootstrapper.OptionsBootstrapper类代码示例

  def test_create_bootstrapped_options(self):
    # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate
    # correctly into regular config.
    with temporary_file() as fp:
      fp.write(dedent("""
      [foo]
      bar: %(pants_workdir)s/baz

      [fruit]
      apple: %(pants_supportdir)s/banana
      """))
      fp.close()
      bootstrapper = OptionsBootstrapper(env={
                                           'PANTS_SUPPORTDIR': '/pear'
                                         },
                                         configpath=fp.name,
                                         args=['--pants-workdir=/qux'])
      opts = bootstrapper.get_full_options(known_scope_infos=[
        ScopeInfo('', ScopeInfo.GLOBAL),
        ScopeInfo('foo', ScopeInfo.TASK),
        ScopeInfo('fruit', ScopeInfo.TASK)
      ])
      opts.register('', '--pants-workdir')  # So we don't choke on it on the cmd line.
      opts.register('foo', '--bar')
      opts.register('fruit', '--apple')
    self.assertEquals('/qux/baz', opts.for_scope('foo').bar)
    self.assertEquals('/pear/banana', opts.for_scope('fruit').apple)

    def test_create_bootstrapped_options(self):
        # Check that we can set a bootstrap option from a cmd-line flag and have that interpolate
        # correctly into regular config.
        with temporary_file() as fp:
            fp.write(
                dedent(
                    """
      [foo]
      bar: %(pants_workdir)s/baz

      [fruit]
      apple: %(pants_supportdir)s/banana
      """
                )
            )
            fp.close()
            args = ["--pants-workdir=/qux"] + self._config_path(fp.name)
            bootstrapper = OptionsBootstrapper(env={"PANTS_SUPPORTDIR": "/pear"}, args=args)
            opts = bootstrapper.get_full_options(
                known_scope_infos=[
                    ScopeInfo("", ScopeInfo.GLOBAL),
                    ScopeInfo("foo", ScopeInfo.TASK),
                    ScopeInfo("fruit", ScopeInfo.TASK),
                ]
            )
            # So we don't choke on these on the cmd line.
            opts.register("", "--pants-workdir")
            opts.register("", "--pants-config-files")

            opts.register("foo", "--bar")
            opts.register("fruit", "--apple")
        self.assertEquals("/qux/baz", opts.for_scope("foo").bar)
        self.assertEquals("/pear/banana", opts.for_scope("fruit").apple)

  def setup(self):
    options_bootstrapper = OptionsBootstrapper()

    # Force config into the cache so we (and plugin/backend loading code) can use it.
    # TODO: Plumb options in explicitly.
    options_bootstrapper.get_bootstrap_options()
    self.config = Config.from_cache()

    # Add any extra paths to python path (eg for loading extra source backends)
    extra_paths = self.config.getlist('backends', 'python-path', [])
    if extra_paths:
      sys.path.extend(extra_paths)

    # Load plugins and backends.
    backend_packages = self.config.getlist('backends', 'packages', [])
    plugins = self.config.getlist('backends', 'plugins', [])
    build_configuration = load_plugins_and_backends(plugins, backend_packages)

    # Now that plugins and backends are loaded, we can gather the known scopes.
    self.targets = []
    known_scopes = ['']
    for goal in Goal.all():
      # Note that enclosing scopes will appear before scopes they enclose.
      known_scopes.extend(filter(None, goal.known_scopes()))

    # Now that we have the known scopes we can get the full options.
    self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
    self.register_options()

    self.run_tracker = RunTracker.from_config(self.config)
    report = initial_reporting(self.config, self.run_tracker)
    self.run_tracker.start(report)
    url = self.run_tracker.run_info.get_info('report_url')
    if url:
      self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
    else:
      self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')

    self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
                                             root_dir=self.root_dir,
                                             run_tracker=self.run_tracker)
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
    self.build_graph = BuildGraph(run_tracker=self.run_tracker,
                                  address_mapper=self.address_mapper)

    with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
      # construct base parameters to be filled in for BuildGraph
      for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
        build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
        # TODO(pl): This is an unfortunate interface leak, but I don't think
        # in the long run that we should be relying on "bootstrap" BUILD files
        # that do nothing except modify global state.  That type of behavior
        # (e.g. source roots, goal registration) should instead happen in
        # project plugins, or specialized configuration files.
        self.build_file_parser.parse_build_file_family(build_file)

    # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
    self.run_tracker.run_info.add_scm_info()

    self._expand_goals_and_specs()

  def run(self):
    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
    global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()

    return self._run(global_bootstrap_options.enable_pantsd,
                     exiter=self._exiter,
                     args=self._args,
                     env=self._env,
                     options_bootstrapper=options_bootstrapper)

  def test_create_bootstrapped_multiple_config_override(self):
    # check with multiple config files, the latest values always get taken
    # in this case worker_count will be overwritten, while fruit stays the same
    with temporary_file() as fp:
      fp.write(dedent("""
      [compile.apt]
      worker_count: 1

      [fruit]
      apple: red
      """))
      fp.close()

      args = ['--config-override={}'.format(fp.name)] + self._config_path(fp.name)
      bootstrapper_single_config = OptionsBootstrapper(args=args)

      opts_single_config  = bootstrapper_single_config.get_full_options(known_scope_infos=[
          ScopeInfo('', ScopeInfo.GLOBAL),
          ScopeInfo('compile.apt', ScopeInfo.TASK),
          ScopeInfo('fruit', ScopeInfo.TASK),
      ])
      # So we don't choke on these on the cmd line.
      opts_single_config.register('', '--pants-config-files')
      opts_single_config.register('', '--config-override', type=list)

      opts_single_config.register('compile.apt', '--worker-count')
      opts_single_config.register('fruit', '--apple')

      self.assertEquals('1', opts_single_config.for_scope('compile.apt').worker_count)
      self.assertEquals('red', opts_single_config.for_scope('fruit').apple)

      with temporary_file() as fp2:
        fp2.write(dedent("""
        [compile.apt]
        worker_count: 2
        """))
        fp2.close()

        args = ['--config-override={}'.format(fp.name),
                '--config-override={}'.format(fp2.name)] + self._config_path(fp.name)

        bootstrapper_double_config = OptionsBootstrapper(args=args)

        opts_double_config = bootstrapper_double_config.get_full_options(known_scope_infos=[
          ScopeInfo('', ScopeInfo.GLOBAL),
          ScopeInfo('compile.apt', ScopeInfo.TASK),
          ScopeInfo('fruit', ScopeInfo.TASK),
        ])
        # So we don't choke on these on the cmd line.
        opts_double_config.register('', '--pants-config-files')
        opts_double_config.register('', '--config-override', type=list)
        opts_double_config.register('compile.apt', '--worker-count')
        opts_double_config.register('fruit', '--apple')

        self.assertEquals('2', opts_double_config.for_scope('compile.apt').worker_count)
        self.assertEquals('red', opts_double_config.for_scope('fruit').apple)

  def run(self):
    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
    global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()

    return self._run(is_remote=global_bootstrap_options.enable_pantsd,
                     exiter=self._exiter,
                     args=self._args,
                     env=self._env,
                     process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
                     options_bootstrapper=options_bootstrapper)

  def prepare_task(self,
                   config=None,
                   args=None,
                   targets=None,
                   build_graph=None,
                   build_file_parser=None,
                   address_mapper=None,
                   console_outstream=None,
                   workspace=None):
    """Prepares a Task for execution.

    task_type: The class of the Task to create.
    config: An optional string representing the contents of a pants.ini config.
    args: optional list of command line flags, these should be prefixed with '--test-'.
    targets: optional list of Target objects passed on the command line.

    Returns a new Task ready to execute.
    """

    task_type = self.task_type()
    assert issubclass(task_type, Task), 'task_type must be a Task subclass, got %s' % task_type

    config = create_config(config or '')
    workdir = os.path.join(config.getdefault('pants_workdir'), 'test', task_type.__name__)

    bootstrap_options = OptionsBootstrapper().get_bootstrap_options()

    options = Options(env={}, config=config, known_scopes=['', 'test'], args=args or [])
    # A lot of basic code uses these options, so always register them.
    register_bootstrap_options(options.register_global)

    # We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod).
    def register_global_wrapper(*args, **kwargs):
      return options.register_global(*args, **kwargs)

    register_global_wrapper.bootstrap = bootstrap_options.for_global_scope()
    register_global_options(register_global_wrapper)

    task_type.options_scope = 'test'
    task_type.register_options_on_scope(options)

    run_tracker = create_run_tracker()

    context = Context(config,
                      options,
                      run_tracker,
                      targets or [],
                      build_graph=build_graph,
                      build_file_parser=build_file_parser,
                      address_mapper=address_mapper,
                      console_outstream=console_outstream,
                      workspace=workspace)
    return task_type(context, workdir)

  def _test_bootstrap_options(self, config, env, args, **expected_entries):
    with temporary_file() as fp:
      fp.write('[DEFAULT]\n')
      if config:
        for k, v in config.items():
          fp.write('{0}: {1}\n'.format(k, v))
      fp.close()

      bootstrapper = OptionsBootstrapper(env=env, configpath=fp.name, args=args)
      vals = bootstrapper.get_bootstrap_options().for_global_scope()

      vals_dict = {k: getattr(vals, k) for k in expected_entries}
      self.assertEquals(expected_entries, vals_dict)

    def test_setting_pants_config_in_config(self):
        # Test that setting pants_config in the config file has no effect.
        with temporary_dir() as tmpdir:
            config1 = os.path.join(tmpdir, "config1")
            config2 = os.path.join(tmpdir, "config2")
            with open(config1, "w") as out1:
                out1.write(b"[DEFAULT]\npants_config_files: ['{}']\nlogdir: logdir1\n".format(config2))
            with open(config2, "w") as out2:
                out2.write(b"[DEFAULT]\nlogdir: logdir2\n")

            ob = OptionsBootstrapper(env={}, args=["--pants-config-files=['{}']".format(config1)])
            logdir = ob.get_bootstrap_options().for_global_scope().logdir
            self.assertEqual("logdir1", logdir)

  def test_create_bootstrapped_multiple_config_override(self):
    # check with multiple config files, the latest values always get taken
    # in this case strategy will be overwritten, while fruit stays the same
    with temporary_file() as fp:
      fp.write(dedent("""
      [compile.apt]
      strategy: global

      [fruit]
      apple: red
      """))
      fp.close()

      bootstrapper_single_config = OptionsBootstrapper(configpath=fp.name,
                                                       args=['--config-override={}'.format(fp.name)])

      opts_single_config  = bootstrapper_single_config.get_full_options(known_scope_infos=[
          ScopeInfo('', ScopeInfo.GLOBAL),
          ScopeInfo('compile.apt', ScopeInfo.TASK),
          ScopeInfo('fruit', ScopeInfo.TASK),
      ])
      opts_single_config.register('', '--config-override')  # So we don't choke on it on the cmd line.
      opts_single_config.register('compile.apt', '--strategy')
      opts_single_config.register('fruit', '--apple')

      self.assertEquals('global', opts_single_config.for_scope('compile.apt').strategy)
      self.assertEquals('red', opts_single_config.for_scope('fruit').apple)

      with temporary_file() as fp2:
        fp2.write(dedent("""
        [compile.apt]
        strategy: isolated
        """))
        fp2.close()

        bootstrapper_double_config = OptionsBootstrapper(
            configpath=fp.name,
            args=['--config-override={}'.format(fp.name),
                  '--config-override={}'.format(fp2.name)])

        opts_double_config = bootstrapper_double_config.get_full_options(known_scope_infos=[
          ScopeInfo('', ScopeInfo.GLOBAL),
          ScopeInfo('compile.apt', ScopeInfo.TASK),
          ScopeInfo('fruit', ScopeInfo.TASK),
        ])
        opts_double_config.register('', '--config-override')  # So we don't choke on it on the cmd line.
        opts_double_config.register('compile.apt', '--strategy')
        opts_double_config.register('fruit', '--apple')

        self.assertEquals('isolated', opts_double_config.for_scope('compile.apt').strategy)
        self.assertEquals('red', opts_double_config.for_scope('fruit').apple)

 def test_file_spec_args(self):
   with tempfile.NamedTemporaryFile() as tmp:
     tmp.write(dedent(
       '''
       foo
       bar
       '''
     ))
     tmp.flush()
     cmdline = './pants --target-spec-file={filename} compile morx fleem'.format(filename=tmp.name)
     bootstrapper = OptionsBootstrapper(args=shlex.split(cmdline))
     bootstrap_options = bootstrapper.get_bootstrap_options().for_global_scope()
     options = self._parse(cmdline, bootstrap_option_values=bootstrap_options)
     sorted_specs = sorted(options.target_specs)
     self.assertEqual(['bar', 'fleem', 'foo', 'morx'], sorted_specs)

 def execute(self):
   try:
     pantsd = PantsDaemon.Factory.create(OptionsBootstrapper.create(), full_init=False)
     with pantsd.lifecycle_lock:
       pantsd.terminate()
   except ProcessManager.NonResponsiveProcess as e:
     raise TaskError('failure while terminating pantsd: {}'.format(e))

  def setUp(self):
    super(BaseTest, self).setUp()
    Goal.clear()
    self.real_build_root = BuildRoot().path
    self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
    self.pants_workdir = os.path.join(self.build_root, '.pants.d')
    safe_mkdir(self.pants_workdir)
    self.options = defaultdict(dict)  # scope -> key-value mapping.
    self.options[''] = {
      'pants_workdir': self.pants_workdir,
      'pants_supportdir': os.path.join(self.build_root, 'build-support'),
      'pants_distdir': os.path.join(self.build_root, 'dist'),
      'pants_configdir': os.path.join(self.build_root, 'config'),
      'cache_key_gen_version': '0-test',
    }
    BuildRoot().path = self.build_root
    Subsystem.reset()

    self.create_file('pants.ini')
    build_configuration = BuildConfiguration()
    build_configuration.register_aliases(self.alias_groups)
    self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
    self.build_graph = BuildGraph(address_mapper=self.address_mapper)
    self.bootstrap_option_values = OptionsBootstrapper().get_bootstrap_options().for_global_scope()

def select(argv):
  # Parse positional arguments to the script.
  args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
  # Resolve bootstrap options with a fake empty command line.
  options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
  subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
  known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set())
  options = options_bootstrapper.get_full_options(known_scope_infos)
  # Initialize Subsystems.
  Subsystem.set_options(options)

  # If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the
  # appropriate Archiver to pass to BinaryUtil.
  archiver_for_current_binary = None
  filename = args.filename or args.util_name
  try:
    archiver_for_current_binary = archiver_for_path(filename)
    # BinaryRequest requires the `name` field to be provided without an extension, as it appends the
    # archiver's extension if one is provided, so we have to remove it here.
    filename = filename[:-(len(archiver_for_current_binary.extension) + 1)]
  except ValueError:
    pass

  binary_util = BinaryUtil.Factory.create()
  binary_request = BinaryRequest(
    supportdir='bin/{}'.format(args.util_name),
    version=args.version,
    name=filename,
    platform_dependent=True,
    external_url_generator=None,
    archiver=archiver_for_current_binary)

  return binary_util.select(binary_request)

  def test_options_parsing_request(self):
    parse_request = OptionsParseRequest.create(
      ['./pants', '-ldebug', '--python-setup-wheel-version=3.13.37', 'binary', 'src/python::'],
      dict(PANTS_ENABLE_PANTSD='True', PANTS_BINARIES_BASEURLS='["https://bins.com"]')
    )

    # TODO: Once we have the ability to get FileContent for arbitrary
    # paths outside of the buildroot, we can move the construction of
    # OptionsBootstrapper into an @rule by cooperatively calling
    # OptionsBootstrapper.produce_and_set_bootstrap_options() which
    # will yield lists of file paths for use as subject values and permit
    # us to avoid the direct file I/O that this currently requires.
    options_bootstrapper = OptionsBootstrapper.from_options_parse_request(parse_request)
    build_config = BuildConfigInitializer.get(options_bootstrapper)

    options = run_rule(parse_options, options_bootstrapper, build_config)[0]

    self.assertIn('binary', options.goals)
    global_options = options.for_global_scope()
    self.assertEquals(global_options.level, 'debug')
    self.assertEquals(global_options.enable_pantsd, True)
    self.assertEquals(global_options.binaries_baseurls, ['https://bins.com'])

    python_setup_options = options.for_scope('python-setup')
    self.assertEquals(python_setup_options.wheel_version, '3.13.37')