Python python_setup.PythonSetup类代码示例

def dump_requirements(builder, interpreter, reqs, log, platforms=None):
  """Multi-platform dependency resolution for PEX files.

  :param builder: Dump the requirements into this builder.
  :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
  :param reqs: A list of :class:`PythonRequirement` to resolve.
  :param log: Use this logger.
  :param platforms: A list of :class:`Platform`s to resolve requirements for.
                    Defaults to the platforms specified by PythonSetup.
  """
  deduped_reqs = OrderedSet(reqs)
  find_links = OrderedSet()
  blacklist = PythonSetup.global_instance().resolver_blacklist
  for req in deduped_reqs:
    log.debug('  Dumping requirement: {}'.format(req))
    if not (req.key in blacklist and interpreter.identity.matches(blacklist[req.key])):
      builder.add_requirement(req.requirement)
    if req.repository:
      find_links.add(req.repository)

  # Resolve the requirements into distributions.
  distributions = _resolve_multi(interpreter, deduped_reqs, platforms, find_links)
  locations = set()
  for platform, dists in distributions.items():
    for dist in dists:
      if dist.location not in locations:
        log.debug('  Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
        builder.add_distribution(dist)
      locations.add(dist.location)

  def execute(self):
    python_tgts_and_reqs = self.context.targets(
      lambda tgt: isinstance(tgt, (PythonTarget, PythonRequirementLibrary))
    )
    if not python_tgts_and_reqs:
      return
    python_tgts = [tgt for tgt in python_tgts_and_reqs if isinstance(tgt, PythonTarget)]
    fs = PythonInterpreterFingerprintStrategy()
    with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
      if (PythonSetup.global_instance().interpreter_search_paths
          and PythonInterpreterCache.pex_python_paths()):
        self.context.log.warn("Detected both PEX_PYTHON_PATH and "
                              "--python-setup-interpreter-search-paths. Ignoring "
                              "--python-setup-interpreter-search-paths.")
      # If there are no relevant targets, we still go through the motions of selecting
      # an interpreter, to prevent downstream tasks from having to check for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
            invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'
      interpreter_path_file = self._interpreter_path_file(target_set_id)
      if not os.path.exists(interpreter_path_file):
        self._create_interpreter_path_file(interpreter_path_file, python_tgts)

    interpreter = self._get_interpreter(interpreter_path_file)
    self.context.products.register_data(PythonInterpreter, interpreter)

def tgt_closure_platforms(tgts):
  """
  Aggregates a dict that maps a platform string to a list of targets that specify the platform.
  If no targets have platforms arguments, return a dict containing platforms inherited from
  the PythonSetup object.

  :param tgts: a list of :class:`Target` objects.
  :returns: a dict mapping a platform string to a list of targets that specify the platform.
  """
  tgts_by_platforms = {}

  def insert_or_append_tgt_by_platform(tgt):
    if tgt.platforms:
      for platform in tgt.platforms:
        if platform in tgts_by_platforms:
          tgts_by_platforms[platform].append(tgt)
        else:
          tgts_by_platforms[platform] = [tgt]

  map(insert_or_append_tgt_by_platform, tgts)
  # If no targets specify platforms, inherit the default platforms.
  if not tgts_by_platforms:
    for platform in PythonSetup.global_instance().platforms:
      tgts_by_platforms[platform] = ['(No target) Platform inherited from either the '
                                     '--platforms option or a pants.ini file.']
  return tgts_by_platforms

def _resolve_multi(interpreter, requirements, platforms, find_links):
  """Multi-platform dependency resolution for PEX files.

  Returns a list of distributions that must be included in order to satisfy a set of requirements.
  That may involve distributions for multiple platforms.

  :param interpreter: The :class:`PythonInterpreter` to resolve for.
  :param requirements: A list of :class:`PythonRequirement` objects to resolve.
  :param platforms: A list of :class:`Platform`s to resolve for.
  :param find_links: Additional paths to search for source packages during resolution.
  :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
           to satisfy the requirements on that platform.
  """
  python_setup = PythonSetup.global_instance()
  python_repos = PythonRepos.global_instance()
  platforms = platforms or python_setup.platforms
  find_links = find_links or []
  distributions = {}
  fetchers = python_repos.get_fetchers()
  fetchers.extend(Fetcher([path]) for path in find_links)

  for platform in platforms:
    requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
                                          str(interpreter.identity))
    distributions[platform] = resolve(
      requirements=[req.requirement for req in requirements],
      interpreter=interpreter,
      fetchers=fetchers,
      platform=None if platform == 'current' else platform,
      context=python_repos.get_network_context(),
      cache=requirements_cache_dir,
      cache_ttl=python_setup.resolver_cache_ttl,
      allow_prereleases=python_setup.resolver_allow_prereleases)

  return distributions

  def test_setuptools_version(self):
    self.create_file('src/python/foo/__init__.py')
    self.create_python_library(
      relpath='src/python/foo/commands',
      name='commands',
      source_contents_map={
        'print_sys_path.py': dedent("""
          import os
          import sys
          from setuptools import Command


          class PrintSysPath(Command):
            user_options = []

            def initialize_options(self):
              pass

            def finalize_options(self):
              pass

            def run(self):
              with open(os.path.join(os.path.dirname(__file__), 'sys_path.txt'), 'w') as fp:
                fp.write(os.linesep.join(sys.path))
          """)
      },
    )
    foo = self.create_python_library(
      relpath='src/python/foo',
      name='foo',
      dependencies=[
        'src/python/foo/commands',
      ],
      provides=dedent("""
      setup_py(
        name='foo',
        version='0.0.0',
      )
      """)
    )
    self.set_options(run='print_sys_path')

    # Make sure setup.py can see our custom distutils Command 'print_sys_path'.
    sdist_srcdir = os.path.join(self.distdir, 'foo-0.0.0', 'src')
    with environment_as(PYTHONPATH=sdist_srcdir):
      with self.run_execute(foo):
        with open(os.path.join(sdist_srcdir, 'foo', 'commands', 'sys_path.txt')) as fp:
          def assert_extra(name, expected_version):
            package = Package.from_href(fp.readline().strip())
            self.assertEqual(name, package.name)
            self.assertEqual(expected_version, package.raw_version)

          # The 1st two elements of the sys.path should be our custom SetupPyRunner Installer's
          # setuptools and wheel mixins, which should match the setuptools and wheel versions
          # specified by the PythonSetup subsystem.
          init_subsystem(PythonSetup)
          python_setup = PythonSetup.global_instance()
          assert_extra('setuptools', python_setup.setuptools_version)
          assert_extra('wheel', python_setup.wheel_version)

 def _acceptable_interpreter_constraints(self):
   default_constraints = PythonSetup.global_instance().interpreter_constraints
   whitelisted_constraints = self.get_options().interpreter_constraints_whitelist
   # The user wants to lint everything.
   if whitelisted_constraints == []:
     return []
   # The user did not pass a whitelist option.
   elif whitelisted_constraints is None:
     whitelisted_constraints = ()
   return [version.parse(v) for v in default_constraints + whitelisted_constraints]

 def _interpreter_cache(self):
   interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                              PythonRepos.global_instance(),
                                              logger=self.context.log.debug)
   # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
   self.context.acquire_lock()
   try:
     interpreter_cache.setup()
   finally:
     self.context.release_lock()
   return interpreter_cache

 def create_chroot(self, interpreter, builder, targets, platforms, extra_requirements):
   return PythonChroot(python_setup=PythonSetup.global_instance(),
                       python_repos=PythonRepos.global_instance(),
                       ivy_bootstrapper=self.ivy_bootstrapper,
                       thrift_binary_factory=self.thrift_binary_factory,
                       interpreter=interpreter,
                       builder=builder,
                       targets=targets,
                       platforms=platforms,
                       extra_requirements=extra_requirements,
                       log=self.context.log)

    def create(cls, builder, log=None):
      options = cls.global_instance().get_options()
      setuptools_requirement = 'setuptools=={}'.format(options.setuptools_version)

      log = log or logging.getLogger(__name__)

      return PexBuilderWrapper(builder=builder,
                               python_repos_subsystem=PythonRepos.global_instance(),
                               python_setup_subsystem=PythonSetup.global_instance(),
                               setuptools_requirement=PythonRequirement(setuptools_requirement),
                               log=log)

  def test_expand_interpreter_search_paths(self):
    with environment_as(PATH='/env/path1:/env/path2'):
      with setup_pexrc_with_pex_python_path(['/pexrc/path1:/pexrc/path2']):
        with fake_pyenv_root(['2.7.14', '3.5.5']) as (pyenv_root, expected_pyenv_paths):
          paths = ['/foo', '<PATH>', '/bar', '<PEXRC>', '/baz', '<PYENV>', '/qux']
          expanded_paths = PythonSetup.expand_interpreter_search_paths(
            paths, pyenv_root_func=lambda: pyenv_root)

    expected = ['/foo', '/env/path1', '/env/path2', '/bar', '/pexrc/path1', '/pexrc/path2',
                '/baz'] + expected_pyenv_paths + ['/qux']
    self.assertListEqual(expected, expanded_paths)

 def _create_interpreter_path_file(self, interpreter_path_file, targets):
   interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                              PythonRepos.global_instance(),
                                              logger=self.context.log.debug)
   interpreter = interpreter_cache.select_interpreter_for_targets(targets)
   safe_mkdir_for(interpreter_path_file)
   with open(interpreter_path_file, 'w') as outfile:
     outfile.write(b'{}\n'.format(interpreter.binary))
     for dist, location in interpreter.extras.items():
       dist_name, dist_version = dist
       outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))

  def test_setup_using_eggs(self):
    def link_egg(repo_root, requirement):
      existing_dist_location = self._interpreter.get_location(requirement)
      if existing_dist_location is not None:
        existing_dist = Package.from_href(existing_dist_location)
        requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)

      distributions = resolve([requirement],
                              interpreter=self._interpreter,
                              precedence=(EggPackage, SourcePackage))
      self.assertEqual(1, len(distributions))
      dist_location = distributions[0].location

      self.assertRegexpMatches(dist_location, r'\.egg$')
      os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))

      return Package.from_href(dist_location).raw_version

    with temporary_dir() as root:
      egg_dir = os.path.join(root, 'eggs')
      os.makedirs(egg_dir)
      setuptools_version = link_egg(egg_dir, 'setuptools')
      wheel_version = link_egg(egg_dir, 'wheel')

      interpreter_requirement = self._interpreter.identity.requirement

      self.context(for_subsystems=[PythonSetup, PythonRepos], options={
        PythonSetup.options_scope: {
          'interpreter_cache_dir': None,
          'pants_workdir': os.path.join(root, 'workdir'),
          'constraints': [interpreter_requirement],
          'setuptools_version': setuptools_version,
          'wheel_version': wheel_version,
        },
        PythonRepos.options_scope: {
          'indexes': [],
          'repos': [egg_dir],
        }
      })
      cache = PythonInterpreterCache(PythonSetup.global_instance(), PythonRepos.global_instance())

      interpereters = cache.setup(paths=[os.path.dirname(self._interpreter.binary)],
                                  filters=[str(interpreter_requirement)])
      self.assertGreater(len(interpereters), 0)

      def assert_egg_extra(interpreter, name, version):
        location = interpreter.get_location('{}=={}'.format(name, version))
        self.assertIsNotNone(location)
        self.assertIsInstance(Package.from_href(location), EggPackage)

      for interpreter in interpereters:
        assert_egg_extra(interpreter, 'setuptools', setuptools_version)
        assert_egg_extra(interpreter, 'wheel', wheel_version)

  def _gather_sources(self, target_roots):
    context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])

    # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
    # to ensure that the interpreter has setuptools and wheel support.
    interpreter = PythonInterpreter.get()
    interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                               PythonRepos.global_instance(),
                                               logger=context.log.debug)
    interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
                                           filters=[str(interpreter.identity.requirement)])
    context.products.get_data(PythonInterpreter, lambda: interpreters[0])

    task = self.create_task(context)
    task.execute()

    return context.products.get_data(GatherSources.PYTHON_SOURCES)

  def create_pex(self, pex_info=None):
    """Returns a wrapped pex that "merges" the other pexes via PEX_PATH."""
    relevant_targets = self.context.targets(
      lambda tgt: isinstance(tgt, (
        PythonDistribution, PythonRequirementLibrary, PythonTarget, Files)))
    with self.invalidated(relevant_targets) as invalidation_check:

      # If there are no relevant targets, we still go through the motions of resolving
      # an empty set of requirements, to prevent downstream tasks from having to check
      # for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
          invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'

      interpreter = self.context.products.get_data(PythonInterpreter)
      path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id))

      # Note that we check for the existence of the directory, instead of for invalid_vts,
      # to cover the empty case.
      if not os.path.isdir(path):
        pexes = [
          self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
          self.context.products.get_data(GatherSources.PYTHON_SOURCES)
        ]

        if self.extra_requirements():
          extra_requirements_pex = self.resolve_requirement_strings(
            interpreter, self.extra_requirements())
          # Add the extra requirements first, so they take precedence over any colliding version
          # in the target set's dependency closure.
          pexes = [extra_requirements_pex] + pexes
        constraints = {constraint for rt in relevant_targets if is_python_target(rt)
                       for constraint in PythonSetup.global_instance().compatibility_or_constraints(rt)}

        with self.merged_pex(path, pex_info, interpreter, pexes, constraints) as builder:
          for extra_file in self.extra_files():
            extra_file.add_to(builder)
          builder.freeze()

    return PEX(path, interpreter)

 def subsystem_dependencies(cls):
   return super(ResolveRequirementsTaskBase, cls).subsystem_dependencies() + (
     PexBuilderWrapper.Factory,
     PythonNativeCode.scoped(cls),
     PythonSetup.scoped(cls),
   )