Python pex_builder.PEXBuilder类代码示例

 def _test_runner(self, targets, workunit):
   interpreter = self.select_interpreter_for_targets(targets)
   builder = PEXBuilder(interpreter=interpreter)
   builder.info.entry_point = 'pytest'
   chroot = PythonChroot(
     context=self.context,
     python_setup=PythonSetup.global_instance(),
     python_repos=PythonRepos.global_instance(),
     targets=targets,
     extra_requirements=self._TESTING_TARGETS,
     builder=builder,
     platforms=('current',),
     interpreter=interpreter)
   try:
     builder = chroot.dump()
     builder.freeze()
     pex = PEX(builder.path(), interpreter=interpreter)
     with self._maybe_shard() as shard_args:
       with self._maybe_emit_junit_xml(targets) as junit_args:
         with self._maybe_emit_coverage_data(targets,
                                             builder.path(),
                                             pex,
                                             workunit) as coverage_args:
           yield pex, shard_args + junit_args + coverage_args
   finally:
     chroot.delete()

  def temporary_chroot(self, interpreter=None, pex_info=None, targets=None,
                       extra_requirements=None, platforms=None, pre_freeze=None):
    """Yields a temporary PythonChroot created with the specified args.

    pre_freeze is an optional function run on the chroot just before freezing its builder,
    to allow for any extra modification.
    """
    path = tempfile.mkdtemp()
    builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
    with self.context.new_workunit('chroot'):
      chroot = PythonChroot(
        context=self.context,
        python_setup=PythonSetup.global_instance(),
        python_repos=PythonRepos.global_instance(),
        targets=targets,
        extra_requirements=extra_requirements,
        builder=builder,
        platforms=platforms,
        interpreter=interpreter)
      chroot.dump()
      if pre_freeze:
        pre_freeze(chroot)
      builder.freeze()
    yield chroot
    chroot.delete()

  def resolve_requirements(self, interpreter, req_libs):
    """Requirements resolution for PEX files.

    :param interpreter: Resolve against this :class:`PythonInterpreter`.
    :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
    :returns: a PEX containing target requirements and any specified python dist targets.
    """
    with self.invalidated(req_libs) as invalidation_check:
      # If there are no relevant targets, we still go through the motions of resolving
      # an empty set of requirements, to prevent downstream tasks from having to check
      # for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
            invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'

      # We need to ensure that we are resolving for only the current platform if we are
      # including local python dist targets that have native extensions.
      tgts = self.context.targets()
      maybe_platforms = ['current'] if build_for_current_platform_only_check(tgts) else None

      path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id))
      # Note that we check for the existence of the directory, instead of for invalid_vts,
      # to cover the empty case.
      if not os.path.isdir(path):
        with safe_concurrent_creation(path) as safe_path:
          builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True)
          dump_requirement_libs(builder, interpreter, req_libs, self.context.log, platforms=maybe_platforms)
          builder.freeze()
    return PEX(path, interpreter=interpreter)

  def nsutil_pex(self):
    interpreter = self.context.products.get_data(PythonInterpreter)
    chroot = os.path.join(self.workdir, 'nsutil', interpreter.version_string)
    if not os.path.exists(chroot):
      pex_info = PexInfo.default(interpreter=interpreter)
      with safe_concurrent_creation(chroot) as scratch:
        builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True)
        with temporary_file(binary_mode=False) as fp:
          declares_namespace_package_code = inspect.getsource(declares_namespace_package)
          fp.write(textwrap.dedent("""
            import sys


            {declares_namespace_package_code}


            if __name__ == '__main__':
              for path in sys.argv[1:]:
                if declares_namespace_package(path):
                  print(path)
          """).strip().format(declares_namespace_package_code=declares_namespace_package_code))
          fp.close()
          builder.set_executable(filename=fp.name, env_filename='main.py')
          builder.freeze()
    return PEX(pex=chroot, interpreter=interpreter)

def test_pex_builder_deterministic_timestamp():
  pb = PEXBuilder()
  with temporary_dir() as td:
    target = os.path.join(td, 'foo.pex')
    pb.build(target, deterministic_timestamp=True)
    with zipfile.ZipFile(target) as zf:
      assert all(zinfo.date_time == (1980, 1, 1, 0, 0, 0) for zinfo in zf.infolist())

  def _create_binary(self, binary_tgt, results_dir):
    """Create a .pex file for the specified binary target."""
    # Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
    # and PYTHON_SOURCES products, because those products are already-built pexes, and there's
    # no easy way to merge them into a single pex file (for example, they each have a __main__.py,
    # metadata, and so on, which the merging code would have to handle specially).
    interpreter = self.context.products.get_data(PythonInterpreter)
    with temporary_dir() as tmpdir:
      # Create the pex_info for the binary.
      run_info_dict = self.context.run_tracker.run_info.get_as_dict()
      build_properties = PexInfo.make_build_properties()
      build_properties.update(run_info_dict)
      pex_info = binary_tgt.pexinfo.copy()
      pex_info.build_properties = build_properties

      builder = PEXBuilder(path=tmpdir, interpreter=interpreter, pex_info=pex_info, copy=True)

      # Find which targets provide sources and which specify requirements.
      source_tgts = []
      req_tgts = []
      for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
        if has_python_sources(tgt) or has_resources(tgt):
          source_tgts.append(tgt)
        elif has_python_requirements(tgt):
          req_tgts.append(tgt)

      # Dump everything into the builder's chroot.
      for tgt in source_tgts:
        dump_sources(builder, tgt, self.context.log)
      dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)

      # Build the .pex file.
      pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
      builder.build(pex_path)
      return pex_path

def write_pex(td, exe_contents):
  with open(os.path.join(td, 'exe.py'), 'w') as fp:
    fp.write(exe_contents)

  pb = PEXBuilder(path=td)
  pb.set_executable(os.path.join(td, 'exe.py'))
  pb.freeze()

  return pb

def test_pex_builder_shebang():
  pb = PEXBuilder()
  pb.set_shebang('foobar')

  with temporary_dir() as td:
    target = os.path.join(td, 'foo.pex')
    pb.build(target)
    expected_preamble = b'#!foobar\n'
    with open(target, 'rb') as fp:
      assert fp.read(len(expected_preamble)) == expected_preamble

 def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
   reqs_pex_path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity),
                                                 vt.cache_key.hash))
   if not os.path.isdir(reqs_pex_path):
     req_libs = filter(has_python_requirements, vt.target.closure())
     with safe_concurrent_creation(reqs_pex_path) as safe_path:
       builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True)
       dump_requirement_libs(builder, interpreter, req_libs, self.context.log)
       builder.freeze()
   return PEX(reqs_pex_path, interpreter=interpreter)

    def build_and_check(path, copy):
      pb = PEXBuilder(path, copy=copy)
      pb.add_source(src, 'exe.py')

      s1 = os.stat(src)
      s2 = os.stat(os.path.join(path, 'exe.py'))
      is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
      if copy:
        assert not is_link
      else:
        assert is_link

  def create_pex(self, pex_info=None):
    """Returns a wrapped pex that "merges" the other pexes via PEX_PATH."""
    relevant_targets = self.context.targets(
      lambda tgt: isinstance(tgt, (PythonRequirementLibrary, PythonTarget, Resources)))
    with self.invalidated(relevant_targets) as invalidation_check:

      # If there are no relevant targets, we still go through the motions of resolving
      # an empty set of requirements, to prevent downstream tasks from having to check
      # for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
          invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'

      interpreter = self.context.products.get_data(PythonInterpreter)
      path = os.path.join(self.workdir, str(interpreter.identity), target_set_id)
      extra_pex_paths_file_path = path + '.extra_pex_paths'
      extra_pex_paths = None

      # Note that we check for the existence of the directory, instead of for invalid_vts,
      # to cover the empty case.
      if not os.path.isdir(path):
        pexes = [
          self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
          self.context.products.get_data(GatherSources.PYTHON_SOURCES)
        ]

        if self.extra_requirements():
          extra_reqs = [PythonRequirement(req_str) for req_str in self.extra_requirements()]
          addr = Address.parse('{}_extra_reqs'.format(self.__class__.__name__))
          self.context.build_graph.inject_synthetic_target(
            addr, PythonRequirementLibrary, requirements=extra_reqs)
          # Add the extra requirements first, so they take precedence over any colliding version
          # in the target set's dependency closure.
          pexes = [self.resolve_requirements([self.context.build_graph.get_target(addr)])] + pexes

        extra_pex_paths = [pex.path() for pex in pexes if pex]

        with safe_concurrent_creation(path) as safe_path:
          builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
          builder.freeze()

        with open(extra_pex_paths_file_path, 'w') as outfile:
          for epp in extra_pex_paths:
            outfile.write(epp)
            outfile.write(b'\n')

    if extra_pex_paths is None:
      with open(extra_pex_paths_file_path, 'r') as infile:
        extra_pex_paths = [p.strip() for p in infile.readlines()]
    return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter)

  def merge_pexes(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None):
    """Generates a merged pex at path."""
    pex_paths = [pex.path() for pex in pexes if pex]
    if pex_paths:
      pex_info = pex_info.copy()
      pex_info.merge_pex_path(':'.join(pex_paths))

    with safe_concurrent_creation(path) as safe_path:
      builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
      if interpeter_constraints:
        for constraint in interpeter_constraints:
          builder.add_interpreter_constraint(constraint)
      builder.freeze()

 def bootstrap_conan(self):
   pex_info = PexInfo.default()
   pex_info.entry_point = 'conans.conan'
   conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
   conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
   interpreter = PythonInterpreter.get()
   if not os.path.exists(conan_pex_path):
     with safe_concurrent_creation(conan_pex_path) as safe_path:
       builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
       reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
       dump_requirements(builder, interpreter, reqs, logger)
       builder.freeze()
   conan_binary = PEX(conan_pex_path, interpreter)
   return self.ConanBinary(pex=conan_binary)

    def build_and_check(path, copy):
      pb = PEXBuilder(path, copy=copy)
      pb.add_source(src, 'exe.py')

      path_clone = os.path.join(path, '__clone')
      pb.clone(into=path_clone)

      for root in path, path_clone:
        s1 = os.stat(src)
        s2 = os.stat(os.path.join(root, 'exe.py'))
        is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
        if copy:
          assert not is_link
        else:
          assert is_link

  def merged_pex(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None):
    """Yields a pex builder at path with the given pexes already merged.

    :rtype: :class:`pex.pex_builder.PEXBuilder`
    """
    pex_paths = [pex.path() for pex in pexes if pex]
    if pex_paths:
      pex_info = pex_info.copy()
      pex_info.merge_pex_path(':'.join(pex_paths))

    with safe_concurrent_creation(path) as safe_path:
      builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
      if interpeter_constraints:
        for constraint in interpeter_constraints:
          builder.add_interpreter_constraint(constraint)
      yield builder