Python ivy_subsystem.IvySubsystem类代码示例

 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type())
   # We locate the workdir below the pants_workdir, which BaseTest locates within
   # the BuildRoot.
   self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir)
   self._test_workdir = os.path.join(self._tmpdir, 'workdir')
   os.mkdir(self._test_workdir)
   Bootstrapper.reset_instance()
   IvySubsystem.reset_global_instance()

  def dumped_chroot(self, targets):
    # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
    # declares it as a dependency. However if we don't then test_antlr() below fails on
    # uninitialized options for that subsystem.  Hopefully my pending (as of 9/2016) change
    # to clean up how we initialize and create instances of subsystems in tests will make
    # this problem go away.
    self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
                                 DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
    python_repos = PythonRepos.global_instance()
    ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
    thrift_binary_factory = ThriftBinary.Factory.global_instance().create

    interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
    interpreter_cache.setup()
    interpreters = list(interpreter_cache.matched_interpreters(
      self.python_setup.interpreter_constraints))
    self.assertGreater(len(interpreters), 0)
    interpreter = interpreters[0]

    with temporary_dir() as chroot:
      pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)

      python_chroot = PythonChroot(python_setup=self.python_setup,
                                   python_repos=python_repos,
                                   ivy_bootstrapper=ivy_bootstrapper,
                                   thrift_binary_factory=thrift_binary_factory,
                                   interpreter=interpreter,
                                   builder=pex_builder,
                                   targets=targets,
                                   platforms=['current'])
      try:
        python_chroot.dump()
        yield pex_builder, python_chroot
      finally:
        python_chroot.delete()

    def ivy_cache_dir(self):
        """The path of the ivy cache dir used for resolves.

    :rtype: string
    """
        # TODO(John Sirois): Fixup the IvySubsystem to encapsulate its properties.
        return IvySubsystem.global_instance().get_options().cache_dir

  def execute(self):
    targets = self.context.targets()
    jars, global_excludes = IvyUtils.calculate_classpath(targets)

    filtered_jars = [jar for jar in jars if self._is_update_coordinate(jar.coordinate)]
    sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier))

    ivyxml = os.path.join(self.workdir, 'ivy.xml')
    IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=['default'])

    args = [
      '-settings', IvySubsystem.global_instance().get_options().ivy_settings,
      '-ivy', ivyxml,
      '-confs', ','.join(self.get_options().confs)
    ]

    result = self.runjava(classpath=self.tool_classpath('dependency-update-checker'),
                          main=self._IVY_DEPENDENCY_UPDATE_MAIN,
                          jvm_options=self.get_options().jvm_options,
                          args=args,
                          workunit_name='dependency-update-checker',
                          workunit_labels=[WorkUnitLabel.LINT])

    self.context.log.debug('java {main} ... exited with result ({result})'.format(
      main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result))

    return result

 def xml_report_path(cls, resolve_hash_name, conf):
   """The path to the xml report ivy creates after a retrieve.
   :param string resolve_hash_name: Hash from the Cache key from the VersionedTargetSet
   used for resolution.
   :param string conf: the ivy conf name (e.g. "default")
   """
   cachedir = IvySubsystem.global_instance().get_options().cache_dir
   return os.path.join(cachedir, '{}-{}-{}.xml'.format(IvyUtils.INTERNAL_ORG_NAME,
                                                       resolve_hash_name, conf))

  def test_parse_proxy_string(self):
    ivy_subsystem =IvySubsystem.global_instance()

    self.assertEquals(('example.com', 1234),
                      ivy_subsystem._parse_proxy_string('http://example.com:1234'))
    self.assertEquals(('secure-example.com', 999),
                      ivy_subsystem._parse_proxy_string('http://secure-example.com:999'))
    # trailing slash is ok
    self.assertEquals(('example.com', 1234),
                      ivy_subsystem._parse_proxy_string('http://example.com:1234/'))

 def test_simple(self):
     ivy_subsystem = IvySubsystem.global_instance()
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(
         ivy_subsystem.get_options().pants_bootstrapdir, "tools", "jvm", "ivy", "bootstrap.jar"
     )
     self.assertTrue(os.path.exists(bootstrap_jar_path))

  def __init__(self, *args, **kwargs):
    super(IvyResolve, self).__init__(*args, **kwargs)

    self._cachedir = IvySubsystem.global_instance().get_options().cache_dir
    self._classpath_dir = os.path.join(self.workdir, 'mapped')
    self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports')
    self._open = self.get_options().open
    self._report = self._open or self.get_options().report
    self._confs = None

    self._args = []
    for arg in self.get_options().args:
      self._args.extend(safe_shlex_split(arg))

  def __init__(self, *args, **kwargs):
    super(IvyResolve, self).__init__(*args, **kwargs)

    self._cachedir = IvySubsystem.global_instance().get_options().cache_dir
    self._classpath_dir = os.path.join(self.workdir, 'mapped')
    self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports')
    self._open = self.get_options().open
    self._report = self._open or self.get_options().report
    self._confs = None

    self._args = []
    for arg in self.get_options().args:
      self._args.extend(safe_shlex_split(arg))

    # Typically this should be a local cache only, since classpaths aren't portable.
    self.setup_artifact_cache()

  def test_proxy_from_env(self):
    ivy_subsystem = IvySubsystem.global_instance()

    self.assertIsNone(ivy_subsystem.http_proxy())
    self.assertIsNone(ivy_subsystem.https_proxy())

    with environment_as(HTTP_PROXY='http://proxy.example.com:456',
                        HTTPS_PROXY='https://secure-proxy.example.com:789'):
      self.assertEquals('http://proxy.example.com:456', ivy_subsystem.http_proxy())
      self.assertEquals('https://secure-proxy.example.com:789', ivy_subsystem.https_proxy())

      self.assertEquals([
        '-Dhttp.proxyHost=proxy.example.com',
        '-Dhttp.proxyPort=456',
        '-Dhttps.proxyHost=secure-proxy.example.com',
        '-Dhttps.proxyPort=789',
      ], ivy_subsystem.extra_jvm_options())

  def _unpack(self, unpacked_archives):
    """Extracts files from the downloaded jar files and places them in a work directory.

    :param UnpackedArchives unpacked_archives: target referencing jar_libraries to unpack.
    """
    self.context.log.info('Unpacking {}'.format(unpacked_archives.address.spec))
    unpack_dir = unpacked_archives.destination
    safe_mkdir(unpack_dir, clean=True)

    unpack_filter = self.get_unpack_filter(unpacked_archives)
    classpath_products =  ClasspathProducts(self.get_options().pants_workdir)
    resolve_hashes = self.resolve(None, unpacked_archives.dependencies, classpath_products)
    ivy_cache_dir = os.path.expanduser(IvySubsystem.global_instance().get_options().cache_dir)

    def to_m2(jar):
      return M2Coordinate(org=jar.org, name=jar.name, rev=jar.rev, classifier=jar.classifier,
                          ext=jar.ext)

    libraries = self.context.build_graph.transitive_subgraph_of_addresses([unpacked_archives.address])
    libraries = [t for t in libraries if isinstance(t, JarLibrary)]
    coords = set()
    for library in libraries:
      coords.update(to_m2(jar) for jar in library.payload.jars)

    for resolve_hash in resolve_hashes:
      path = IvyUtils.xml_report_path(ivy_cache_dir, resolve_hash, 'default')
      info = IvyUtils.parse_xml_report('default', path)
      refs_for_libraries = set()
      for ref in info.modules_by_ref.keys():
        if to_m2(ref) in coords:
          refs_for_libraries.add(ref)

      memo = {}
      for ref in tuple(refs_for_libraries):
        info.traverse_dependency_graph(ref, refs_for_libraries.add, memo)

      for ref in sorted(refs_for_libraries):
        module = info.modules_by_ref[ref]
        artifact_path = module.artifact
        self.context.log.debug('Extracting {} to {}.'.format(to_m2(ref), unpack_dir))
        if artifact_path.endswith('.zip') or artifact_path.endswith('.jar'):
          ZIP.extract(artifact_path, unpack_dir, filter_func=unpack_filter)
        else:
          self._extract_tar(artifact_path, unpack_dir, filter_func=unpack_filter)

 def ivy_resolution_cache_dir(self):
   return IvySubsystem.global_instance().resolution_cache_dir()

 def ivy_repository_cache_dir(self):
   return IvySubsystem.global_instance().repository_cache_dir()

  def ivy_resolve(self,
                  targets,
                  executor=None,
                  silent=False,
                  workunit_name=None,
                  confs=None,
                  custom_args=None):
    """Executes an ivy resolve for the relevant subset of the given targets.

    Returns the resulting classpath, and the set of relevant targets. Also populates
    the 'ivy_resolve_symlink_map' product for jars resulting from the resolve."""

    if not targets:
      return ([], set())

    # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
    # the generated module, which in turn determines the location of the XML report file
    # ivy generates. We recompute this name from targets later in order to find that file.
    # TODO: This is fragile. Refactor so that we're not computing the name twice.
    ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)

    ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')

    fingerprint_strategy = IvyResolveFingerprintStrategy(confs)

    with self.invalidated(targets,
                          invalidate_dependents=False,
                          silent=silent,
                          fingerprint_strategy=fingerprint_strategy) as invalidation_check:
      if not invalidation_check.all_vts:
        return ([], set())
      global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)

      # If a report file is not present, we need to exec ivy, even if all the individual
      # targets up to date... See https://rbcommons.com/s/twitter/r/2015
      report_missing = False
      report_confs = confs or ['default']
      report_paths = []
      for conf in report_confs:
        report_path = IvyUtils.xml_report_path(global_vts.targets, conf)
        if not os.path.exists(report_path):
          report_missing = True
          break
        else:
          report_paths.append(report_path)

      target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
      target_classpath_file = os.path.join(target_workdir, 'classpath')
      raw_target_classpath_file = target_classpath_file + '.raw'
      raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
      # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
      # in artifact-cached analysis files are consistent across systems.
      # Note that we have one global, well-known symlink dir, again so that paths are
      # consistent across builds.
      symlink_dir = os.path.join(ivy_workdir, 'jars')

      # Note that it's possible for all targets to be valid but for no classpath file to exist at
      # target_classpath_file, e.g., if we previously built a superset of targets.
      if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
        args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])

        self.exec_ivy(
            target_workdir=target_workdir,
            targets=global_vts.targets,
            args=args,
            executor=executor,
            ivy=ivy,
            workunit_name=workunit_name,
            confs=confs,
            use_soft_excludes=self.get_options().soft_excludes)

        if not os.path.exists(raw_target_classpath_file_tmp):
          raise TaskError('Ivy failed to create classpath file at {}'
                          .format(raw_target_classpath_file_tmp))
        shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
        logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))

        if self.artifact_cache_writes_enabled():
          self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
      else:
        logger.debug("Using previously resolved reports: {}".format(report_paths))

    # Make our actual classpath be symlinks, so that the paths are uniform across systems.
    # Note that we must do this even if we read the raw_target_classpath_file from the artifact
    # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
    with IvyTaskMixin.symlink_map_lock:
      products = self.context.products
      existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict())
      symlink_map = IvyUtils.symlink_cachepath(
        IvySubsystem.global_instance().get_options().cache_dir,
        raw_target_classpath_file,
        symlink_dir,
        target_classpath_file,
        existing_symlinks_map)
      existing_symlinks_map.update(symlink_map)

    with IvyUtils.cachepath(target_classpath_file) as classpath:
      stripped_classpath = [path.strip() for path in classpath]
      return (stripped_classpath, global_vts.targets)

 def __init__(self, ivy_subsystem=None):
   """Creates an ivy bootstrapper."""
   self._ivy_subsystem = ivy_subsystem or IvySubsystem.global_instance()
   self._version_or_ivyxml = self._ivy_subsystem.get_options().ivy_profile
   self._classpath = None