Python jvm_tool_mixin.JvmToolMixin类代码示例

  def setUp(self):
    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for proper isolation when bootstrapping within the test.
    bootstrap_scope = 'bootstrap_scope'
    self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
    JvmToolMixin.reset_registered_tools()

    # Set some options:

    # 1. Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    #    -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    #    tool, constructing a fat jar and then shading that fat jar.
    #
    # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
    #    each have to resolve and shade them every single time, which is a huge slowdown.
    #    Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
    #    tests to write to it.
    #
    # Note that we don't have access to the invoking pants instance's options, so we assume that
    # its artifact cache is in the standard location.  If it isn't, worst case the tests will
    # populate a second cache at the standard location, which is no big deal.
    # TODO: We really need a straightforward way for pants's own tests to get to the enclosing
    # pants instance's options values.
    artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')]
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
    self.set_options_for_scope('cache.{}'.format(bootstrap_scope),
                               read_from=artifact_caches,
                               write_to=artifact_caches)

    # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
    # into our test workspace.
    shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)

    Bootstrapper.reset_instance()

  def setUp(self):
    # Ensure we get a read of the real pants.ini config
    Config.reset_default_bootstrap_option_values()
    real_config = Config.from_cache()

    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for bootstrapping within the test, to isolate this from
    # any bootstrapping the pants run executing the test might need.
    self.bootstrap_task_type, bootstrap_scope = self.synthesize_task_subtype(BootstrapJvmTools)
    JvmToolMixin.reset_registered_tools()

    # Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    # tool, constructing a fat jar and then shading that fat jar.
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])

    def link_or_copy(src, dest):
      try:
        os.link(src, dest)
      except OSError as e:
        if e.errno == errno.EXDEV:
          shutil.copy(src, dest)
        else:
          raise e

    def link(path, optional=False, force=False):
      src = os.path.join(self.real_build_root, path)
      if not optional or os.path.exists(src):
        dest = os.path.join(self.build_root, path)
        safe_mkdir(os.path.dirname(dest))
        try:
          link_or_copy(src, dest)
        except OSError as e:
          if force and e.errno == errno.EEXIST:
            os.unlink(dest)
            link_or_copy(src, dest)
          else:
            raise e
        return dest

    def link_tree(path, optional=False, force=False):
      src = os.path.join(self.real_build_root, path)
      if not optional or os.path.exists(src):
        for abspath, dirs, files in safe_walk(src):
          for f in files:
            link(os.path.relpath(os.path.join(abspath, f), self.real_build_root), force=force)

    # TODO(John Sirois): Find a way to do this cleanly
    link('pants.ini', force=True)

    # TODO(pl): Note that this pulls in a big chunk of the hairball to every test that
    # depends on it, because BUILD contains source_roots that specify a variety of types
    # from different backends.
    link('BUILD', force=True)

    link('BUILD.tools', force=True)
    support_dir = real_config.getdefault('pants_supportdir')
    link_tree(os.path.relpath(os.path.join(support_dir, 'ivy'), self.real_build_root), force=True)

 def execute(self):
   registered_tools = JvmToolMixin.get_registered_tools()
   if registered_tools:
     # Map of scope -> (map of key -> callback).
     callback_product_map = self.context.products.get_data('jvm_build_tools_classpath_callbacks',
                                                           init_func=lambda: defaultdict(dict))
     # We leave a callback in the products map because we want these Ivy calls
     # to be done lazily (they might never actually get executed) and we want
     # to hit Task.invalidated (called in Task._ivy_resolve) on the instance of
     # BootstrapJvmTools rather than the instance of whatever class requires
     # the bootstrap tools.  It would be awkward and possibly incorrect to call
     # self.invalidated twice on a Task that does meaningful invalidation on its
     # targets. -pl
     for jvm_tool in registered_tools:
       dep_spec = jvm_tool.dep_spec(self.context.options)
       callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool)
       callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
     if self.get_options().eager:
       with self.context.new_workunit('eager'):
         for scope, callbacks_by_key in callback_product_map.items():
           for key, callback in callbacks_by_key.items():
             try:
               callback()
             except self.ToolUnderspecified:
               pass  # We don't want to fail for placeholder registrations

  def get_alternate_target_roots(cls, options, address_mapper, build_graph):
    processed = set()
    for jvm_tool in JvmToolMixin.get_registered_tools():
      dep_spec = jvm_tool.dep_spec(options)
      dep_address = Address.parse(dep_spec)
      # Some JVM tools are requested multiple times, we only need to handle them once.
      if dep_address not in processed:
        processed.add(dep_address)
        try:
          if build_graph.resolve_address(dep_address):
            # The user has defined a tool classpath override - we let that stand.
            continue
        except AddressLookupError as e:
          if jvm_tool.classpath is None:
            raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
          else:
            if not jvm_tool.is_default(options):
              # The user specified a target spec for this jvm tool that doesn't actually exist.
              # We want to error out here instead of just silently using the default option while
              # appearing to respect their config.
              raise cls.ToolResolveError(dedent("""
                  Failed to resolve target for tool: {tool}. This target was obtained from
                  option {option} in scope {scope}.

                  Make sure you didn't make a typo in the tool's address. You specified that the
                  tool should use the target found at "{tool}".

                  This target has a default classpath configured, so you can simply remove:
                    [{scope}]
                    {option}: {tool}
                  from pants.ini (or any other config file) to use the default tool.

                  The default classpath is: {default_classpath}

                  Note that tool target addresses in pants.ini should be specified *without* quotes.
                """).strip().format(tool=dep_spec,
                                    option=jvm_tool.key,
                                    scope=jvm_tool.scope,
                                    default_classpath=':'.join(map(str, jvm_tool.classpath or ()))))
            if jvm_tool.classpath:
              tool_classpath_target = JarLibrary(name=dep_address.target_name,
                                                 address=dep_address,
                                                 build_graph=build_graph,
                                                 jars=jvm_tool.classpath)
            else:
              # The tool classpath is empty by default, so we just inject a dummy target that
              # ivy resolves as the empty list classpath.  JarLibrary won't do since it requires
              # one or more jars, so we just pick a target type ivy has no resolve work to do for.
              tool_classpath_target = Target(name=dep_address.target_name,
                                             address=dep_address,
                                             build_graph=build_graph)
            build_graph.inject_target(tool_classpath_target, synthetic=True)

    # We use the trick of not returning alternate roots, but instead just filling the dep_spec
    # holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
    # no over-riding targets present. This means we do modify the build_graph, but we at least do
    # it at a time in the engine lifecycle cut out for handling that.
    return None

  def setUp(self):
    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for proper isolation when bootstrapping within the test.
    bootstrap_scope = 'bootstrap_scope'
    self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
    JvmToolMixin.reset_registered_tools()

    # Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    # tool, constructing a fat jar and then shading that fat jar.
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])

    # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
    # into our test workspace.
    shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)

    Bootstrapper.reset_instance()

 def execute(self):
   context = self.context
   if JvmToolMixin.get_registered_tools():
     # Map of scope -> (map of key -> callback).
     callback_product_map = (context.products.get_data('jvm_build_tools_classpath_callbacks') or
                             defaultdict(dict))
     # We leave a callback in the products map because we want these Ivy calls
     # to be done lazily (they might never actually get executed) and we want
     # to hit Task.invalidated (called in Task.ivy_resolve) on the instance of
     # BootstrapJvmTools rather than the instance of whatever class requires
     # the bootstrap tools.  It would be awkward and possibly incorrect to call
     # self.invalidated twice on a Task that does meaningful invalidation on its
     # targets. -pl
     for scope, key, main, custom_rules in JvmToolMixin.get_registered_tools():
       option = key.replace('-', '_')
       deplist = self.context.options.for_scope(scope)[option]
       callback_product_map[scope][key] = self.cached_bootstrap_classpath_callback(
           key, scope, deplist, main=main, custom_rules=custom_rules)
     context.products.safe_create_data('jvm_build_tools_classpath_callbacks',
                                       lambda: callback_product_map)

 def execute(self):
     registered_tools = JvmToolMixin.get_registered_tools()
     if registered_tools:
         # Map of scope -> (map of key -> callback).
         callback_product_map = self.context.products.get_data(
             "jvm_build_tools_classpath_callbacks", init_func=lambda: defaultdict(dict)
         )
         # We leave a callback in the products map because we want these Ivy calls
         # to be done lazily (they might never actually get executed) and we want
         # to hit Task.invalidated (called in Task.ivy_resolve) on the instance of
         # BootstrapJvmTools rather than the instance of whatever class requires
         # the bootstrap tools.  It would be awkward and possibly incorrect to call
         # self.invalidated twice on a Task that does meaningful invalidation on its
         # targets. -pl
         for jvm_tool in registered_tools:
             dep_spec = jvm_tool.dep_spec(self.context.options)
             callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool)
             callback_product_map[jvm_tool.scope][jvm_tool.key] = callback

    def _alternate_target_roots(cls, options, address_mapper, build_graph):
        processed = set()
        for jvm_tool in JvmToolMixin.get_registered_tools():
            dep_spec = jvm_tool.dep_spec(options)
            dep_address = Address.parse(dep_spec)
            # Some JVM tools are requested multiple times, we only need to handle them once.
            if dep_address not in processed:
                processed.add(dep_address)
                try:
                    if build_graph.contains_address(dep_address) or address_mapper.resolve(dep_address):
                        # The user has defined a tool classpath override - we let that stand.
                        continue
                except AddressLookupError as e:
                    if jvm_tool.classpath is None:
                        raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
                    else:
                        if jvm_tool.classpath:
                            tool_classpath_target = JarLibrary(
                                name=dep_address.target_name,
                                address=dep_address,
                                build_graph=build_graph,
                                jars=jvm_tool.classpath,
                            )
                        else:
                            # The tool classpath is empty by default, so we just inject a dummy target that
                            # ivy resolves as the empty list classpath.  JarLibrary won't do since it requires
                            # one or more jars, so we just pick a target type ivy has no resolve work to do for.
                            tool_classpath_target = Target(
                                name=dep_address.target_name, address=dep_address, build_graph=build_graph
                            )
                        build_graph.inject_target(tool_classpath_target)

        # We use the trick of not returning alternate roots, but instead just filling the dep_spec
        # holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
        # no over-riding targets present. This means we do modify the build_graph, but we at least do
        # it at a time in the engine lifecycle cut out for handling that.
        return None