Python bootstrapper.Bootstrapper类代码示例

  def setUp(self):
    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for proper isolation when bootstrapping within the test.
    bootstrap_scope = 'bootstrap_scope'
    self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
    JvmToolMixin.reset_registered_tools()

    # Set some options:

    # 1. Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    #    -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    #    tool, constructing a fat jar and then shading that fat jar.
    #
    # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
    #    each have to resolve and shade them every single time, which is a huge slowdown.
    #    Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
    #    tests to write to it.
    #
    # Note that we don't have access to the invoking pants instance's options, so we assume that
    # its artifact cache is in the standard location.  If it isn't, worst case the tests will
    # populate a second cache at the standard location, which is no big deal.
    # TODO: We really need a straightforward way for pants's own tests to get to the enclosing
    # pants instance's options values.
    artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')]
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
    self.set_options_for_scope('cache.{}'.format(bootstrap_scope),
                               read_from=artifact_caches,
                               write_to=artifact_caches)

    # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
    # into our test workspace.
    shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)

    Bootstrapper.reset_instance()

  def execute_junit_runner(self, content):

    # Create the temporary base test directory
    test_rel_path = 'tests/java/org/pantsbuild/foo'
    test_abs_path = os.path.join(self.build_root, test_rel_path)
    self.create_dir(test_rel_path)

    # Generate the temporary java test source code.
    test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
    test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
    self.create_file(test_java_file_rel_path, content)

    # Invoke ivy to resolve classpath for junit.
    classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
    with subsystem_instance(IvySubsystem) as ivy_subsystem:
      distribution = DistributionLocator.cached(jdk=True)
      ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
      ivy.execute(args=['-cachepath', classpath_file_abs_path,
                        '-dependency', 'junit', 'junit-dep', '4.10'],
                  executor=SubprocessExecutor(distribution=distribution))

    with open(classpath_file_abs_path) as fp:
      classpath = fp.read()

    # Now directly invoking javac to compile the test java code into java class
    # so later we can inject the class into products mapping for JUnitRun to execute
    # the test on.
    javac = distribution.binary('javac')
    subprocess.check_call(
      [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])

    # Create a java_tests target and a synthetic resource target.
    java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
    resources = self.make_target('some_resources', Resources)

    # Set the context with the two targets, one java_tests target and
    # one synthetic resources target.
    # The synthetic resources target is to make sure we won't regress
    # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
    # in that bug, the resources target must be the first one in the list.
    context = self.context(target_roots=[resources, java_tests])

    # Before we run the task, we need to inject the "classes_by_target" with
    # the compiled test java classes that JUnitRun will know which test
    # classes to execute. In a normal run, this "classes_by_target" will be
    # populated by java compiling step.
    class_products = context.products.get_data(
      'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
    java_tests_products = MultipleRootedProducts()
    java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
    class_products[java_tests] = java_tests_products

    # Also we need to add the FooTest.class's classpath to the compile_classpath
    # products data mapping so JUnitRun will be able to add that into the final
    # classpath under which the junit will be executed.
    self.populate_compile_classpath(context=context, classpath=[test_abs_path])

    # Finally execute the task.
    self.execute(context)

  def execute_junit_runner(self, content, create_some_resources=True, **kwargs):
    # Create the temporary base test directory
    test_rel_path = 'tests/java/org/pantsbuild/foo'
    test_abs_path = self.create_dir(test_rel_path)

    # Generate the temporary java test source code.
    test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
    test_java_file_abs_path = self.create_file(test_java_file_rel_path, content)

    # Create the temporary classes directory under work dir
    test_classes_abs_path = self.create_workdir_dir(test_rel_path)

    # Invoke ivy to resolve classpath for junit.
    classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
    ivy_subsystem = global_subsystem_instance(IvySubsystem)
    distribution = DistributionLocator.cached(jdk=True)
    ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
    ivy.execute(args=['-cachepath', classpath_file_abs_path,
                      '-dependency', 'junit', 'junit-dep', '4.10'],
                executor=SubprocessExecutor(distribution=distribution))

    with open(classpath_file_abs_path) as fp:
      classpath = fp.read()

    # Now directly invoking javac to compile the test java code into java class
    # so later we can inject the class into products mapping for JUnitRun to execute
    # the test on.
    javac = distribution.binary('javac')
    subprocess.check_call(
      [javac, '-d', test_classes_abs_path, '-cp', classpath, test_java_file_abs_path])

    # If a target_name is specified create a target with it, otherwise create a java_tests target.
    if 'target_name' in kwargs:
      target = self.target(kwargs['target_name'])
    else:
      target = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])

    target_roots = []
    if create_some_resources:
      # Create a synthetic resource target.
      target_roots.append(self.make_target('some_resources', Resources))
    target_roots.append(target)

    # Set the context with the two targets, one java_tests target and
    # one synthetic resources target.
    # The synthetic resources target is to make sure we won't regress
    # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
    # in that bug, the resources target must be the first one in the list.
    context = self.context(target_roots=target_roots)

    # Before we run the task, we need to inject the "runtime_classpath" with
    # the compiled test java classes that JUnitRun will know which test
    # classes to execute. In a normal run, this "runtime_classpath" will be
    # populated by java compilation step.
    self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path])

    # Finally execute the task.
    self.execute(context)

 def test_simple(self):
   with subsystem_instance(IvySubsystem) as ivy_subsystem:
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir,
                                       'tools', 'jvm', 'ivy', 'bootstrap.jar')
     self.assertTrue(os.path.exists(bootstrap_jar_path))

 def test_fresh_bootstrap(self):
   with temporary_dir() as fresh_bootstrap_dir:
     self.set_bootstrap_options(pants_bootstrapdir=fresh_bootstrap_dir)
     # Initialize the Ivy subsystem
     self.context()
     Bootstrapper.default_ivy()
     bootstrap_jar_path = os.path.join(fresh_bootstrap_dir,
                                       'tools', 'jvm', 'ivy', 'bootstrap.jar')
     self.assertTrue(os.path.exists(bootstrap_jar_path))

  def test_parse_proxy_string(self):
    bootstrapper = Bootstrapper().instance()

    self.assertEquals(('example.com', 1234),
                      bootstrapper._parse_proxy_string('http://example.com:1234'))
    self.assertEquals(('secure-example.com', 999),
                      bootstrapper._parse_proxy_string('http://secure-example.com:999'))
    # trailing slash is ok
    self.assertEquals(('example.com', 1234),
                      bootstrapper._parse_proxy_string('http://example.com:1234/'))

 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type())
   # We locate the workdir below the pants_workdir, which BaseTest locates within
   # the BuildRoot.
   self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir)
   self._test_workdir = os.path.join(self._tmpdir, 'workdir')
   os.mkdir(self._test_workdir)
   Bootstrapper.reset_instance()
   IvySubsystem.reset_global_instance()

 def test_simple(self):
     ivy_subsystem = IvySubsystem.global_instance()
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(
         ivy_subsystem.get_options().pants_bootstrapdir, "tools", "jvm", "ivy", "bootstrap.jar"
     )
     self.assertTrue(os.path.exists(bootstrap_jar_path))

 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type())
   # We locate the workdir below the pants_workdir, which BaseTest locates within
   # the BuildRoot.
   self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir)
   self._test_workdir = os.path.join(self._tmpdir, 'workdir')
   os.mkdir(self._test_workdir)
   # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
   # action in this non-JVM-specific, high-level base class.
   Bootstrapper.reset_instance()

 def _temp_ivy_cache_dir(self):
   # We also reset the Bootstrapper since it hangs on to a ivy subsystem.
   old_instance = Bootstrapper._INSTANCE
   try:
     with temporary_dir() as ivy_cache_dir:
       Bootstrapper.reset_instance()
       # We set a temp ivy cache to ensure that the ivy we're using won't cache the temporary jar.
       self.set_options_for_scope('ivy', cache_dir=ivy_cache_dir)
       yield ivy_cache_dir
   finally:
     Bootstrapper._INSTANCE = old_instance

 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope)
   # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot.
   # BaseTest cleans this up, so we don't need to.  We give it a stable name, so that we can
   # use artifact caching to speed up tests.
   self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name())
   os.mkdir(self._test_workdir)
   # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
   # action in this non-JVM-specific, high-level base class.
   Bootstrapper.reset_instance()

    def execute_junit_runner(self, content):
        # Create the temporary base test directory
        test_rel_path = "tests/java/org/pantsbuild/foo"
        test_abs_path = self.create_dir(test_rel_path)

        # Generate the temporary java test source code.
        test_java_file_rel_path = os.path.join(test_rel_path, "FooTest.java")
        test_java_file_abs_path = self.create_file(test_java_file_rel_path, content)

        # Create the temporary classes directory under work dir
        test_classes_abs_path = self.create_workdir_dir(test_rel_path)

        # Invoke ivy to resolve classpath for junit.
        classpath_file_abs_path = os.path.join(test_abs_path, "junit.classpath")
        with subsystem_instance(IvySubsystem) as ivy_subsystem:
            distribution = DistributionLocator.cached(jdk=True)
            ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
            ivy.execute(
                args=["-cachepath", classpath_file_abs_path, "-dependency", "junit", "junit-dep", "4.10"],
                executor=SubprocessExecutor(distribution=distribution),
            )

        with open(classpath_file_abs_path) as fp:
            classpath = fp.read()

        # Now directly invoking javac to compile the test java code into java class
        # so later we can inject the class into products mapping for JUnitRun to execute
        # the test on.
        javac = distribution.binary("javac")
        subprocess.check_call([javac, "-d", test_classes_abs_path, "-cp", classpath, test_java_file_abs_path])

        # Create a java_tests target and a synthetic resource target.
        java_tests = self.create_library(test_rel_path, "java_tests", "foo_test", ["FooTest.java"])
        resources = self.make_target("some_resources", Resources)

        # Set the context with the two targets, one java_tests target and
        # one synthetic resources target.
        # The synthetic resources target is to make sure we won't regress
        # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
        # in that bug, the resources target must be the first one in the list.
        context = self.context(target_roots=[resources, java_tests])

        # Before we run the task, we need to inject the "runtime_classpath" with
        # the compiled test java classes that JUnitRun will know which test
        # classes to execute. In a normal run, this "runtime_classpath" will be
        # populated by java compilation step.
        self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path])

        # Finally execute the task.
        self.execute(context)

          def publish(ivyxml_path):
            try:
              ivy = Bootstrapper.default_ivy()
            except Bootstrapper.Error as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))

            ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
            args = [
              '-settings', ivysettings,
              '-ivy', ivyxml_path,
              '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
              '-publish', resolver,
              '-publishpattern', '%s/[organisation]/[module]/'
                                 '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
              '-revision', newver.version(),
              '-m2compatible',
            ]

            if LogOptions.stderr_log_level() == logging.DEBUG:
              args.append('-verbose')

            if self.snapshot:
              args.append('-overwrite')

            try:
              ivy.execute(jvm_options=jvm_args, args=args,
                          workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
            except Ivy.Error as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))

  def publish(self, ivyxml_path, jar, entry, repo, published):
    """Run ivy to publish a jar.  ivyxml_path is the path to the ivy file; published
    is a list of jars published so far (including this one). entry is a pushdb entry."""
    jvm_args = self._ivy_jvm_args(repo)
    resolver = repo['resolver']
    path = repo.get('path')

    try:
      ivy = Bootstrapper.default_ivy()
    except Bootstrapper.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))

    ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
    args = [
      '-settings', ivysettings,
      '-ivy', ivyxml_path,
      '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
      '-publish', resolver,
      '-publishpattern', '%s/[organisation]/[module]/'
                         '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
      '-revision', entry.version().version(),
      '-m2compatible',
    ]

    if LogOptions.stderr_log_level() == logging.DEBUG:
      args.append('-verbose')

    if self.local_snapshot:
      args.append('-overwrite')

    try:
      ivy.execute(jvm_options=jvm_args, args=args,
                  workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
    except Ivy.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))

  def exec_ivy(cls, ivy, confs, ivyxml, args,
               jvm_options,
               executor,
               workunit_name,
               workunit_factory):
    """
    :API: public
    """
    ivy = ivy or Bootstrapper.default_ivy()

    ivy_args = ['-ivy', ivyxml]
    ivy_args.append('-confs')
    ivy_args.extend(confs)
    ivy_args.extend(args)

    ivy_jvm_options = list(jvm_options)
    # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
    ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

    runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
    try:
      with ivy.resolution_lock:
        result = execute_runner(runner, workunit_factory=workunit_factory,
                                workunit_name=workunit_name)
      if result != 0:
        raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
    except runner.executor.Error as e:
      raise IvyUtils.IvyError(e)