Python executor.SubprocessExecutor类代码示例

  def create_scaladoc_command(self, classpath, gendir, *targets):
    sources = []
    for target in targets:
      sources.extend(target.sources_relative_to_buildroot())
      # TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
      # A '.scala' owning target may not have java_sources, eg: junit_tests
      if hasattr(target, 'java_sources'):
        for java_target in target.java_sources:
          sources.extend(java_target.sources_relative_to_buildroot())

    if not sources:
      return None

    scala_platform = ScalaPlatform.global_instance()
    tool_classpath = [cp_entry.path for cp_entry in scala_platform.compiler_classpath_entries(
      self.context.products, self.context._scheduler)]

    args = ['-usejavacp',
            '-classpath', ':'.join(classpath),
            '-d', gendir]

    args.extend(self.args)

    args.extend(sources)

    java_executor = SubprocessExecutor(DistributionLocator.cached())
    runner = java_executor.runner(jvm_options=self.jvm_options,
                                  classpath=tool_classpath,
                                  main='scala.tools.nsc.ScalaDoc',
                                  args=args)
    return runner.command

  def create_scaladoc_command(self, classpath, gendir, *targets):
    sources = []
    for target in targets:
      sources.extend(target.sources_relative_to_buildroot())
      # TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
      for java_target in target.java_sources:
        sources.extend(java_target.sources_relative_to_buildroot())

    if not sources:
      return None

    scala_platform = ScalaPlatform.global_instance()
    tool_classpath = scala_platform.compiler_classpath(self.context.products)

    args = ['-usejavacp',
            '-classpath', ':'.join(classpath),
            '-d', gendir]

    args.extend(self.args)

    args.extend(sources)

    java_executor = SubprocessExecutor()
    runner = java_executor.runner(jvm_options=self.jvm_options,
                                  classpath=tool_classpath,
                                  main='scala.tools.nsc.ScalaDoc',
                                  args=args)
    return runner.command

 def execute_tool(self, classpath, main, args=None):
   init_subsystem(DistributionLocator)
   executor = SubprocessExecutor(DistributionLocator.cached())
   process = executor.spawn(classpath, main, args=args,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   out, err = process.communicate()
   self.assertEqual(0, process.returncode)
   self.assertEqual('', err.strip())
   yield out

 def assert_run_ant_version(classpath):
   with subsystem_instance(DistributionLocator):
     executor = SubprocessExecutor(DistributionLocator.cached())
     process = executor.spawn(classpath, 'org.apache.tools.ant.Main', args=['-version'],
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = process.communicate()
     self.assertEqual(0, process.returncode)
     self.assertTrue(out.strip().startswith('Apache Ant(TM) version 1.9.4'))
     self.assertEqual('', err.strip())

 def do_test_jre_env_var(self, env_var, env_value, scrubbed=True):
   with self.jre(env_var=env_var) as jre:
     executor = SubprocessExecutor(Distribution(bin_path=jre))
     with environment_as(**{env_var: env_value}):
       self.assertEqual(env_value, os.getenv(env_var))
       process = executor.spawn(classpath=['dummy/classpath'],
                                main='dummy.main',
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
       _, stderr = process.communicate()
       self.assertEqual(0, process.returncode)
       self.assertEqual('' if scrubbed else env_value, stderr.strip())

  def create_javadoc_command(self, classpath, gendir, *targets):
    sources = []
    for target in targets:
      sources.extend(target.sources_relative_to_buildroot())

    if not sources:
      return None

    # Without a JDK/tools.jar we have no javadoc tool and cannot proceed, so check/acquire early.
    jdk = DistributionLocator.cached(jdk=True)
    tool_classpath = jdk.find_libs(['tools.jar'])

    args = ['-quiet',
            '-encoding', 'UTF-8',
            '-notimestamp',
            '-use',
            '-Xmaxerrs', '10000', # the default is 100
            '-Xmaxwarns', '10000', # the default is 100
            '-d', gendir]

    # Always provide external linking for java API
    offlinelinks = {'http://download.oracle.com/javase/8/docs/api/'}

    def link(target):
      for jar in target.jar_dependencies:
        if jar.apidocs:
          offlinelinks.add(jar.apidocs)
    for target in targets:
      target.walk(link, lambda t: isinstance(t, (JvmTarget, JarLibrary)))

    for link in offlinelinks:
      args.extend(['-linkoffline', link, link])

    args.extend(self.args)

    javadoc_classpath_file = os.path.join(self.workdir, '{}.classpath'.format(os.path.basename(gendir)))
    with open(javadoc_classpath_file, 'w') as f:
      f.write('-classpath ')
      f.write(':'.join(classpath))
    args.extend(['@{}'.format(javadoc_classpath_file)])

    javadoc_sources_file = os.path.join(self.workdir, '{}.source.files'.format(os.path.basename(gendir)))
    with open(javadoc_sources_file, 'w') as f:
      f.write('\n'.join(sources))
    args.extend(['@{}'.format(javadoc_sources_file)])

    java_executor = SubprocessExecutor(jdk)
    runner = java_executor.runner(jvm_options=self.jvm_options,
                                  classpath=tool_classpath,
                                  main='com.sun.tools.javadoc.Main',
                                  args=args)
    return runner.command

  def post_fork_child(self, fingerprint, jvm_options, classpath, stdout, stderr):
    """Post-fork() child callback for ProcessManager.daemon_spawn()."""
    java = SubprocessExecutor(self._distribution)

    subproc = java.spawn(classpath=classpath,
                         main='com.martiansoftware.nailgun.NGServer',
                         jvm_options=jvm_options,
                         args=[':0'],
                         stdin=safe_open('/dev/null', 'r'),
                         stdout=safe_open(self._ng_stdout, 'w'),
                         stderr=safe_open(self._ng_stderr, 'w'),
                         close_fds=True)

    self.write_pid(subproc.pid)

  def _bundle_and_run(self, bundle_args, classpath):
    self.assert_success(self.run_pants(['clean-all']))
    pants_command = list(bundle_args)
    pants_command.append('testprojects/src/java/org/pantsbuild/testproject/shading:third')
    self.assert_success(self.run_pants(pants_command))

    main_class = 'org.pantsbuild.testproject.shading.Third'
    with subsystem_instance(DistributionLocator):
      executor = SubprocessExecutor(DistributionLocator.cached(minimum_version='1.7'))
      p = executor.spawn(classpath, main_class, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      out, err = p.communicate()
      self.assertEqual(0, p.returncode, err)
      class_names = json.loads(out.strip())
      self.assertEqual({
        'Gson': 'moc.elgoog.nosg.Gson',
        'Third': 'org.pantsbuild.testproject.shading.Third',
        'Second': 'hello.org.pantsbuild.testproject.shading.Second',
      }, class_names)

  def create_javadoc_command(self, classpath, gendir, *targets):
    sources = []
    for target in targets:
      sources.extend(target.sources_relative_to_buildroot())

    if not sources:
      return None

    # Without a JDK/tools.jar we have no javadoc tool and cannot proceed, so check/acquire early.
    jdk = DistributionLocator.cached(jdk=True)
    tool_classpath = jdk.find_libs(['tools.jar'])

    args = ['-quiet',
            '-encoding', 'UTF-8',
            '-notimestamp',
            '-use',
            '-classpath', ':'.join(classpath),
            '-d', gendir]

    # Always provide external linking for java API
    offlinelinks = {'http://download.oracle.com/javase/6/docs/api/'}

    def link(target):
      for jar in target.jar_dependencies:
        if jar.apidocs:
          offlinelinks.add(jar.apidocs)
    for target in targets:
      target.walk(link, lambda t: t.is_jvm)

    for link in offlinelinks:
      args.extend(['-linkoffline', link, link])

    args.extend(self.args)

    args.extend(sources)

    java_executor = SubprocessExecutor(jdk)
    runner = java_executor.runner(jvm_options=self.jvm_options,
                                  classpath=tool_classpath,
                                  main='com.sun.tools.javadoc.Main',
                                  args=args)
    return runner.command

  def _spawn_nailgun_server(self, fingerprint, jvm_options, classpath, stdout, stderr):
    logger.debug('No ng server found with fingerprint {fingerprint}, spawning...'
                 .format(fingerprint=fingerprint))

    with safe_open(self._ng_out, 'w'):
      pass  # truncate

    pid = os.fork()
    if pid != 0:
      # In the parent tine - block on ng being up for connections
      return self._await_nailgun_server(stdout, stderr,
                                        'jvm_options={jvm_options} classpath={classpath}'
                                        .format(jvm_options=jvm_options, classpath=classpath))


    os.setsid()
    in_fd = open('/dev/null', 'r')
    out_fd = safe_open(self._ng_out, 'w')
    err_fd = safe_open(self._ng_err, 'w')

    java = SubprocessExecutor(self._distribution)

    jvm_options = jvm_options + [self._PANTS_NG_ARG,
                           self.create_owner_arg(self._workdir),
                           self._create_fingerprint_arg(fingerprint)]

    process = java.spawn(classpath=classpath,
                         main='com.martiansoftware.nailgun.NGServer',
                         jvm_options=jvm_options,
                         args=[':0'],
                         stdin=in_fd,
                         stdout=out_fd,
                         stderr=err_fd,
                         close_fds=True)

    logger.debug('Spawned ng server with fingerprint {fingerprint} @ {pid}'
                 .format(fingerprint=fingerprint, pid=process.pid))
    # Prevents finally blocks and atexit handlers from being executed, unlike sys.exit(). We
    # don't want to execute finally blocks because we might, e.g., clean up tempfiles that the
    # parent still needs.
    os._exit(0)

#.........这里部分代码省略.........

    if self._fail_fast:
      self._args.append('-fail-fast')
    self._args.append('-outdir')
    self._args.append(self.workdir)

    if options.per_test_timer:
      self._args.append('-per-test-timer')
    if options.default_parallel:
      self._args.append('-default-parallel')
    self._args.append('-parallel-threads')
    self._args.append(str(options.parallel_threads))

    if options.test_shard:
      self._args.append('-test-shard')
      self._args.append(options.test_shard)

    self._executor = None

  def preferred_jvm_distribution_for_targets(self, targets):
    return self.preferred_jvm_distribution([target.platform for target in targets
                                            if isinstance(target, JvmTarget)])

  def preferred_jvm_distribution(self, platforms):
    """Returns a jvm Distribution with a version that should work for all the platforms."""
    if not platforms:
      return DistributionLocator.cached()
    min_version = max(platform.target_level for platform in platforms)
    max_version = Revision(*(min_version.components + [9999])) if self._strict_jvm_version else None
    return DistributionLocator.cached(minimum_version=min_version, maximum_version=max_version)

  def execute_java_for_targets(self, targets, executor=None, *args, **kwargs):
    distribution = self.preferred_jvm_distribution_for_targets(targets)
    self._executor = executor or SubprocessExecutor(distribution)
    return distribution.execute_java(*args, executor=self._executor, **kwargs)

  def _collect_test_targets(self, targets):
    """Returns a mapping from test names to target objects for all tests that
    are included in targets. If self._tests_to_run is set, return {test: None}
    for these tests instead.
    """

    tests_from_targets = dict(list(self._calculate_tests_from_targets(targets)))

    if targets and self._tests_to_run:
      # If there are some junit_test targets in the graph, find ones that match the requested
      # test(s).
      tests_with_targets = {}
      unknown_tests = []
      for test in self._get_tests_to_run():
        # A test might contain #specific_method, which is not needed to find a target.
        test_class_name = test.partition('#')[0]
        target = tests_from_targets.get(test_class_name)
        if target is None:
          unknown_tests.append(test)
        else:
          tests_with_targets[test] = target

      if len(unknown_tests) > 0:
        raise TaskError("No target found for test specifier(s):\n\n  '{}'\n\nPlease change " \
                        "specifier or bring in the proper target(s)."
                        .format("'\n  '".join(unknown_tests)))

      return tests_with_targets
    else:
      return tests_from_targets

  def _run_tests(self, tests_to_targets):

    if self._coverage:
      extra_jvm_options = self._coverage.extra_jvm_options
      classpath_prepend = self._coverage.classpath_prepend
      classpath_append = self._coverage.classpath_append
    else:
      extra_jvm_options = []
      classpath_prepend = ()
      classpath_append = ()

    tests_by_properties = self._tests_by_properties(tests_to_targets,
                                                    self._infer_workdir,
                                                    lambda target: target.test_platform)

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for (workdir, platform), tests in tests_by_properties.items():
      for (target_jvm_options, target_tests) in self._partition_by_jvm_options(tests_to_targets,
                                                                               tests):
        for batch in self._partition(target_tests):
          # Batches of test classes will likely exist within the same targets: dedupe them.
          relevant_targets = set(map(tests_to_targets.get, batch))
          complete_classpath = OrderedSet()
          complete_classpath.update(classpath_prepend)
          complete_classpath.update(self.tool_classpath('junit'))
          complete_classpath.update(self.classpath(relevant_targets,
                                                   classpath_product=classpath_product))
          complete_classpath.update(classpath_append)
          distribution = self.preferred_jvm_distribution([platform])
          with binary_util.safe_args(batch, self.get_options()) as batch_tests:
            self.context.log.debug('CWD = {}'.format(workdir))
            self.context.log.debug('platform = {}'.format(platform))
            self._executor = SubprocessExecutor(distribution)
            result += abs(distribution.execute_java(
              executor=self._executor,
              classpath=complete_classpath,
              main=JUnitRun._MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + target_jvm_options,
              args=self._args + batch_tests + [u'-xmlreport'],
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=workdir,
              synthetic_jar_dir=self.workdir,
            ))

            if result != 0 and self._fail_fast:
              break

    if result != 0:
      failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
      failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
          for test in sorted(failed_targets_and_tests[target]):
            error_message_lines.append('{0}{1}'.format(' '*8, test))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))