Python jvm_dependency_analyzer.JvmDependencyAnalyzer类代码示例

  def __init__(self, context, options, workdir, analysis_tools, language, sources_predicate):
    super(JvmCompileGlobalStrategy, self).__init__(context, options, workdir, analysis_tools,
                                                   language, sources_predicate)

    # Various working directories.
    # NB: These are grandfathered in with non-strategy-specific names, but to prevent
    # collisions within the buildcache, strategies should use strategy-specific subdirectories.
    self._analysis_dir = os.path.join(workdir, 'analysis')
    self._classes_dir = os.path.join(workdir, 'classes')

    self._delete_scratch = options.delete_scratch

    self._analysis_file = os.path.join(self._analysis_dir, 'global_analysis.valid')
    self._invalid_analysis_file = os.path.join(self._analysis_dir, 'global_analysis.invalid')

    self._target_sources_dir = os.path.join(workdir, 'target_sources')

    # A temporary, but well-known, dir in which to munge analysis/dependency files in before
    # caching. It must be well-known so we know where to find the files when we retrieve them from
    # the cache.
    self._analysis_tmpdir = os.path.join(self._analysis_dir, 'artifact_cache_tmpdir')

    # The rough number of source files to build in each compiler pass.
    self._partition_size_hint = options.partition_size_hint

    # Set up dep checking if needed.
    def munge_flag(flag):
      flag_value = getattr(options, flag, None)
      return None if flag_value == 'off' else flag_value

    check_missing_deps = munge_flag('missing_deps')
    check_missing_direct_deps = munge_flag('missing_direct_deps')
    check_unnecessary_deps = munge_flag('unnecessary_deps')

    if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
      target_whitelist = options.missing_deps_whitelist
      # Must init it here, so it can set requirements on the context.
      self._dep_analyzer = JvmDependencyAnalyzer(self.context,
                                                 check_missing_deps,
                                                 check_missing_direct_deps,
                                                 check_unnecessary_deps,
                                                 target_whitelist)
    else:
      self._dep_analyzer = None

    # Computed lazily as needed.
    self._upstream_class_to_path = None

    # If non-zero, and we have fewer than this number of locally-changed targets,
    # then we partition them separately, to preserve stability in the face of repeated
    # compilations.
    self._changed_targets_heuristic_limit = options.changed_targets_heuristic_limit

    # Sources (relative to buildroot) present in the last analysis that have since been deleted.
    # Populated in prepare_compile().
    self._deleted_sources = None

  def prepare(cls, options, round_manager):
    super(JvmCompile, cls).prepare(options, round_manager)

    # This task uses JvmDependencyAnalyzer as a helper, get its product needs
    JvmDependencyAnalyzer.prepare(options, round_manager)

    round_manager.require_data('compile_classpath')
    round_manager.require_data('ivy_resolve_symlink_map')

    # Require codegen we care about
    # TODO(John Sirois): roll this up in Task - if the list of labels we care about for a target
    # predicate to filter the full build graph is exposed, the requirement can be made automatic
    # and in turn codegen tasks could denote the labels they produce automating wiring of the
    # produce side
    round_manager.require_data('java')
    round_manager.require_data('scala')

    # Allow the deferred_sources_mapping to take place first
    round_manager.require_data('deferred_sources')

#.........这里部分代码省略.........
        self._invalid_analysis_file = os.path.join(self._analysis_dir, "global_analysis.invalid")

        # A temporary, but well-known, dir in which to munge analysis/dependency files in before
        # caching. It must be well-known so we know where to find the files when we retrieve them from
        # the cache.
        self._analysis_tmpdir = os.path.join(self._analysis_dir, "artifact_cache_tmpdir")

        # We can't create analysis tools until after construction.
        self._lazy_analysis_tools = None

        # The rough number of source files to build in each compiler pass.
        self._partition_size_hint = self._get_lang_specific_option("partition_size_hint")
        if self._partition_size_hint == -1:
            self._partition_size_hint = self.context.config.getint(config_section, "partition_size_hint", default=1000)

        # JVM options for running the compiler.
        self._jvm_options = self.context.config.getlist(config_section, "jvm_args")

        # The ivy confs for which we're building.
        self._confs = self.context.config.getlist(config_section, "confs", default=["default"])

        # Set up dep checking if needed.
        def munge_flag(flag):
            return None if flag == "off" else flag

        check_missing_deps = munge_flag(self._get_lang_specific_option("missing_deps"))
        check_missing_direct_deps = munge_flag(self._get_lang_specific_option("missing_direct_deps"))
        check_unnecessary_deps = munge_flag(self._get_lang_specific_option("unnecessary_deps"))

        if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
            target_whitelist = self.context.config.getlist("jvm", "missing_deps_target_whitelist", default=[])

            # Must init it here, so it can set requirements on the context.
            self._dep_analyzer = JvmDependencyAnalyzer(
                self.context, check_missing_deps, check_missing_direct_deps, check_unnecessary_deps, target_whitelist
            )
        else:
            self._dep_analyzer = None

        # If non-zero, and we have fewer than this number of locally-changed targets,
        # then we partition them separately, to preserve stability in the face of repeated
        # compilations.
        self._locally_changed_targets_heuristic_limit = self.context.config.getint(
            config_section, "locally_changed_targets_heuristic_limit", 0
        )

        self._upstream_class_to_path = None  # Computed lazily as needed.
        self.setup_artifact_cache_from_config(config_section=config_section)

        # Sources (relative to buildroot) present in the last analysis that have since been deleted.
        # Populated in prepare_execute().
        self._deleted_sources = None

        # Map of target -> list of sources (relative to buildroot), for all targets in all chunks.
        # Populated in prepare_execute().
        self._sources_by_target = None

    def configure_args(self, args_defaults=[], warning_defaults=[], no_warning_defaults=[]):
        """
   Setup the compiler command line arguments, optionally providing default values.  It is mandatory
   to call this from __init__() of your subclass.
   :param list args_default:  compiler flags that should be invoked for all invocations
   :param list warning_defaults: compiler flags to turn on warnings
   :param list no_warning_defaults:  compiler flags to turn off all warnings
   """
        self._args = self.context.config.getlist(self._config_section, "args", default=args_defaults)

    def __init__(self, *args, **kwargs):
        super(JvmCompile, self).__init__(*args, **kwargs)
        config_section = self.config_section

        # Global workdir.
        self._pants_workdir = self.context.config.getdefault("pants_workdir")

        # Various working directories.
        self._classes_dir = os.path.join(self.workdir, "classes")
        self._resources_dir = os.path.join(self.workdir, "resources")
        self._analysis_dir = os.path.join(self.workdir, "analysis")
        self._target_sources_dir = os.path.join(self.workdir, "target_sources")

        self._delete_scratch = self._get_lang_specific_option("delete_scratch")

        safe_mkdir(self._classes_dir)
        safe_mkdir(self._analysis_dir)
        safe_mkdir(self._target_sources_dir)

        self._analysis_file = os.path.join(self._analysis_dir, "global_analysis.valid")
        self._invalid_analysis_file = os.path.join(self._analysis_dir, "global_analysis.invalid")

        # A temporary, but well-known, dir in which to munge analysis/dependency files in before
        # caching. It must be well-known so we know where to find the files when we retrieve them from
        # the cache.
        self._analysis_tmpdir = os.path.join(self._analysis_dir, "artifact_cache_tmpdir")

        # We can't create analysis tools until after construction.
        self._lazy_analysis_tools = None

        # The rough number of source files to build in each compiler pass.
        self._partition_size_hint = self._get_lang_specific_option("partition_size_hint")
        if self._partition_size_hint == -1:
            self._partition_size_hint = self.context.config.getint(config_section, "partition_size_hint", default=1000)

        # JVM options for running the compiler.
        self._jvm_options = self.context.config.getlist(config_section, "jvm_args")

        # The ivy confs for which we're building.
        self._confs = self.context.config.getlist(config_section, "confs", default=["default"])

        # Set up dep checking if needed.
        def munge_flag(flag):
            return None if flag == "off" else flag

        check_missing_deps = munge_flag(self._get_lang_specific_option("missing_deps"))
        check_missing_direct_deps = munge_flag(self._get_lang_specific_option("missing_direct_deps"))
        check_unnecessary_deps = munge_flag(self._get_lang_specific_option("unnecessary_deps"))

        if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
            target_whitelist = self.context.config.getlist("jvm", "missing_deps_target_whitelist", default=[])

            # Must init it here, so it can set requirements on the context.
            self._dep_analyzer = JvmDependencyAnalyzer(
                self.context, check_missing_deps, check_missing_direct_deps, check_unnecessary_deps, target_whitelist
            )
        else:
            self._dep_analyzer = None

        # If non-zero, and we have fewer than this number of locally-changed targets,
        # then we partition them separately, to preserve stability in the face of repeated
        # compilations.
        self._locally_changed_targets_heuristic_limit = self.context.config.getint(
            config_section, "locally_changed_targets_heuristic_limit", 0
        )

        self._upstream_class_to_path = None  # Computed lazily as needed.
        self.setup_artifact_cache_from_config(config_section=config_section)

        # Sources (relative to buildroot) present in the last analysis that have since been deleted.
        # Populated in prepare_execute().
        self._deleted_sources = None

        # Map of target -> list of sources (relative to buildroot), for all targets in all chunks.
        # Populated in prepare_execute().
        self._sources_by_target = None

class JvmCompileGlobalStrategy(JvmCompileStrategy):
  """A strategy for JVM compilation that uses a global classpath and analysis."""

  class InternalTargetPartitioningError(Exception):
    """Error partitioning targets by jvm platform settings."""

  @classmethod
  def register_options(cls, register, compile_task_name, supports_concurrent_execution):
    register('--missing-deps', advanced=True, choices=['off', 'warn', 'fatal'], default='warn',
             fingerprint=True,
             help='Check for missing dependencies in code compiled with {0}. Reports actual '
                  'dependencies A -> B where there is no transitive BUILD file dependency path '
                  'from A to B. If fatal, missing deps are treated as a build error.'.format(
               compile_task_name))

    register('--missing-direct-deps', advanced=True, choices=['off', 'warn', 'fatal'],
             default='off',
             fingerprint=True,
             help='Check for missing direct dependencies in code compiled with {0}. Reports actual '
                  'dependencies A -> B where there is no direct BUILD file dependency path from '
                  'A to B. This is a very strict check; In practice it is common to rely on '
                  'transitive, indirect dependencies, e.g., due to type inference or when the main '
                  'target in a BUILD file is modified to depend on other targets in the same BUILD '
                  'file, as an implementation detail. However it may still be useful to use this '
                  'on occasion. '.format(compile_task_name))

    register('--missing-deps-whitelist', advanced=True, type=list_option,
             fingerprint=True,
             help="Don't report these targets even if they have missing deps.")

    register('--unnecessary-deps', advanced=True, choices=['off', 'warn', 'fatal'], default='off',
             fingerprint=True,
             help='Check for declared dependencies in code compiled with {0} that are not needed. '
                  'This is a very strict check. For example, generated code will often '
                  'legitimately have BUILD dependencies that are unused in practice.'.format(
               compile_task_name))

    register('--changed-targets-heuristic-limit', advanced=True, type=int, default=0,
             help='If non-zero, and we have fewer than this number of locally-changed targets, '
                  'partition them separately, to preserve stability when compiling repeatedly.')

  def __init__(self, context, options, workdir, analysis_tools, compile_task_name,
               sources_predicate):
    super(JvmCompileGlobalStrategy, self).__init__(context, options, workdir, analysis_tools,
                                                   compile_task_name, sources_predicate)

    # Various working directories.
    # NB: These are grandfathered in with non-strategy-specific names, but to prevent
    # collisions within the buildcache, strategies should use strategy-specific subdirectories.
    self._analysis_dir = os.path.join(workdir, 'analysis')
    self._classes_dir = os.path.join(workdir, 'classes')

    self._analysis_file = os.path.join(self._analysis_dir, 'global_analysis.valid')
    self._invalid_analysis_file = os.path.join(self._analysis_dir, 'global_analysis.invalid')

    self._target_sources_dir = os.path.join(workdir, 'target_sources')

    # The rough number of source files to build in each compiler pass.
    self._partition_size_hint = options.partition_size_hint

    # Set up dep checking if needed.
    def munge_flag(flag):
      flag_value = getattr(options, flag, None)
      return None if flag_value == 'off' else flag_value

    check_missing_deps = munge_flag('missing_deps')
    check_missing_direct_deps = munge_flag('missing_direct_deps')
    check_unnecessary_deps = munge_flag('unnecessary_deps')

    if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
      target_whitelist = options.missing_deps_whitelist
      # Must init it here, so it can set requirements on the context.
      self._dep_analyzer = JvmDependencyAnalyzer(self.context,
                                                 check_missing_deps,
                                                 check_missing_direct_deps,
                                                 check_unnecessary_deps,
                                                 target_whitelist)
    else:
      self._dep_analyzer = None

    # Computed lazily as needed.
    self._upstream_class_to_path = None

    # If non-zero, and we have fewer than this number of locally-changed targets,
    # then we partition them separately, to preserve stability in the face of repeated
    # compilations.
    self._changed_targets_heuristic_limit = options.changed_targets_heuristic_limit

    # Sources (relative to buildroot) present in the last analysis that have since been deleted.
    # Populated in prepare_compile().
    self._deleted_sources = None

  def name(self):
    return 'global'

  def compile_context(self, target):
    """Returns the default/stable compile context for the given target.

    Temporary compile contexts are private to the strategy.
    """
#.........这里部分代码省略.........

  def __init__(self, context, workdir, minimum_version=None, jdk=False):
    # TODO(John Sirois): XXX plumb minimum_version via config or flags
    super(JvmCompile, self).__init__(context, workdir, minimum_version=minimum_version, jdk=jdk)
    concrete_class = type(self)
    config_section = concrete_class._config_section

    def get_lang_specific_option(opt):
      full_opt_name = self._language + '_' + opt
      return getattr(context.options, full_opt_name, None)

    # Global workdir.
    self._pants_workdir = context.config.getdefault('pants_workdir')

    # Various working directories.
    self._classes_dir = os.path.join(self.workdir, 'classes')
    self._resources_dir = os.path.join(self.workdir, 'resources')
    self._analysis_dir = os.path.join(self.workdir, 'analysis')
    self._target_sources_dir = os.path.join(self.workdir, 'target_sources')

    self._delete_scratch = get_lang_specific_option('delete_scratch')

    safe_mkdir(self._classes_dir)
    safe_mkdir(self._analysis_dir)
    safe_mkdir(self._target_sources_dir)

    self._analysis_file = os.path.join(self._analysis_dir, 'global_analysis.valid')
    self._invalid_analysis_file = os.path.join(self._analysis_dir, 'global_analysis.invalid')

    # A temporary, but well-known, dir in which to munge analysis/dependency files in before
    # caching. It must be well-known so we know where to find the files when we retrieve them from
    # the cache.
    self._analysis_tmpdir = os.path.join(self._analysis_dir, 'artifact_cache_tmpdir')

    # We can't create analysis tools until after construction.
    self._lazy_analysis_tools = None

    # Compiler options.
    self._args = context.config.getlist(config_section, 'args')
    if get_lang_specific_option('compile_warnings'):
      self._args.extend(context.config.getlist(config_section, 'warning_args'))
    else:
      self._args.extend(context.config.getlist(config_section, 'no_warning_args'))

    # The rough number of source files to build in each compiler pass.
    self._partition_size_hint = get_lang_specific_option('partition_size_hint')
    if self._partition_size_hint == -1:
      self._partition_size_hint = context.config.getint(config_section, 'partition_size_hint',
                                                        default=1000)

    # JVM options for running the compiler.
    self._jvm_options = context.config.getlist(config_section, 'jvm_args')

    # The ivy confs for which we're building.
    self._confs = context.config.getlist(config_section, 'confs', default=['default'])

    # Runtime dependencies.
    runtime_deps = context.config.getlist(config_section, 'runtime-deps', default=[])
    if runtime_deps:
      self._runtime_deps_key = self._language + '-runtime-deps'
      self.register_jvm_tool(self._runtime_deps_key, runtime_deps)
    else:
      self._runtime_deps_key = None

    # Set up dep checking if needed.
    def munge_flag(flag):
      return None if flag == 'off' else flag
    check_missing_deps = munge_flag(get_lang_specific_option('missing_deps'))
    check_missing_direct_deps = munge_flag(get_lang_specific_option('missing_direct_deps'))
    check_unnecessary_deps = munge_flag(get_lang_specific_option('unnecessary_deps'))

    if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
      # Must init it here, so it can set requirements on the context.
      self._dep_analyzer = JvmDependencyAnalyzer(self.context,
                                                 check_missing_deps,
                                                 check_missing_direct_deps,
                                                 check_unnecessary_deps)
    else:
      self._dep_analyzer = None

    # If non-zero, and we have fewer than this number of locally-changed targets,
    # then we partition them separately, to preserve stability in the face of repeated
    # compilations.
    self._locally_changed_targets_heuristic_limit = context.config.getint(config_section,
        'locally_changed_targets_heuristic_limit', 0)

    self._class_to_jarfile = None  # Computed lazily as needed.

    self.context.products.require_data('exclusives_groups')
    self.setup_artifact_cache_from_config(config_section=config_section)

    # Sources (relative to buildroot) present in the last analysis that have since been deleted.
    # Populated in prepare_execute().
    self._deleted_sources = None

    # Map of target -> list of sources (relative to buildroot), for all targets in all chunks.
    # Populated in prepare_execute().
    self._sources_by_target = None

  def __init__(self, *args, **kwargs):
    super(JvmCompile, self).__init__(*args, **kwargs)
    config_section = self.config_section

    # Various working directories.
    self._classes_dir = os.path.join(self.workdir, 'classes')
    self._resources_dir = os.path.join(self.workdir, 'resources')
    self._analysis_dir = os.path.join(self.workdir, 'analysis')
    self._target_sources_dir = os.path.join(self.workdir, 'target_sources')

    self._delete_scratch = self.get_options().delete_scratch

    self._analysis_file = os.path.join(self._analysis_dir, 'global_analysis.valid')
    self._invalid_analysis_file = os.path.join(self._analysis_dir, 'global_analysis.invalid')

    # A temporary, but well-known, dir in which to munge analysis/dependency files in before
    # caching. It must be well-known so we know where to find the files when we retrieve them from
    # the cache.
    self._analysis_tmpdir = os.path.join(self._analysis_dir, 'artifact_cache_tmpdir')

    # We can't create analysis tools until after construction.
    self._lazy_analysis_tools = None

    # The rough number of source files to build in each compiler pass.
    self._partition_size_hint = self.get_options().partition_size_hint

    # JVM options for running the compiler.
    self._jvm_options = self.context.config.getlist(config_section, 'jvm_args')

    # The ivy confs for which we're building.
    self._confs = self.context.config.getlist(config_section, 'confs', default=['default'])

    self._args = list(self.get_options().args)
    if self.get_options().warnings:
      self._args.extend(self.get_options().warning_args)
    else:
      self._args.extend(self.get_options().no_warning_args)

    # Set up dep checking if needed.
    def munge_flag(flag):
      flag_value = getattr(self.get_options(), flag, None)
      return None if flag_value == 'off' else flag_value

    check_missing_deps = munge_flag('missing_deps')
    check_missing_direct_deps = munge_flag('missing_direct_deps')
    check_unnecessary_deps = munge_flag('unnecessary_deps')

    if check_missing_deps or check_missing_direct_deps or check_unnecessary_deps:
      target_whitelist = self.context.config.getlist('jvm', 'missing_deps_target_whitelist', default=[])

      # Must init it here, so it can set requirements on the context.
      self._dep_analyzer = JvmDependencyAnalyzer(self.context,
                                                 check_missing_deps,
                                                 check_missing_direct_deps,
                                                 check_unnecessary_deps,
                                                 target_whitelist)
    else:
      self._dep_analyzer = None

    # If non-zero, and we have fewer than this number of locally-changed targets,
    # then we partition them separately, to preserve stability in the face of repeated
    # compilations.
    self._locally_changed_targets_heuristic_limit = self.context.config.getint(config_section,
        'locally_changed_targets_heuristic_limit', 0)

    self._upstream_class_to_path = None  # Computed lazily as needed.
    self.setup_artifact_cache_from_config(config_section=config_section)

    # Sources (relative to buildroot) present in the last analysis that have since been deleted.
    # Populated in prepare_execute().
    self._deleted_sources = None

    # Map of target -> list of sources (relative to buildroot), for all targets in all chunks.
    # Populated in prepare_execute().
    self._sources_by_target = None

class JvmCompile(NailgunTaskBase, GroupMember):
  """A common framework for JVM compilation.

  To subclass for a specific JVM language, implement the static values and methods
  mentioned below under "Subclasses must implement".
  """

  @classmethod
  def register_options(cls, register):
    super(JvmCompile, cls).register_options(register)
    register('--partition-size-hint', type=int, default=sys.maxint, metavar='<# source files>',
             help='Roughly how many source files to attempt to compile together. Set to a large '
                  'number to compile all sources together. Set to 0 to compile target-by-target.')

    register('--jvm-options', type=Options.list,
             help='Run the compiler with these JVM options.')

    register('--args', action='append', default=list(cls.get_args_default(register.bootstrap)),
             help='Pass these args to the compiler.')

    register('--confs', type=Options.list, default=['default'],
             help='Compile for these Ivy confs.')

    register('--warnings', default=True, action='store_true',
             help='Compile with all configured warnings enabled.')

    register('--warning-args', action='append', default=list(cls.get_warning_args_default()),
             help='Extra compiler args to use when warnings are enabled.')

    register('--no-warning-args', action='append', default=list(cls.get_no_warning_args_default()),
             help='Extra compiler args to use when warnings are disabled.')

    register('--missing-deps', choices=['off', 'warn', 'fatal'], default='warn',
             help='Check for missing dependencies in {0} code. Reports actual dependencies A -> B '
                  'where there is no transitive BUILD file dependency path from A to B. If fatal, '
                  'missing deps are treated as a build error.'.format(cls._language))

    register('--missing-direct-deps', choices=['off', 'warn', 'fatal'], default='off',
             help='Check for missing direct dependencies in {0} code. Reports actual dependencies '
                  'A -> B where there is no direct BUILD file dependency path from A to B. This is '
                  'a very strict check; In practice it is common to rely on transitive, indirect '
                  'dependencies, e.g., due to type inference or when the main target in a BUILD '
                  'file is modified to depend on other targets in the same BUILD file, as an '
                  'implementation detail. However it may still be useful to use this on '
                  'occasion. '.format(cls._language))

    register('--missing-deps-whitelist', type=Options.list,
             help="Don't report these targets even if they have missing deps.")

    register('--unnecessary-deps', choices=['off', 'warn', 'fatal'], default='off',
             help='Check for declared dependencies in {0} code that are not needed. This is a very '
                  'strict check. For example, generated code will often legitimately have BUILD '
                  'dependencies that are unused in practice.'.format(cls._language))

    register('--changed-targets-heuristic-limit', type=int, default=0,
             help='If non-zero, and we have fewer than this number of locally-changed targets, '
                  'partition them separately, to preserve stability when compiling repeatedly.')

    register('--delete-scratch', default=True, action='store_true',
             help='Leave intermediate scratch files around, for debugging build problems.')

  @classmethod
  def product_types(cls):
    return ['classes_by_target', 'classes_by_source', 'resources_by_target']

  @classmethod
  def prepare(cls, options, round_manager):
    super(JvmCompile, cls).prepare(options, round_manager)

    # This task uses JvmDependencyAnalyzer as a helper, get its product needs
    JvmDependencyAnalyzer.prepare(options, round_manager)

    round_manager.require_data('compile_classpath')
    round_manager.require_data('ivy_cache_dir')
    round_manager.require_data('ivy_resolve_symlink_map')

    # Require codegen we care about
    # TODO(John Sirois): roll this up in Task - if the list of labels we care about for a target
    # predicate to filter the full build graph is exposed, the requirement can be made automatic
    # and in turn codegen tasks could denote the labels they produce automating wiring of the
    # produce side
    round_manager.require_data('java')
    round_manager.require_data('scala')

    # Allow the deferred_sources_mapping to take place first
    round_manager.require_data('deferred_sources')

  # Subclasses must implement.
  # --------------------------
  _language = None
  _file_suffix = None

  @classmethod
  def name(cls):
    return cls._language

  @classmethod
  def get_args_default(cls, bootstrap_option_values):
    """Override to set default for --args option.

#.........这里部分代码省略.........