Python PackageIndex.fetch_distribution方法代码示例

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
 def fetch_distribution(
     self, requirement, tmpdir, force_scan=False, source=False, develop_ok=False, local_index=None
 ):
     distribute_req = pkg_resources.Requirement.parse("distribute>=0.6.14")
     if pkg_resources.get_distribution("distribute") in distribute_req:
         # The local_index parameter is only in distribute>=0.6.14
         dist = PackageIndex.fetch_distribution(
             self, requirement, tmpdir, force_scan, source, develop_ok, local_index
         )
     else:
         dist = PackageIndex.fetch_distribution(self, requirement, tmpdir, force_scan, source, develop_ok)
     if dist:
         log.info("Using %s from %s" % (dist, dist.location))
     return dist

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
 def fetch_requirement(req, dest_dir, force_download):
     from setuptools.package_index import PackageIndex  # @Reimport
     from pkg_resources import working_set  # @Reimport  # NOQA
     i = PackageIndex()
     if force_download:
         [i.remove(i[req.key][0]) for _ in xrange(len(i[req.key]))]
         d = i.download(req, dest_dir)
     else:
         d = i.fetch_distribution(req, dest_dir, force_scan=True)
     d = getattr(d, 'location', d) if d else ''
     return (d if d else working_set.resolve([req])[0].location)

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
                    , 'Flask-OpenID>=1.0.1'
                    , 'webassets>=0.7.1'
                    ]

import os, shutil, sys
from pkg_resources import Requirement
from setuptools.package_index import PackageIndex

def die(message):
    sys.stderr.write(message)
    sys.exit(1)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print "Please supply a destination directory for the fetched packages!"
        sys.exit(1)
    dest_dir = os.path.abspath(sys.argv[1])

    print "Fetching packages:"
    pkg_index = PackageIndex()
    with open(os.path.join(dest_dir, 'install_order'), 'w') as fd:
        for pkg in required_packages:
            print "(---  Processing requirement '{0}'".format(pkg)
            dist = pkg_index.fetch_distribution(Requirement.parse(pkg),
                                                dest_dir, True, True)
            if dist is None:
                die("Couldn't find package satisfying '{0}'!".format(pkg))
            print " ---) Fetched {0} {1}".format(dist.project_name,
                    dist.version)
            fd.write(os.path.basename(dist.location) + '\n')

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
class SetupToolsCommand(Command):
    """setuptools Command"""

    description = "Setuptools WSS plugin"

    user_options = [("pathConfig=", "p", "Configuration file path"), ("debug=", "d", "Show debugging output")]

    def initialize_options(self):
        self.debug = None
        self.proxySetting = None
        self.service = None
        self.configDict = None
        self.pathConfig = None
        self.token = None
        self.userEnvironment = None
        self.distDepend = None
        self.pkgIndex = PackageIndex()
        self.dependencyList = []
        self.projectCoordinates = None
        self.tmpdir = tempfile.mkdtemp(prefix="wss_python_plugin-")

    def finalize_options(self):
        # log file activation and config
        if self.debug == "y":
            logging.basicConfig(
                format="%(asctime)s%(levelname)s:%(message)s", level=logging.DEBUG, filename="wss_plugin.log"
            )

        # load and import config file
        try:
            sys.path.append(self.pathConfig)
            self.configDict = __import__("config_file").config_info
            logging.info("Loading config_file was successful")
        except Exception as err:
            sys.exit("Can't import the config file." + err.message)

        # load proxy setting if exist
        if "proxy" in self.configDict:
            self.proxySetting = self.configDict["proxy"]
        if "index_url" in self.configDict:
            self.pkgIndex = PackageIndex(index_url=self.configDict["index_url"])

        self.projectCoordinates = Coordinates.create_project_coordinates(self.distribution)
        self.userEnvironment = pk_res.Environment(get_python_lib(), platform=None, python=None)
        distribution_specification = self.distribution.get_name() + "==" + self.distribution.get_version()
        distribution_requirement = pk_res.Requirement.parse(distribution_specification)

        # resolve all dependencies
        try:
            self.distDepend = pk_res.working_set.resolve([distribution_requirement], env=self.userEnvironment)
            self.distDepend.pop(0)
            logging.info("Finished resolving dependencies")
        except Exception as err:
            print "distribution was not found on this system, and is required by this application", err.message

    def run(self):
        self.validate_config_file()
        self.scan_modules()
        self.create_service()
        self.run_plugin()

    def validate_config_file(self):
        """ Validate content of config file params """

        # org token
        if "org_token" in self.configDict:
            if self.configDict["org_token"] == "":
                sys.exit("Organization token is empty")
        else:
            sys.exit("No organization token option exists")

        logging.info("Validation of config file was successful")
        # Todo: check existence of other keys in dict

    def scan_modules(self):
        """ Downloads all the dependencies calculates their sha1 and creates a list of dependencies info"""

        if self.distDepend is not None:
            for dist in self.distDepend:
                try:
                    # create a dist instance from requirement instance
                    current_requirement = dist.as_requirement()
                    current_distribution = self.pkgIndex.fetch_distribution(
                        current_requirement, self.tmpdir, force_scan=True, source=True, develop_ok=True
                    )

                    # create dep. root
                    if current_distribution is not None:
                        self.dependencyList.append(create_dependency_record(current_distribution))

                except Exception as err:
                    print "Error in fetching dists " + dist.key + " " + dist.version
            logging.info("Finished calculation for all dependencies")
        else:
            logging.info("No dependencies were found")

        shutil.rmtree(self.tmpdir)

    def create_service(self):
        """ Creates a WssServiceClient with the destination url"""
#.........这里部分代码省略.........

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
class SetupToolsCommand(Command):
    """setuptools Command"""
    description = "Setuptools WSS plugin"

    user_options = [
        ('offline=', 'o', 'Offline flag'),
        ('pathConfig=', 'p', 'Configuration file path'),
        ('debug=', 'd', 'Show debugging output'),
    ]

    def initialize_options(self):
        self.offline = None
        self.debug = None
        self.proxySetting = None
        self.service = None
        self.configDict = None
        self.pathConfig = None
        self.token = None
        self.userEnvironment = None
        self.distDepend = None
        self.pkgIndex = PackageIndex()
        self.dependencyList = []
        self.projectCoordinates = None
        self.tmpdir = tempfile.mkdtemp(prefix="wss_python_plugin-")

    def finalize_options(self):
        # log file activation and config
        if self.debug == 'y':
            logging.basicConfig(format='%(asctime)s%(levelname)s:%(message)s', level=logging.DEBUG,
                                filename='wss_plugin.log')

        # load and import config file
        try:
            sys.path.append(self.pathConfig)
            if sys.version_info.major >= 3:
                config_file_spec = importlib.util.spec_from_file_location('config_file', self.pathConfig)
                config_file_module = importlib.util.module_from_spec(config_file_spec)
                config_file_spec.loader.exec_module(config_file_module)
                self.configDict = config_file_module.config_info
            else:
                self.configDict = imp.load_source('config_file', self.pathConfig).config_info
            logging.info('Loading config_file was successful')
        except Exception as err:
            print("Can't import the config file.")
            sys.exit(err)

        # load proxy setting if exist
        if 'proxy' in self.configDict:
            self.proxySetting = self.configDict['proxy']
        if 'index_url' in self.configDict:
            self.pkgIndex = PackageIndex(index_url=self.configDict['index_url'])

        self.projectCoordinates = Coordinates.create_project_coordinates(self.distribution)
        self.userEnvironment = pk_res.Environment([get_python_lib()], platform=None, python=None)
        distribution_specification = self.distribution.get_name() + "==" + self.distribution.get_version()
        distribution_requirement = pk_res.Requirement.parse(distribution_specification)

        # resolve all dependencies
        try:
            self.distDepend = pk_res.working_set.resolve([distribution_requirement], env=self.userEnvironment)
            self.distDepend.pop(0)
            logging.info("Finished resolving dependencies")
        except Exception as err:
            print("distribution was not found on this system, and is required by this application", err.message)

    def run(self):
        self.validate_config_file()
        self.scan_modules()
        self.create_service()
        self.run_plugin()

    def validate_config_file(self):
        """ Validate content of config file params """

        # org token
        if 'org_token' in self.configDict:
            if self.configDict['org_token'] == '':
                sys.exit("Organization token is empty")
        else:
            sys.exit("No organization token option exists")

        logging.info("Validation of config file was successful")
        # Todo: check existence of other keys in dict

    def scan_modules(self):
        """ Downloads all the dependencies calculates their sha1 and creates a list of dependencies info"""

        if self.distDepend is not None:
            for dist in self.distDepend:
                try:
                    # create a dist instance from requirement instance
                    current_requirement = dist.as_requirement()
                    current_distribution = self.pkgIndex.fetch_distribution(
                        current_requirement, self.tmpdir, force_scan=True, source=True, develop_ok=True)

                    # create dep. root
                    if current_distribution is not None:
                        self.dependencyList.append(create_dependency_record(current_distribution))

                except Exception as err:
#.........这里部分代码省略.........

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (
            make_option("-o", "--owner", help="add packages as OWNER",
                        metavar="OWNER", default=None),
        )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        self.pypi = PackageIndex()
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            reqs = pkg_resources.parse_requirements(label)
            for req in reqs:
                try:
                    package = self.pypi.fetch_distribution(req, tmp, source=True)
                except Exception as err:
                    print "Could not add %s: %s." % (req, err)
                else:
                    self._save_package(package.location, options["owner"])

    def _save_package(self, path, ownerid):
        meta = self._get_meta(path)

        try:
            # can't use get_or_create as that demands there be an owner
            package = Package.objects.get(name=meta.name)
            isnewpackage = False
        except Package.DoesNotExist:
            package = Package(name=meta.name)
            isnewpackage = True

        release = package.get_release(meta.version)
        if not isnewpackage and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if ownerid:
            try:
                if "@" in ownerid:
                    owner = User.objects.get(email=ownerid)
                else:
                    owner = User.objects.get(username=ownerid)
            except User.DoesNotExist:
                pass
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                pass

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.

        package.owner = owner
        # Some packages don't have proper licence, seems to be a problem
        # with setup.py upload. Use "UNKNOWN"
        package.license = meta.license or "Unknown"
        package.metadata_version = meta.metadata_version
        package.author = meta.author
        package.home_page = meta.home_page
        package.download_url = meta.download_url
        package.summary = meta.summary
        package.description = meta.description
        package.author_email = meta.author_email

        package.save()

        # TODO: Do I need add classifieres objects???
#        for classifier in meta.classifiers:
#            package.classifiers.add(
#                    Classifier.objects.get_or_create(name=classifier)[0])
        release = Release()
        release.version = meta.version
        release.package = package
        release.package_info = self._get_pkg_info(meta)
        # Classifiers is processed separatily since it is a list a must be
        # properly set son getlist returns the right result
#.........这里部分代码省略.........

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
    def run(self):
        # Prepare for iterations.
        pkgreqmap = reqmap()
        for reqarg in self.reqarglist: pkgreqmap.append_arg(reqarg)
        pkgreqmap.resolve_matchlist(self.logobj, self.options['--url'],
                                    self.options['--skip-logged'])

        pkgidx = PackageIndex(index_url = self.options['--url'])

        show_sepline = False
        # Main loop.
        distlist = []
        ok_packages = []
        while len(pkgreqmap) > 0:
            new_pkgreqmap = reqmap()
            for idx, total, pkgreqobj in pkgreqmap.reqobj_seq():

                pkgname = pkgreqobj.project_name
                if pkgname in ok_packages: continue
                ok_packages.append(pkgname)
                reqstr = str(pkgreqobj)

                if show_sepline: self.pkgsys.sepline()
                else: show_sepline = True

                self.pkgsys.info('======== %s: %d/%d ========' % \
                                     (pkgname, idx + 1, total))

                if self.options['--skip-broken']:
                    try: self.logobj.check_broken(pkgname)
                    except: continue

                # Collect values into args step by step.
                args = copy.copy(self.options)
                args['self'] = self.arg0

                self.pkgsys.begin('Downloading %s' % reqstr)
                try:
                    dist = pkgidx.fetch_distribution(pkgreqobj,
                                                     self.options['--download-dir'],
                                                     source = True)
                    if dist is None:
                        raise RuntimeError, 'None'
                except:
                    self.pkgsys.end(False)
                    self.logobj.in_except(pkgname,
                                          'Download %s failed' % reqstr)
                    continue
                else:
                    self.pkgsys.end(True)

                self.pkgsys.begin('Unpacking %s' % dist.location)
                try: smart_archive(args, dist, self.options['--unpack-dir'])
                except:
                    self.pkgsys.end(False)
                    self.logobj.in_except(pkgname, 'Unpack %s failed' % reqstr)
                    continue
                else:
                    self.pkgsys.end(True)
                unpackpath = args['unpackpath']

                config_secs = ['%s-%s' % (dist.project_name, dist.version),
                               dist.project_name]

                for secname in config_secs:
                    for name, value in config.items(secname):
                        if name not in args: args[name] = value
                if not 'patches' in args: args['patches'] = []
                else: args['patches'] = args['patches'].split()

                # Apply patches.
                for patch in config.patches(config_secs):
                    self.pkgsys.begin('Applying %s' % os.path.basename(patch))
                    os.system('(cd %s; patch -p0 < %s) > /dev/null' % \
                                  (unpackpath, patch))
                    self.pkgsys.end(True)
                    if os.path.isfile(os.path.join(unpackpath, 'fixsetup.py')):
                        os.system('(cd %s; python fixsetup.py)' % unpackpath)

                self.pkgsys.begin('Get package args')
                try: get_package_args(args, dist)
                except:
                    self.pkgsys.end(False)
                    self.logobj.in_except(pkgname, 'Get package args failed')
                    continue
                else:
                    self.pkgsys.end(True)

                self.pkgsys.begin('Setup args')
                try: self.pkgsys.setup_args(args)
                except:
                    self.pkgsys.end(False)
                    self.logobj.in_except(pkgname, 'pkgsys.setup_args failed')
                    continue
                else:
                    self.pkgsys.end(True)

                self.pkgsys.begin('Writing %s' % args['output'])
                try:
                    ensure_dir(os.path.dirname(args['output']))
#.........这里部分代码省略.........

# 需要导入模块: from setuptools.package_index import PackageIndex [as 别名]
# 或者: from setuptools.package_index.PackageIndex import fetch_distribution [as 别名]
def runit():
    # process command-line options
    bool_opts = map(translate_longopt, stdeb_cmd_bool_opts)
    bool_opts.append("process-dependencies")
    parser = FancyGetopt(stdeb_cmdline_opts + [("help", "h", "show detailed help message")] + EXTRA_OPTS)
    optobj = OptObj()
    args = parser.getopt(object=optobj)
    idx = PackageIndex()
    for option in optobj.__dict__:
        value = getattr(optobj, option)
        is_string = type(value) == str
        if option in bool_opts and is_string:
            setattr(optobj, option, strtobool(value))

    if hasattr(optobj, "help"):
        print USAGE
        parser.set_option_table(stdeb_cmdline_opts + EXTRA_OPTS)
        parser.print_help("Options:")
        return 0

    if len(args) != 1:
        log.error("not given single argument (distfile), args=%r", args)
        print USAGE
        return 1

    sdist_file = args[0]

    package = None

    final_dist_dir = optobj.__dict__.get("dist_dir", "deb_dist")
    tmp_dist_dir = os.path.join(final_dist_dir, "tmp_py2dsc")
    if os.path.exists(tmp_dist_dir):
        shutil.rmtree(tmp_dist_dir)
    os.makedirs(tmp_dist_dir)

    if not os.path.isfile(sdist_file):
        for ext in EXTENSIONS:
            if sdist_file.endswith(ext):
                raise IOError, "File not found"
        package = Requirement.parse(sdist_file)
        log.info("Package %s not found, trying PyPI..." % sdist_file)
        dist = idx.fetch_distribution(package, final_dist_dir, force_scan=True, source=True)
        if hasattr(dist, "location"):
            sdist_file = dist.location
        else:
            raise Exception, "Distribution not found on PyPi"
        log.info("Got %s", sdist_file)

    dist = list(distros_for_filename(sdist_file))[0]
    idx.scan_egg_links(dist.location)
    package = idx.obtain(Requirement.parse(dist.project_name))

    if hasattr(optobj, "process_dependencies"):
        if bool(int(getattr(optobj, "process_dependencies"))):
            backup_argv = sys.argv[:]
            oldargv = sys.argv[:]
            oldargv.pop(-1)

            if package.requires():
                log.info("Processing package dependencies for %s", package)
            for req in package.requires():
                #                print >> sys.stderr
                new_argv = oldargv + ["%s" % req]
                log.info("Bulding dependency package %s", req)
                log.info("  running '%s'", " ".join(new_argv))
                sys.argv = new_argv
                runit()
            #                print >> sys.stderr
            if package.requires():
                log.info("Completed building dependencies " "for %s, continuing...", package)
        sys.argv = backup_argv

    if package is not None and hasattr(optobj, "extra_cfg_file"):
        # Allow one to have patch-files setup on config file for example
        local_parser = SafeConfigParser()
        local_parser.readfp(open(optobj.__dict__.get("extra_cfg_file")))
        if local_parser.has_section(package.project_name):
            for opt in local_parser.options(package.project_name):
                _opt = opt.replace("_", "-")
                if parser.has_option(_opt) or parser.has_option(_opt + "="):
                    setattr(optobj, opt, local_parser.get(package.project_name, opt))

    patch_file = optobj.__dict__.get("patch_file", None)
    patch_level = int(optobj.__dict__.get("patch_level", 0))
    patch_posix = int(optobj.__dict__.get("patch_posix", 0))

    expand_dir = os.path.join(tmp_dist_dir, "stdeb_tmp")
    if os.path.exists(expand_dir):
        shutil.rmtree(expand_dir)
    if not os.path.exists(tmp_dist_dir):
        os.mkdir(tmp_dist_dir)
    os.mkdir(expand_dir)

    expand_sdist_file(os.path.abspath(sdist_file), cwd=expand_dir)

    # now the sdist package is expanded in expand_dir
    expanded_root_files = os.listdir(expand_dir)
    assert len(expanded_root_files) == 1
    repackaged_dirname = expanded_root_files[0]
    fullpath_repackaged_dirname = os.path.join(tmp_dist_dir, repackaged_dirname)
#.........这里部分代码省略.........