Python fetch.Fetch类代码示例

    def go(self, d, urls = []):
        """Fetch urls"""
        if not urls:
            urls = self.urls

        for loc in urls:
            (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))

            tag = gettag(parm)
            proto = getprotocol(parm)

            gitsrcname = '%s%s' % (host, path.replace('/', '.'))

            repofilename = 'git_%s.tar.gz' % (gitsrcname)
            repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
            repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)

            coname = '%s' % (tag)
            codir = os.path.join(repodir, coname)

            cofile = self.localpath(loc, d)

            # tag=="master" must always update
            if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)):
                bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % cofile)
                continue

            if not os.path.exists(repodir):
                if Fetch.try_mirror(d, repofilename):    
                    bb.mkdirhier(repodir)
                    os.chdir(repodir)
                    rungitcmd("tar -xzf %s" % (repofile),d)
                else:
                    rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d)

            os.chdir(repodir)
            rungitcmd("git pull %s://%s%s" % (proto, host, path),d)
            rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d)
            rungitcmd("git prune-packed", d)
            rungitcmd("git pack-redundant --all | xargs -r rm", d)
            # Remove all but the .git directory
            rungitcmd("rm * -Rf", d)
            # old method of downloading tags
            #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)

            os.chdir(repodir)
            bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
            rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)

            if os.path.exists(codir):
                prunedir(codir)

            bb.mkdirhier(codir)
            os.chdir(repodir)
            rungitcmd("git read-tree %s" % (tag),d)
            rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)

            os.chdir(codir)
            bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
            rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)

    def go(self, loc, ud, d):
        """Fetch url"""

        if Fetch.try_mirror(d, ud.localfile):
            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
            return

        if ud.user:
            username = ud.user + '@'
        else:
            username = ""

        gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))

        repofilename = 'git_%s.tar.gz' % (gitsrcname)
        repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
        repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)

        coname = '%s' % (ud.tag)
        codir = os.path.join(repodir, coname)

        if not os.path.exists(repodir):
            if Fetch.try_mirror(d, repofilename):    
                bb.mkdirhier(repodir)
                os.chdir(repodir)
                runfetchcmd("tar -xzf %s" % (repofile), d)
            else:
                runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d)

        os.chdir(repodir)
        # Remove all but the .git directory
        if not self._contains_ref(ud.tag, d):
            runfetchcmd("rm * -Rf", d)
            runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
            runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
            runfetchcmd("git prune-packed", d)
            runfetchcmd("git pack-redundant --all | xargs -r rm", d)

        os.chdir(repodir)
        mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
        if mirror_tarballs != "0": 
            bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
            runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)

        if os.path.exists(codir):
            bb.utils.prunedir(codir)

        bb.mkdirhier(codir)
        os.chdir(repodir)
        runfetchcmd("git read-tree %s" % (ud.tag), d)
        runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)

        os.chdir(codir)
        bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)

        os.chdir(repodir)
        bb.utils.prunedir(codir)

    def go(self, loc, ud, d):
        """Fetch url"""

        if Fetch.try_mirror(d, ud.localfile):
            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
            return

        gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))

        repofilename = 'git_%s.tar.gz' % (gitsrcname)
        repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
        repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)

        coname = '%s' % (ud.tag)
        codir = os.path.join(repodir, coname)

        if not os.path.exists(repodir):
            if Fetch.try_mirror(d, repofilename):    
                bb.mkdirhier(repodir)
                os.chdir(repodir)
                runfetchcmd("tar -xzf %s" % (repofile), d)
            else:
                runfetchcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir), d)

        os.chdir(repodir)
        # Remove all but the .git directory
        runfetchcmd("rm * -Rf", d)
        runfetchcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path), d)
        runfetchcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path), d)
        runfetchcmd("git prune-packed", d)
        runfetchcmd("git pack-redundant --all | xargs -r rm", d)
        # old method of downloading tags
        #runfetchcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")), d)

        os.chdir(repodir)
        bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
        runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)

        if os.path.exists(codir):
            prunedir(codir)

        bb.mkdirhier(codir)
        os.chdir(repodir)
        runfetchcmd("git read-tree %s" % (ud.tag), d)
        runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)

        os.chdir(codir)
        bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)

        os.chdir(repodir)
        prunedir(codir)

    def go(self, loc, ud, d):
        """Fetch url"""

        # try to use the tarball stash
        if Fetch.try_mirror(d, ud.localfile):
            bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
            return

        if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
            bzrcmd = self._buildbzrcommand(ud, d, "update")
            bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)
            os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
            runfetchcmd(bzrcmd, d)
        else:
            os.system("rm -rf %s" % os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)))
            bzrcmd = self._buildbzrcommand(ud, d, "fetch")
            bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Checkout %s" % loc)
            bb.mkdirhier(ud.pkgdir)
            os.chdir(ud.pkgdir)
            bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % bzrcmd)
            runfetchcmd(bzrcmd, d)

        os.chdir(ud.pkgdir)
        # tar them up to a defined filename
        try:
            runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.pkgdir)), d)
        except:
            t, v, tb = sys.exc_info()
            try:
                os.unlink(ud.localpath)
            except OSError:
                pass
            raise t, v, tb

    def localpath(url, d):
        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
        if "localpath" in parm:
            #           if user overrides local path, use it.
            return parm["localpath"]

        if not "module" in parm:
            raise MissingParameterError("cvs method needs a 'module' parameter")
        else:
            module = parm["module"]
        if "tag" in parm:
            tag = parm["tag"]
        else:
            tag = ""
        if "date" in parm:
            date = parm["date"]
        else:
            if not tag:
                date = Fetch.getSRCDate(d)
            else:
                date = ""

        return os.path.join(
            data.getVar("DL_DIR", d, 1),
            data.expand("%s_%s_%s_%s.tar.gz" % (module.replace("/", "."), host, tag, date), d),
        )

    def localpath(url, d):
        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
        if "localpath" in parm:
            #           if user overrides local path, use it.
            return parm["localpath"]

        if not "module" in parm:
            raise MissingParameterError("svn method needs a 'module' parameter")
        else:
            module = parm["module"]
        if "rev" in parm:
            revision = parm["rev"]
        else:
            revision = ""

        date = Fetch.getSRCDate(d)

        if "srcdate" in parm:
            date = parm["srcdate"]

        if revision:
            date = ""

        return os.path.join(
            data.getVar("DL_DIR", d, 1),
            data.expand(
                "%s_%s_%s_%s_%s.tar.gz" % (module.replace("/", "."), host, path.replace("/", "."), revision, date), d
            ),
        )

    def localpath(self, url, ud, d):

        if 'protocol' in ud.parm:
            ud.proto = ud.parm['protocol']
        elif not ud.host:
            ud.proto = 'file'
        else:
            ud.proto = "rsync"

        ud.branch = ud.parm.get("branch", "master")

        tag = Fetch.srcrev_internal_helper(ud, d)
        if tag is True:
            ud.tag = self.latest_revision(url, ud, d)	
        elif tag:
            ud.tag = tag
        else:
            ud.tag = ""

        if not ud.tag or ud.tag == "master":
            ud.tag = self.latest_revision(url, ud, d)

        ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d)

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)

    def localpath(self, url, ud, d):
        if not "module" in ud.parm:
            raise MissingParameterError("hg method needs a 'module' parameter")

        ud.module = ud.parm["module"]

        # Create paths to mercurial checkouts
        relpath = self._strip_leading_slashes(ud.path)
        ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
        ud.moddir = os.path.join(ud.pkgdir, ud.module)

        if 'rev' in ud.parm:
            ud.revision = ud.parm['rev']
        else:
            tag = Fetch.srcrev_internal_helper(ud, d)
            if tag is True:
                ud.revision = self.latest_revision(url, ud, d)
            elif tag:
                ud.revision = tag
            else:
                ud.revision = self.latest_revision(url, ud, d)

        ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)

    def localpath(self, url, ud, d):
        if not "module" in ud.parm:
            raise MissingParameterError("osc method needs a 'module' parameter.")

        ud.module = ud.parm["module"]

        # Create paths to osc checkouts
        relpath = ud.path
        if relpath.startswith('/'):
            # Remove leading slash as os.path.join can't cope
            relpath = relpath[1:]
        ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host)
        ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)

        if 'rev' in ud.parm:
            ud.revision = ud.parm['rev']
        else:
            pv = data.getVar("PV", d, 0)
            rev = Fetch.srcrev_internal_helper(ud, d)
            if rev and rev != True:
                ud.revision = rev
            else:
                ud.revision = ""

        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)

    def go(self, loc, ud, d):
        """Fetch urls"""

        if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
            return

        svkroot = ud.host + ud.path

        # pyflakes claims date is not known... it looks right
        svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module)

        if ud.revision:
            svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module)

        # create temp directory
        localdata = data.createCopy(d)
        data.update_data(localdata)
        bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
        bb.mkdirhier(data.expand('${WORKDIR}', localdata))
        data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
        tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
        tmpfile = tmppipe.readline().strip()
        if not tmpfile:
            bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
            raise FetchError(ud.module)

        # check out sources there
        os.chdir(tmpfile)
        bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
        bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd)
        myret = os.system(svkcmd)
        if myret != 0:
            try:
                os.rmdir(tmpfile)
            except OSError:
                pass
            raise FetchError(ud.module)

        os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
        # tar them up to a defined filename
        myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
        if myret != 0:
            try:
                os.unlink(ud.localpath)
            except OSError:
                pass
            raise FetchError(ud.module)
        # cleanup
        os.system('rm -rf %s' % tmpfile)

    def localpath(self, url, ud, d):
        if not "module" in ud.parm:
            raise MissingParameterError("svn method needs a 'module' parameter")

        ud.module = ud.parm["module"]

        # Create paths to svn checkouts
        relpath = ud.path
        if relpath.startswith("/"):
            # Remove leading slash as os.path.join can't cope
            relpath = relpath[1:]
        ud.pkgdir = os.path.join(data.expand("${SVNDIR}", d), ud.host, relpath)
        ud.moddir = os.path.join(ud.pkgdir, ud.module)

        if "rev" in ud.parm:
            ud.date = ""
            ud.revision = ud.parm["rev"]
        elif "date" in ud.date:
            ud.date = ud.parm["date"]
            ud.revision = ""
        else:
            #
            # ***Nasty hack***
            # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE)
            # Should warn people to switch to SRCREV here
            #
            pv = data.getVar("PV", d, 0)
            if "DATE" in pv:
                ud.revision = ""
            else:
                rev = Fetch.srcrev_internal_helper(ud, d)
                if rev is True:
                    ud.revision = self.latest_revision(url, ud, d)
                    ud.date = ""
                elif rev:
                    ud.revision = rev
                    ud.date = ""
                else:
                    ud.revision = ""

        ud.localfile = data.expand(
            "%s_%s_%s_%s_%s.tar.gz"
            % (ud.module.replace("/", "."), ud.host, ud.path.replace("/", "."), ud.revision, ud.date),
            d,
        )

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)

    def sortable_revision(self, url, ud, d):
        """

        """
        pd = bb.persist_data.persist(d)
        localcounts = pd['BB_URI_LOCALCOUNT']
        key = self.generate_revision_key(url, ud, d, branch=True)
        oldkey = self.generate_revision_key(url, ud, d, branch=False)

        latest_rev = self._build_revision(url, ud, d)
        last_rev = localcounts[key + '_rev']
        if last_rev is None:
            last_rev = localcounts[oldkey + '_rev']
            if last_rev is not None:
                del localcounts[oldkey + '_rev']
                localcounts[key + '_rev'] = last_rev

        uselocalcount = bb.data.getVar("BB_LOCALCOUNT_OVERRIDE", d, True) or False
        count = None
        if uselocalcount:
            count = Fetch.localcount_internal_helper(ud, d)
        if count is None:
            count = localcounts[key + '_count']
        if count is None:
            count = localcounts[oldkey + '_count']
            if count is not None:
                del localcounts[oldkey + '_count']
                localcounts[key + '_count'] = count

        if last_rev == latest_rev:
            return str(count + "+" + latest_rev)

        buildindex_provided = hasattr(self, "_sortable_buildindex")
        if buildindex_provided:
            count = self._sortable_buildindex(url, ud, d, latest_rev)
        if count is None:
            count = "0"
        elif uselocalcount or buildindex_provided:
            count = str(count)
        else:
            count = str(int(count) + 1)

        localcounts[key + '_rev'] = latest_rev
        localcounts[key + '_count'] = count

        return str(count + "+" + latest_rev)

    def localpath(url, d):
        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
        if "localpath" in parm:
#           if user overrides local path, use it.
            return parm["localpath"]

        if not "module" in parm:
            raise MissingParameterError("svk method needs a 'module' parameter")
        else:
            module = parm["module"]
        if 'rev' in parm:
            revision = parm['rev']
        else:
            revision = ""

        date = Fetch.getSRCDate(d)

        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s_%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, path.replace('/', '.'), revision, date), d))

    def localpath (self, url, ud, d):

        # Create paths to bzr checkouts
        relpath = self._strip_leading_slashes(ud.path)
        ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath)

        revision = Fetch.srcrev_internal_helper(ud, d)
        if revision is True:
            ud.revision = self.latest_revision(url, ud, d)
        elif revision:
            ud.revision = revision

        if not ud.revision:
            ud.revision = self.latest_revision(url, ud, d)

        ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)

    def localpath(self, url, ud, d):

        if 'protocol' in ud.parm:
            ud.proto = ud.parm['protocol']
        elif not ud.host:
            ud.proto = 'file'
        else:
            ud.proto = "rsync"

        ud.branch = ud.parm.get("branch", "master")

        gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
        ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname)
        ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)

        tag = Fetch.srcrev_internal_helper(ud, d)
        if tag is True:
            ud.tag = self.latest_revision(url, ud, d)
        elif tag:
            ud.tag = tag

        if not ud.tag or ud.tag == "master":
            ud.tag = self.latest_revision(url, ud, d)

        subdir = ud.parm.get("subpath", "")
        if subdir != "":
            if subdir.endswith("/"):
                subdir = subdir[:-1]
            subdirpath = os.path.join(ud.path, subdir);
        else:
            subdirpath = ud.path;

        if 'fullclone' in ud.parm:
            ud.localfile = ud.mirrortarball
        else:
            ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)

        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"

        if 'noclone' in ud.parm:
            ud.localfile = None
            return None

        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)