Python SpecData.SPECS类代码示例

 def loadPackagesData(self):
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     cmdUtils = CommandUtils()
     for package in listPackages:
         for version in SPECS.getData().getVersions(package):
             release = SPECS.getData().getRelease(package, version)
             listRPMPackages = SPECS.getData().getRPMPackages(package, version)
             srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
             srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
             srpmFile = None
             if len(srpmFiles) == 1:
                 srpmFile = srpmFiles[0]
             debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
             debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
             debugrpmFile = None
             if len(debugrpmFiles) == 1:
                 debugrpmFile = debugrpmFiles[0]
             pkgUtils = PackageUtils(self.logName, self.logPath)
             for rpmPkg in listRPMPackages:
                 rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
                 if rpmFile is not None:
                     listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
                                          "debugrpm":debugrpmFile}
                     self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
                     self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
                 else:
                     self.logger.debug("Missing rpm file for package:" + rpmPkg)

 def findRPMFileForGivenPackage(self, package,version="*", index=0):
     cmdUtils = CommandUtils()
     release = "*"
     if version == "*":
             version = SPECS.getData().getVersion(package, index)
             release = SPECS.getData().getRelease(package, index)
     listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
                                                platform.machine()+".rpm",
                                                constants.rpmPath),
                              cmdUtils.findFile(package + "-" + version + "-" + release +
                                                ".noarch.rpm",
                                                constants.rpmPath)], [])
     if constants.inputRPMSPath is not None:
         listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    "." + platform.machine()+".rpm",
                                                    constants.inputRPMSPath),
                                  cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    ".noarch.rpm", constants.inputRPMSPath)],
                                 listFoundRPMFiles)
     if len(listFoundRPMFiles) == 1:
         return listFoundRPMFiles[0]
     if len(listFoundRPMFiles) == 0:
         return None
     if len(listFoundRPMFiles) > 1:
         self.logger.error("Found multiple rpm files for given package in rpm directory." +
                           "Unable to determine the rpm file for package:" + package)
         raise Exception("Multiple rpm files found")

    def _verifyShaAndGetSourcePath(self, source, package, version):
        cmdUtils = CommandUtils()
        # Fetch/verify sources if sha1 not None.
        sha1 = SPECS.getData().getSHA1(package, version, source)
        if sha1 is not None:
            PullSources.get(package, source, sha1, constants.sourcePath,
                            constants.getPullSourcesURLs(package), self.logger)

        sourcePath = cmdUtils.findFile(source, constants.sourcePath)
        if not sourcePath:
            sourcePath = cmdUtils.findFile(source, os.path.dirname(SPECS.getData().getSpecFile(package, version)))
            if not sourcePath:
                if sha1 is None:
                    self.logger.error("No sha1 found or missing source for " + source)
                    raise Exception("No sha1 found or missing source for " + source)
                else:
                    self.logger.error("Missing source: " + source +
                                      ". Cannot find sources for package: " + package)
                    raise Exception("Missing source")
        else:
            if sha1 is None:
                self.logger.error("No sha1 found for "+source)
                raise Exception("No sha1 found")
        if len(sourcePath) > 1:
            self.logger.error("Multiple sources found for source:" + source + "\n" +
                              ",".join(sourcePath) +"\nUnable to determine one.")
            raise Exception("Multiple sources found")
        return sourcePath

    def _getListNextPackagesReadyToBuild():
        for pkg in Scheduler.listOfPackagesToBuild:
            if pkg in Scheduler.listOfPackagesCurrentlyBuilding:
                continue
            listRequiredSubPackages = list(set(SPECS.getData().getBuildRequiresForPkg(pkg) + \
                                   SPECS.getData().getRequiresAllForPkg(pkg)))

            # extend to full Requires tree
            for p in listRequiredSubPackages:
                reqs = SPECS.getData().getRequiresAllForPkg(p)
                for r in reqs:
                    if r not in listRequiredSubPackages:
                        listRequiredSubPackages.append(r)

            # convert subpackages to basepkg
            listRequiredPackages = set()
            for p in listRequiredSubPackages:
                listRequiredPackages.add(SPECS.getData().getBasePkg(p))

            canBuild = True
            for reqPkg in listRequiredPackages:
                if reqPkg not in Scheduler.listOfAlreadyBuiltPackages:
                    canBuild = False
                    break
            if canBuild:
                Scheduler.listOfPackagesNextToBuild.put((-Scheduler._getPriority(pkg), pkg))
                Scheduler.logger.debug("Adding " + pkg + " to the schedule list")

    def buildRPMSForGivenPackage(self, package, chrootID, destLogPath=None, index=0):
        self.logger.info("Building rpm's for package:" + package)

        listSourcesFiles = SPECS.getData().getSources(package, index)
        listPatchFiles = SPECS.getData().getPatches(package, index)
        specFile = SPECS.getData().getSpecFile(package, index)
        specName = SPECS.getData().getSpecName(package) + ".spec"

        chrootSourcePath = chrootID + constants.topDirPath + "/SOURCES/"
        chrootSpecPath = constants.topDirPath + "/SPECS/"
        chrootLogsFilePath = chrootID + constants.topDirPath + "/LOGS/" + package + ".log"
        chrootCmd = self.runInChrootCommand + " " + chrootID
        shutil.copyfile(specFile, chrootID + chrootSpecPath + specName)

        # FIXME: some sources are located in SPECS/.. how to mount?
        #        if os.geteuid()==0:
        self._copySourcesTobuildroot(listSourcesFiles, package, chrootSourcePath, index)
        self._copySourcesTobuildroot(listPatchFiles, package, chrootSourcePath, index)
        macros = []

        listAdditionalFiles, macros = self._getAdditionalBuildFiles(package)
        self._copyAdditionalBuildFiles(listAdditionalFiles, chrootID)

        #Adding rpm macros
        listRPMMacros = constants.userDefinedMacros
        for macroName, value in listRPMMacros.items():
            macros.append(macroName + " " + value)

        listRPMFiles = []
        listSRPMFiles = []
        try:
            listRPMFiles, listSRPMFiles = self._buildRPM(chrootSpecPath + specName,
                                                         chrootLogsFilePath, chrootCmd,
                                                         package, macros)
            self.logger.info("Successfully built rpm:" + package)
        except Exception as e:
            self.logger.error("Failed while building rpm:" + package)
            raise e
        finally:
            if destLogPath is not None:
                if (constants.rpmCheck and
                        package in constants.testForceRPMS and
                        SPECS.getData().isCheckAvailable(package)):
                    cmd = ("sed -i '/^Executing(%check):/,/^Processing files:/{//!b};d' " +
                           chrootLogsFilePath)
                    logFile = destLogPath + "/adjustTestFile.log"
                    returnVal = CommandUtils().runCommandInShell(cmd, logFile)
                    testLogFile = destLogPath + "/" + package + "-test.log"
                    shutil.copyfile(chrootLogsFilePath, testLogFile)
                else:
                    shutil.copy2(chrootLogsFilePath, destLogPath)
        self.logger.info("RPM build is successful")

        for rpmFile in listRPMFiles:
            self._copyRPM(chrootID + "/" + rpmFile, constants.rpmPath)

        for srpmFile in listSRPMFiles:
            srpmDestFile = self._copyRPM(chrootID + "/" + srpmFile, constants.sourceRpmPath)

 def updateLevels(self, mapDependencies, inPkg, parent, level):
     listPackages = SPECS.getData().getPackagesForPkg(inPkg)
     for depPkg in SPECS.getData().getRequiresForPkg(inPkg):
         if depPkg in listPackages:
             continue
         if depPkg in mapDependencies and mapDependencies[depPkg] < level + 1:
             mapDependencies[depPkg] = level + 1
             parent[depPkg] = inPkg
             self.updateLevels(mapDependencies, depPkg, parent, mapDependencies[depPkg])

 def getBasePackagesRequired(self, pkg):
     listBasePackagesRequired=[]
     listPackagesRequired = SPECS.getData().getBuildRequiresForPkg(pkg)
     listPackagesRequired.extend(SPECS.getData().getRequiresAllForPkg(pkg))
     for p in listPackagesRequired:
         basePkg = SPECS.getData().getBasePkg(p)
         if basePkg not in listBasePackagesRequired:
             listBasePackagesRequired.append(basePkg)
     return listBasePackagesRequired

 def _checkIfPackageIsAlreadyBuilt(self):
     basePkg = SPECS.getData().getSpecName(self.package)
     listRPMPackages = SPECS.getData().getRPMPackages(basePkg)
     packageIsAlreadyBuilt = True
     pkgUtils = PackageUtils(self.logName, self.logPath)
     for pkg in listRPMPackages:
         if pkgUtils.findRPMFileForGivenPackage(pkg) is None:
             packageIsAlreadyBuilt = False
             break
     return packageIsAlreadyBuilt

 def findTotalWhoNeeds(self, depList, whoNeeds):
     while depList:
         pkg = depList.pop(0)
         for depPackage in SPECS.getData().getListPackages():
             for version in SPECS.getData().getVersions(depPackage):
                 depBasePkg = depPackage+"-"+version
                 if depBasePkg in whoNeeds:
                     continue
                 if pkg in self.getBasePackagesRequired(depBasePkg):
                     whoNeeds.append(depBasePkg)
                     if depBasePkg not in depList:
                         depList.append(depBasePkg)

 def calculateSpecDependency(self, inputPackages, mapDependencies, parent):
     depQue = queue.Queue()
     for package in inputPackages:
         if SPECS.getData().isRPMPackage(package):
             version = SPECS.getData().getHighestVersion(package)
             pkg = package+"-"+version
             if pkg not in mapDependencies:
                 mapDependencies[pkg] = 0
                 parent[pkg] = ""
                 depQue.put(pkg)
                 self.findTotalRequires(mapDependencies, depQue, parent)
         else:
             self.logger.info("Could not find spec for " + package)

def buildSRPMList(srpmPath, yamlDir, blackListPkgs, dist_tag, logger, singleFile=True):
    cmdUtils = CommandUtils()
    yamlSrpmDir = os.path.join(yamlDir, "yaml_srpms")
    if not os.path.isdir(yamlSrpmDir):
        cmdUtils.runCommandInShell("mkdir -p " + yamlSrpmDir)
    if singleFile:
        yamlFile = open(yamlSrpmDir + "/srpm_list.yaml", "w")
    listPackages = SPECS.getData().getListPackages()
    listPackages.sort()
    for package in listPackages:
        if package in blackListPkgs:
            continue
        ossname = package
        for ossversion in SPECS.getData().getVersions(package):
            ossrelease = SPECS.getData().getRelease(package, ossversion)
            srpm_file_name = "%s-%s-%s%s.src.rpm" % (ossname, ossversion, ossrelease, dist_tag)
            logger.info("srpm name is %s" % (srpm_file_name))
            listFoundSRPMFiles = cmdUtils.findFile(srpm_file_name, srpmPath)

            srpmName = None
            if len(listFoundSRPMFiles) == 1:
                srpmFullPath = listFoundSRPMFiles[0]
                srpmName = os.path.basename(srpmFullPath)
                cpcmd = "cp " + srpmFullPath + " " + yamlSrpmDir + "/"
                returnVal = cmdUtils.runCommandInShell(cpcmd)
                if returnVal != 0:
                    logger.error("Copy SRPM File is failed for package:" + ossname)
            else:
                logger.error("SRPM file is not found:" + ossname)

            if not singleFile:
                yamlFile = open(yamlSrpmDir + "/" + ossname + "-" + ossversion + "-"
                                + ossrelease + ".yaml", "w")

            yamlFile.write("baseos:" + ossname + ":" + ossversion + "-" + ossrelease + dist_tag +  ":\n")
            yamlFile.write("  repository: BaseOS\n")
            yamlFile.write("  name: '" + ossname + "'\n")
            yamlFile.write("  version: '" + ossversion + "-" + ossrelease + dist_tag +"'\n")
            yamlFile.write("  url: 'http://www.vmware.com'\n")
            yamlFile.write("  baseos-style: rpm\n")
            yamlFile.write("  baseos-source: '" + str(srpmName) + "'\n")
            yamlFile.write("  baseos-osname: 'photon'\n")
            yamlFile.write("\n")
            if not singleFile:
                yamlFile.close()

    if singleFile:
        yamlFile.close()
    logger.debug("Generated SRPM yaml files for all packages")

    def _calculateParams(self, listPackages):
        self.mapCyclesToPackageList.clear()
        self.mapPackageToCycle.clear()
        self.sortedPackageList = []

        self.listOfPackagesAlreadyBuilt = self._readAlreadyAvailablePackages()

        updateBuiltRPMSList = False
        while not updateBuiltRPMSList:
            updateBuiltRPMSList = True
            listOfPackagesAlreadyBuilt = list(self.listOfPackagesAlreadyBuilt)
            for pkg in listOfPackagesAlreadyBuilt:
                listDependentRpmPackages = SPECS.getData().getRequiresAllForPackage(pkg)
                needToRebuild = False
                for dependentPkg in listDependentRpmPackages:
                    if dependentPkg not in self.listOfPackagesAlreadyBuilt:
                        needToRebuild = True
                        updateBuiltRPMSList = False
                if needToRebuild:
                    self.listOfPackagesAlreadyBuilt.remove(pkg)

        listPackagesToBuild = copy.copy(listPackages)
        for pkg in listPackages:
            if (pkg in self.listOfPackagesAlreadyBuilt and
                    not constants.rpmCheck):
                listPackagesToBuild.remove(pkg)

        if not self._readPackageBuildData(listPackagesToBuild):
            return False
        return True

    def _buildGivenPackages(self, listPackages, buildThreads):
        # Extend listPackages from ["name1", "name2",..] to ["name1-vers1", "name2-vers2",..]
        listPackageNamesAndVersions=set()
        for pkg in listPackages:
            base = SPECS.getData().getSpecName(pkg)
            for version in SPECS.getData().getVersions(base):
                listPackageNamesAndVersions.add(base+"-"+version)

        returnVal = self._calculateParams(listPackageNamesAndVersions)
        if not returnVal:
            self.logger.error("Unable to set parameters. Terminating the package manager.")
            raise Exception("Unable to set parameters")

        statusEvent = threading.Event()
        self._initializeScheduler(statusEvent)
        self._initializeThreadPool(statusEvent)

        for i in range(0, buildThreads):
            workerName = "WorkerThread" + str(i)
            ThreadPool.addWorkerThread(workerName)
            ThreadPool.startWorkerThread(workerName)

        statusEvent.wait()
        Scheduler.stopScheduling = True
        self.logger.debug("Waiting for all remaining worker threads")
        ThreadPool.join_all()

        setFailFlag = False
        allPackagesBuilt = False
        if Scheduler.isAnyPackagesFailedToBuild():
            setFailFlag = True

        if Scheduler.isAllPackagesBuilt():
            allPackagesBuilt = True

        if setFailFlag:
            self.logger.error("Some of the packages failed:")
            self.logger.error(Scheduler.listOfFailedPackages)
            raise Exception("Failed during building package")

        if not setFailFlag:
            if allPackagesBuilt:
                self.logger.debug("All packages built successfully")
            else:
                self.logger.error("Build stopped unexpectedly.Unknown error.")
                raise Exception("Unknown error")

    def _readAlreadyAvailablePackages(self):
        listAvailablePackages = set()
        pkgUtils = PackageUtils(self.logName, self.logPath)
        listPackages = SPECS.getData().getListPackages()
        for package in listPackages:
            for version in SPECS.getData().getVersions(package):
                # Mark package available only if all subpackages are available
                packageIsAlreadyBuilt=True
                listRPMPackages = SPECS.getData().getRPMPackages(package, version)
                for rpmPkg in listRPMPackages:
                    if pkgUtils.findRPMFile(rpmPkg, version) is None:
                        packageIsAlreadyBuilt=False
                        break;
                if packageIsAlreadyBuilt:
                    listAvailablePackages.add(package+"-"+version)

        return listAvailablePackages

    def __getRequiredTypePackages(pkg, requiresType):
        listRPMPackages = []
        if requiresType == "build":
            listRPMPackages.extend(SPECS.getData().getBuildRequiresForPkg(pkg))
        elif requiresType == "install":
            listRPMPackages.extend(SPECS.getData().getRequiresAllForPkg(pkg))

        # Remove duplicates.
        listRPMPackages = list(set(listRPMPackages))

        listPackages = set()

        for reqPkg in listRPMPackages:
            basePkg = SPECS.getData().getBasePkg(reqPkg)
            listPackages.add(basePkg)

        return list(listPackages)