Python CommandUtils.CommandUtils类代码示例

 def buildRPM(self,specFile,logFile,chrootCmd,package,macros):
     
     rpmBuildcmd= self.rpmbuildBinary+" "+self.rpmbuildBuildallOption+" "+self.rpmbuildDistOption
     if not constants.rpmCheck:
         rpmBuildcmd+=" "+self.rpmbuildNocheckOption
     for macro in macros:
         rpmBuildcmd+=' --define \\\"%s\\\"' % macro
     rpmBuildcmd+=" "+self.rpmbuildBuildNum+" "+self.rpmbuildReleaseVer
     rpmBuildcmd+=" "+specFile
     
     cmdUtils = CommandUtils()
     self.logger.info("Building rpm....")
     self.logger.info(rpmBuildcmd)
     returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
     if not returnVal:
         self.logger.error("Building rpm is failed "+specFile)
         raise Exception("RPM Build failed")
     
     #Extracting rpms created from log file
     logfile=open(logFile,'r')
     fileContents=logfile.readlines()
     logfile.close()
     listRPMFiles=[]
     listSRPMFiles=[]
     for i in range(0,len(fileContents)):
         if re.search("^Wrote:",fileContents[i]):
             listcontents=fileContents[i].split()
             if (len(listcontents) == 2) and listcontents[1].strip()[-4:] == ".rpm" and listcontents[1].find("/RPMS/") != -1:
                 listRPMFiles.append(listcontents[1])
             if (len(listcontents) == 2) and listcontents[1].strip()[-8:] == ".src.rpm" and listcontents[1].find("/SRPMS/") != -1:
                 listSRPMFiles.append(listcontents[1])
     return listRPMFiles,listSRPMFiles    

def get(package, source, sha1, sourcesPath, URLs, logger):
    cmdUtils = CommandUtils()
    sourcePath = cmdUtils.findFile(source, sourcesPath)
    if sourcePath is not None and len(sourcePath) > 0:
        if len(sourcePath) > 1:
            raise Exception("Multiple sources found for source:" + source + "\n" +
                            ",".join(sourcePath) +"\nUnable to determine one.")
        if sha1 == getFileHash(sourcePath[0]):
            # Use file from sourcesPath
            return
        else:
            logger.info("sha1 of " + sourcePath[0] + " does not match. " + sha1 +
                        " vs " + getFileHash(sourcePath[0]))
    for baseurl in URLs:
        #form url: https://dl.bintray.com/vmware/photon_sources/1.0/<filename>.
        url = '%s/%s' % (baseurl, source)
        destfile = os.path.join(sourcesPath, source)
        logger.debug("Downloading: " + url)
        try:
            downloadFile(url, destfile)
            if sha1 != getFileHash(destfile):
                raise Exception('Invalid sha1 for package %s file %s' % package, source)
            return
        except requests.exceptions.HTTPError as e:
            logger.exception(e)
            # on any HTTP errors - try next config
            continue
        except Exception as e:
            logger.exception(e)
    raise Exception("Missing source: " + source)

 def loadPackagesData(self):
     listPackages =  constants.specData.getListPackages()
     listPackages.sort()
     listRPMFiles = []
     cmdUtils = CommandUtils()
     for package in listPackages:
         release = constants.specData.getRelease(package)
         version = constants.specData.getVersion(package)
         listRPMPackages = constants.specData.getRPMPackages(package)
         srpmFileName = package+"-"+version+"-"+release+".src.rpm"
         srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
         srpmFile = None
         if len(srpmFiles) == 1:
             srpmFile = srpmFiles[0]
         debugrpmFileName = package+"-debuginfo-"+version+"-"+release+"*"
         debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
         debugrpmFile = None
         if len(debugrpmFiles) == 1:
             debugrpmFile = debugrpmFiles[0]
         pkgUtils = PackageUtils(self.logName,self.logPath)
         for rpmPkg in listRPMPackages:
             rpmFile = pkgUtils.findRPMFileForGivenPackage(rpmPkg)
             if rpmFile is not None:
                 listRPMFiles.append(rpmFile)
                 listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile, "debugrpm":debugrpmFile}
                 self.pkgList[rpmPkg] = listPkgAttributes
                 self.logger.debug("Added "+rpmPkg +" rpm package to the list")
             else:
                 self.logger.error("Missing rpm file for package:"+rpmPkg)

 def installRPMSInAOneShot(self,chrootID,destLogPath):
     chrootCmd=self.runInChrootCommand+" "+chrootID
     rpmInstallcmd=self.rpmBinary+" "+ self.installRPMPackageOptions
     if self.noDepsRPMFilesToInstallInAOneShot != "":
         self.logger.info("Installing nodeps rpms: " + self.noDepsPackagesToInstallInAOneShot)
         logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms_nodeps.log"
         cmdUtils = CommandUtils()
         cmd = rpmInstallcmd+" "+self.nodepsRPMPackageOptions + " " + self.noDepsRPMFilesToInstallInAOneShot
         returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
         if destLogPath is not None:
             shutil.copy2(logFile, destLogPath)
         if not returnVal:
             self.logger.error("Unable to install rpms")
             raise Exception("RPM installation failed")
     if self.rpmFilesToInstallInAOneShot != "":
         self.logger.info("Installing rpms: " + self.packagesToInstallInAOneShot)
         logFile=chrootID+constants.topDirPath+"/LOGS/install_rpms.log"
         cmdUtils = CommandUtils()
         cmd=rpmInstallcmd+" "+self.rpmFilesToInstallInAOneShot
         returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
         if destLogPath is not None:
             shutil.copy2(logFile, destLogPath)
         if not returnVal:
             self.logger.error("Unable to install rpms")
             raise Exception("RPM installation failed")

 def buildCoreToolChainPackages(self):
     self.logger.info("Building core tool chain packages.....")
     chrootID=None
     try:
         pkgUtils=PackageUtils(self.logName,self.logPath)
         for package in constants.listCoreToolChainRPMPackages:
             rpmPkg=pkgUtils.findRPMFileForGivenPackage(package)
             if rpmPkg is not None:
                 continue
             chrUtils = ChrootUtils(self.logName,self.logPath)
             chrootName="build-core-toolchain"
             destLogPath=constants.logPath+"/build-"+package
             if not os.path.isdir(destLogPath):
                 cmdUtils = CommandUtils()
                 cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
             returnVal,chrootID = chrUtils.createChroot(chrootName)
             if not returnVal:
                 self.logger.error("Creating chroot failed")
                 raise Exception("creating chroot failed")
             self.installToolChainRPMS(chrootID)
             pkgUtils.adjustGCCSpecs(package, chrootID, destLogPath)
             pkgUtils.buildRPMSForGivenPackage(package, chrootID,destLogPath)
             chrUtils.destroyChroot(chrootID)
             chrootID=None
         self.logger.info("Successfully built toolchain")
         if chrootID is not None:
             chrUtils.destroyChroot(chrootID)
     except Exception as e:
         self.logger.error("Unable to build tool chain.")
         # print stacktrace
         traceback.print_exc()
         raise e

    def buildRPM(self, specFile, logFile, chrootCmd):

        rpmBuildcmd = self.rpmbuildBinary + " " + self.rpmbuildBuildallOption + " " + self.rpmbuildNocheckOption
        rpmBuildcmd += " " + specFile

        cmdUtils = CommandUtils()
        returnVal = cmdUtils.runCommandInShell(rpmBuildcmd, logFile, chrootCmd)
        if not returnVal:
            self.logger.error("Building rpm is failed " + specFile)
            raise Exception("RPM Build failed")

        # Extracting rpms created from log file
        logfile = open(logFile, "r")
        fileContents = logfile.readlines()
        logfile.close()
        listRPMFiles = []
        for i in range(0, len(fileContents)):
            if re.search("^Wrote:", fileContents[i]):
                listcontents = fileContents[i].split()
                if (
                    (len(listcontents) == 2)
                    and listcontents[1].strip()[-4:] == ".rpm"
                    and listcontents[1].find("/RPMS/") != -1
                ):
                    listRPMFiles.append(listcontents[1])

        return listRPMFiles

 def findRPMFileForGivenPackage(self, package,version="*", index=0):
     cmdUtils = CommandUtils()
     release = "*"
     if version == "*":
             version = SPECS.getData().getVersion(package, index)
             release = SPECS.getData().getRelease(package, index)
     listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release + "." +
                                                platform.machine()+".rpm",
                                                constants.rpmPath),
                              cmdUtils.findFile(package + "-" + version + "-" + release +
                                                ".noarch.rpm",
                                                constants.rpmPath)], [])
     if constants.inputRPMSPath is not None:
         listFoundRPMFiles = sum([cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    "." + platform.machine()+".rpm",
                                                    constants.inputRPMSPath),
                                  cmdUtils.findFile(package + "-" + version + "-" + release +
                                                    ".noarch.rpm", constants.inputRPMSPath)],
                                 listFoundRPMFiles)
     if len(listFoundRPMFiles) == 1:
         return listFoundRPMFiles[0]
     if len(listFoundRPMFiles) == 0:
         return None
     if len(listFoundRPMFiles) > 1:
         self.logger.error("Found multiple rpm files for given package in rpm directory." +
                           "Unable to determine the rpm file for package:" + package)
         raise Exception("Multiple rpm files found")

    def installRPM(self, package, chrootID, noDeps=False, destLogPath=None):
        self.logger.info("Installing rpm for package:" + package)
        self.logger.debug("No deps:" + str(noDeps))

        rpmfile = self.findRPMFileForGivenPackage(package)
        if rpmfile is None:
            self.logger.error("No rpm file found for package:" + package)
            raise Exception("Missing rpm file")

        rpmDestFile = self.copyRPM(rpmfile, chrootID + constants.topDirPath + "/RPMS")
        rpmFile = rpmDestFile.replace(chrootID, "")
        chrootCmd = self.runInChrootCommand + " " + chrootID
        logFile = chrootID + constants.topDirPath + "/LOGS" + "/" + package + ".completed"

        rpmInstallcmd = self.rpmBinary + " " + self.installRPMPackageOptions
        if noDeps:
            rpmInstallcmd += " " + self.nodepsRPMPackageOptions
        rpmInstallcmd += " " + rpmFile

        cmdUtils = CommandUtils()
        returnVal = cmdUtils.runCommandInShell(rpmInstallcmd, logFile, chrootCmd)
        if destLogPath is not None:
            shutil.copy2(logFile, destLogPath)
        if not returnVal:
            self.logger.error("Unable to install rpm:" + rpmFile)
            raise Exception("RPM installation failed")

 def buildPackage(self,package):
     #should initialize a logger based on package name
     chrUtils = ChrootUtils(self.logName,self.logPath)
     chrootName="build-"+package
     chrootID=None
     isToolChainPackage=False
     if package in constants.listToolChainPackages:
         isToolChainPackage=True
     try:
         chrootID = self.prepareBuildRoot(chrootName,isToolChainPackage)
         destLogPath=constants.logPath+"/build-"+package
         if not os.path.isdir(destLogPath):
             cmdUtils = CommandUtils()
             cmdUtils.runCommandInShell("mkdir -p "+destLogPath)
         
         listInstalledPackages=self.findInstalledPackages(chrootID)
         self.logger.info("List of installed packages")
         self.logger.info(listInstalledPackages)
         listDependentPackages=self.findBuildTimeRequiredPackages(package)
         
         if len(listDependentPackages) != 0:
             self.logger.info("Installing the build time dependent packages......")
             for pkg in listDependentPackages:
                 self.installPackage(pkg,chrootID,destLogPath,listInstalledPackages)
             self.logger.info("Finished installing the build time dependent packages......")
         self.adjustGCCSpecs(package, chrootID, destLogPath)
         pkgUtils = PackageUtils(self.logName,self.logPath)
         pkgUtils.buildRPMSForGivenPackage(package,chrootID,destLogPath)
         self.logger.info("Successfully built the package:"+package)
     except Exception as e:
         self.logger.error("Failed while building package:" + package)
         self.logger.debug("Chroot with ID: " + chrootID + " not deleted for debugging.")
         raise e
     if chrootID is not None:
         chrUtils.destroyChroot(chrootID)

 def loadPackagesData(self):
     listPackages = SPECS.getData().getListPackages()
     listPackages.sort()
     cmdUtils = CommandUtils()
     for package in listPackages:
         for version in SPECS.getData().getVersions(package):
             release = SPECS.getData().getRelease(package, version)
             listRPMPackages = SPECS.getData().getRPMPackages(package, version)
             srpmFileName = package + "-" + version + "-" + release + ".src.rpm"
             srpmFiles = cmdUtils.findFile(srpmFileName, constants.sourceRpmPath)
             srpmFile = None
             if len(srpmFiles) == 1:
                 srpmFile = srpmFiles[0]
             debugrpmFileName = package + "-debuginfo-" + version + "-" + release + "*"
             debugrpmFiles = cmdUtils.findFile(debugrpmFileName, constants.rpmPath)
             debugrpmFile = None
             if len(debugrpmFiles) == 1:
                 debugrpmFile = debugrpmFiles[0]
             pkgUtils = PackageUtils(self.logName, self.logPath)
             for rpmPkg in listRPMPackages:
                 rpmFile = pkgUtils.findRPMFile(rpmPkg, version)
                 if rpmFile is not None:
                     listPkgAttributes = {"sourcerpm":srpmFile, "rpm":rpmFile,
                                          "debugrpm":debugrpmFile}
                     self.pkgList[rpmPkg+"-"+version] = listPkgAttributes
                     self.logger.debug("Added " + rpmPkg + "-" + version + " to the package info json")
                 else:
                     self.logger.debug("Missing rpm file for package:" + rpmPkg)

    def _verifyShaAndGetSourcePath(self, source, package, index=0):
        cmdUtils = CommandUtils()
        # Fetch/verify sources if sha1 not None.
        sha1 = SPECS.getData().getSHA1(package, source, index)
        if sha1 is not None:
            PullSources.get(package, source, sha1, constants.sourcePath,
                            constants.pullsourcesConfig, self.logger)

        sourcePath = cmdUtils.findFile(source, constants.sourcePath)
        if not sourcePath:
            sourcePath = cmdUtils.findFile(source, constants.specPath)
            if not sourcePath:
                if sha1 is None:
                    self.logger.error("No sha1 found or missing source for " + source)
                    raise Exception("No sha1 found or missing source for " + source)
                else:
                    self.logger.error("Missing source: " + source +
                                      ". Cannot find sources for package: " + package)
                    raise Exception("Missing source")
        else:
            if sha1 is None:
                self.logger.error("No sha1 found for "+source)
                raise Exception("No sha1 found")
        if len(sourcePath) > 1:
            self.logger.error("Multiple sources found for source:" + source + "\n" +
                              ",".join(sourcePath) +"\nUnable to determine one.")
            raise Exception("Multiple sources found")
        return sourcePath

    def installCustomToolChainRPMS(self, chrootID, listOfToolChainPkgs, packageName):
        self.logger.info("Installing package specific tool chain RPMs for " + packageName +
                         ".......")
        rpmFiles = ""
        packages = ""
        cmdUtils = CommandUtils()
        for package in listOfToolChainPkgs:
            pkgUtils = PackageUtils(self.logName, self.logPath)
            print("DEBUG:" + package)
            if "openjre8" in packageName or "openjdk8" in packageName:
                # x86_64 has openjdk/jre as a published rpms but aarch64 has openjdk8/jre8
                # Remove this condition after publishxrpms for x86_^4 got updated
                if ((package == "openjdk" or package == "openjre") and
                        platform.machine() == "aarch64"):
                    package = package + "8"
                rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishXRPMRepo)
            else:
                rpmFile = self.findRPMFileInGivenLocation(package, constants.prevPublishRPMRepo)
            if rpmFile is None:
                self.logger.error("Unable to find rpm "+ package +
                                  " in current and previous versions")
                raise Exception("Input Error")
            rpmFiles += " " + rpmFile
            packages += " " + package

        self.logger.debug("Installing custom rpms:" + packages)
        cmd = (self.rpmCommand + " -i -v --nodeps --noorder --force --root " +
               chrootID + " --define \'_dbpath /var/lib/rpm\' " + rpmFiles)
        retVal = cmdUtils.runCommandInShell(cmd, self.logPath +
                                            "/install_custom_toolchain_rpms.log")
        if not retVal:
            self.logger.debug("Command Executed:" + cmd)
            self.logger.error("Installing tool chain  failed")
            raise Exception("RPM installation failed")
        self.logger.info("Successfully installed all Tool Chain X RPMS")

 def _copySourcesToContainer(self, listSourceFiles, package, containerID, destDir, index=0):
     cmdUtils = CommandUtils()
     for source in listSourceFiles:
         sourcePath = self._verifyShaAndGetSourcePath(source, package, index)
         self.logger.info("Copying source file: " + sourcePath[0])
         copyCmd = "docker cp " + sourcePath[0] + " " + containerID.short_id + ":" + destDir
         cmdUtils.runCommandInShell(copyCmd)

    def adjustGCCSpecs(self, package, chrootID, logPath):
        opt = ""
        # TODO: reading of hardening flag from spec files
        if package == "linux" or package == "glibc":
            opt = " clean"
        elif package.startswith("xf86-") or package.startswith("xorg-server") :
            opt = " nonow"

        shutil.copy2(self.adjustGCCSpecScript,  chrootID+"/tmp/"+self.adjustGCCSpecScript)
        cmdUtils=CommandUtils()
        cmd = "/tmp/"+self.adjustGCCSpecScript+opt
        logFile = logPath+"/adjustGCCSpecScript.log"
        chrootCmd=self.runInChrootCommand+" "+chrootID
        retryCount=10
        returnVal=False
        while retryCount > 0:
            returnVal = cmdUtils.runCommandInShell(cmd, logFile, chrootCmd)
            if returnVal:
                return
            self.logger.error("Failed while adjusting gcc specs")
            self.logger.error("Retrying again .....")
            retryCount = retryCount - 1
            sleep(5)
        if not returnVal:
            self.logger.error("Failed while adjusting gcc specs")
            raise "Failed while adjusting gcc specs"

    def copySourcesTobuildroot(self,listSourceFiles,package,destDir):
        cmdUtils = CommandUtils()
        for source in listSourceFiles:
            # Fetch/verify sources if sha1 not None.
            sha1 = constants.specData.getSHA1(package, source)
            if sha1 is not None:
                PullSources.get(source, sha1, constants.sourcePath, constants.pullsourcesConfig)

            sourcePath = cmdUtils.findFile(source,constants.sourcePath)
            if sourcePath is None or len(sourcePath) == 0:
                sourcePath = cmdUtils.findFile(source,constants.specPath)
                if sourcePath is None or len(sourcePath) == 0:
                    if sha1 is None:
                        self.logger.error("No sha1 found or missing source for "+source)
                        raise Exception("No sha1 found or missing source")
                    else:
                        self.logger.error("Missing source: "+source+". Cannot find sources for package: "+package)
                        raise Exception("Missing source")
            else:
                if sha1 is None:
                    self.logger.error("No sha1 found for "+source)
                    raise Exception("No sha1 found")
            if len(sourcePath) > 1:
                self.logger.error("Multiple sources found for source:"+source+"\n"+ ",".join(sourcePath) +"\nUnable to determine one.")
                raise Exception("Multiple sources found")
            self.logger.info("Copying... Source path :" + source + " Source filename: " + sourcePath[0])
            shutil.copy2(sourcePath[0], destDir)