Python FileUtils.indentScript方法代码示例

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[outputfileClassName].append(outputFile)

    if not patternsToZip:
        if not any(outputFilesProcessedOnWN.values()):
            return ""

    logger.debug("Process: '%s' on WN" % str(outputFilePostProcessingOnWN))

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        if os.path.isfile(currentFile):
            os.system("gzip %s" % currentFile)

postprocesslocations = open(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'a+')  
"""

    from Ganga.GPIDev.Lib.File import FileUtils
    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
def getWNCodeForDownloadingInputFiles(job, indent):
    """
    Generate the code to be run on the WN to download input files
    """

    from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset

    if (
        job.inputfiles is None
        or len(job.inputfiles) == 0
        and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles))
    ):
        return ""

    insertScript = """\n
"""

    # first, go over any LocalFiles in GangaDatasets to be transferred
    # The LocalFiles in inputfiles have already been dealt with
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        for inputFile in job.inputdata.files:
            inputfileClassName = getName(inputFile)

            if inputfileClassName == "LocalFile":

                # special case for LocalFile
                if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]:
                    # create symlink
                    shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
   os.symlink(f, os.path.basename(f)) 
"""
                    shortScript = FileUtils.indentScript(shortScript, "###INDENT####")

                    insertScript += shortScript

                    insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList())

    # if GangaDataset is used, check if they want the inputfiles transferred
    inputfiles_list = job.inputfiles
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        inputfiles_list += job.inputdata.files

    for inputFile in inputfiles_list:

        inputfileClassName = getName(inputFile)

        if outputFilePostProcessingOnWN(job, inputfileClassName):
            inputFile.processWildcardMatches()
            if inputFile.subfiles:
                for subfile in inputFile.subfiles:
                    insertScript += subfile.getWNScriptDownloadCommand(indent)
            else:
                insertScript += inputFile.getWNScriptDownloadCommand(indent)

    insertScript = insertScript.replace("###INDENT###", indent)

    return insertScript

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[
                    outputfileClassName].append(outputFile)

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        os.system("gzip %s" % currentFile)

postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')  
"""

    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
def getWNCodeForDownloadingInputFiles(job, indent):
    """
    Generate the code to be run on the WN to download input files
    """

    if len(job.inputfiles) == 0 and (not job.inputdata or job.inputdata._name != "GangaDataset" or not job.inputdata.treat_as_inputfiles):
        return ""

    insertScript = """\n
"""

    # first, go over any LocalFiles in GangaDatasets to be transferred
    # The LocalFiles in inputfiles have already been dealt with
    if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
        for inputFile in job.inputdata.files:
            inputfileClassName = stripProxy(inputFile).__class__.__name__

            if inputfileClassName == "LocalFile":

                # special case for LocalFile
                if stripProxy(job.backend).__class__.__name__ in ['Localhost', 'Batch', 'LSF', 'Condor', 'PBS']:
                    # create symlink
                    shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
   os.symlink(f, os.path.basename(f)) 
"""
                    shortScript = FileUtils.indentScript(shortScript, '###INDENT####')

                    insertScript += shortScript

                    insertScript = insertScript.replace('###FILELIST###', "%s" % inputFile.getFilenameList())

    # if GangaDataset is used, check if they want the inputfiles transferred
    inputfiles_list = job.inputfiles
    if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
        inputfiles_list += job.inputdata.files

    for inputFile in inputfiles_list:

        inputfileClassName = stripProxy(inputFile).__class__.__name__

        if outputFilePostProcessingOnWN(job, inputfileClassName):
            inputFile.processWildcardMatches()
            if inputFile.subfiles:
                for subfile in inputFile.subfiles:
                    insertScript += subfile.getWNScriptDownloadCommand(indent)
            else:
                insertScript += inputFile.getWNScriptDownloadCommand(indent)

    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
    def _WN_wildcard_script(self, namePattern, lfnBase, compressed):
        wildcard_str =  """
for f in glob.glob('###NAME_PATTERN###'):
    processes.append(uploadFile(os.path.basename(f), '###LFN_BASE###', ###COMPRESSED###, '###NAME_PATTERN###'))
"""
        wildcard_str = FileUtils.indentScript(wildcard_str, '###INDENT###')

        replace_dict = { '###NAME_PATTERN###' : namePattern,
                         '###LFN_BASE###' : lfnBase,
                         '###COMPRESSED###' : compressed }

        for k, v in replace_dict.iteritems():
            wildcard_str = wildcard_str.replace(str(k), str(v))

        return wildcard_str

# 需要导入模块: from Ganga.GPIDev.Lib.File import FileUtils [as 别名]
# 或者: from Ganga.GPIDev.Lib.File.FileUtils import indentScript [as 别名]
    def getWNScriptDownloadCommand(self, indent):

        # create symlink
        shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
    if not os.path.exists(os.path.basename(f)):
        os.symlink(f, os.path.basename(f))
"""
        from Ganga.GPIDev.Lib.File import FileUtils
        shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

        shortScript = shortScript.replace('###FILELIST###', "%s" % self.getFilenameList())

        return shortScript