Python File.FileUtils类代码示例

    def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
        """
        Returns script that have to be injected in the jobscript for postprocessing on the WN
        """

        import inspect
        script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
        script_location = os.path.join( script_path, 'uploadScript.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        upload_script = FileUtils.loadScript(script_location, '')

        WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py' )
        script = FileUtils.loadScript(WNscript_location, '###INDENT###')

        selfConstructedLFNs = False

        if self.remoteDir == '' and self.lfn == '':
            import datetime
            t = datetime.datetime.now()
            this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
            self.lfn = os.path.join(configDirac['DiracLFNBase'], 'GangaFiles_%s' % this_date)
            selfConstructedLFNs = True

        if self.remoteDir == '' and self.lfn != '':
            self.remoteDir = configDirac['DiracLFNBase']

        if self.remoteDir[:4] == 'LFN:':
            lfn_base = self.remoteDir[4:]
        else:
            lfn_base = self.remoteDir


        for this_file in outputFiles:
            isCompressed = this_file.namePattern in patternsToZip

            if not regex.search(this_file.namePattern) is None:
                script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
            else:
                script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
                script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))


        if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
            script_env = self._getDiracEnvStr()
        else:
            script_env = str(None)

        script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])

        replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
                        '###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
                        '###INDENT###' : indent,
                        '###LOCATIONSFILE###' : postProcessLocationsFP,
                        '###DIRAC_ENV###' : script_env}

        for k, v in replace_dict.iteritems():
            script = script.replace(str(k), str(v))

        return script

    def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
        """
        Returns script that have to be injected in the jobscript for postprocessing on the WN
        """

        script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
        script_location = os.path.join( script_path, 'uploadScript.py.template')

        upload_script = FileUtils.loadScript(script_location, '')

        WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py.template' )
        script = FileUtils.loadScript(WNscript_location, '')

        if not self.remoteDir:
            try:
                job = self.getJobObject()
                lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('.'), "OutputFiles")
            except AssertionError:
                t = datetime.datetime.now()
                this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
                lfn_folder = os.path.join('GangaFiles_%s' % this_date)
            lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), lfn_folder)
        else:
            lfn_base = oa.path.join(DiracFile.diracLFNBase(self.credential_requirements), self.remoteDir)


        for this_file in outputFiles:
            isCompressed = this_file.namePattern in patternsToZip

            if not regex.search(this_file.namePattern) is None:
                script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
            else:
                script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
                script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))


        if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
            script_env = self._getDiracEnvStr()
        else:
            script_env = str(None)

        script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])

        replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
                        '###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
                        '###INDENT###' : indent,
                        '###LOCATIONSFILE###' : postProcessLocationsFP,
                        '###DIRAC_ENV###' : script_env}

        for k, v in replace_dict.iteritems():
            script = script.replace(str(k), str(v))

        return script

def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[outputfileClassName].append(outputFile)

    if not patternsToZip:
        if not any(outputFilesProcessedOnWN.values()):
            return ""

    logger.debug("Process: '%s' on WN" % str(outputFilePostProcessingOnWN))

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        if os.path.isfile(currentFile):
            os.system("gzip %s" % currentFile)

postprocesslocations = open(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'a+')  
"""

    from Ganga.GPIDev.Lib.File import FileUtils
    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

    def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
        """
        Returns script that have to be injected in the jobscript for postprocessing on the WN
        """
        lcgCommands = []

        for outputFile in outputFiles:
            lcgCommands.append('lcgse %s %s %s' % (
                outputFile.namePattern, outputFile.lfc_host,  outputFile.getUploadCmd()))
            logger.debug("OutputFile (%s) cmd for WN script is: %s" %
                         (outputFile.namePattern, outputFile.getUploadCmd()))

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                        'scripts/LCGSEFileWNScript.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '###INDENT###')

        script = script.replace('###LCGCOMMANDS###', str(lcgCommands))
        script = script.replace('###PATTERNSTOZIP###', str(patternsToZip))
        script = script.replace('###INDENT###', indent)
        script = script.replace('###POSTPROCESSLOCATIONSFP###', postProcessLocationsFP)

        return script

def getWNCodeForDownloadingInputFiles(job, indent):
    """
    Generate the code to be run on the WN to download input files
    """

    from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset

    if (
        job.inputfiles is None
        or len(job.inputfiles) == 0
        and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles))
    ):
        return ""

    insertScript = """\n
"""

    # first, go over any LocalFiles in GangaDatasets to be transferred
    # The LocalFiles in inputfiles have already been dealt with
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        for inputFile in job.inputdata.files:
            inputfileClassName = getName(inputFile)

            if inputfileClassName == "LocalFile":

                # special case for LocalFile
                if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]:
                    # create symlink
                    shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
   os.symlink(f, os.path.basename(f)) 
"""
                    shortScript = FileUtils.indentScript(shortScript, "###INDENT####")

                    insertScript += shortScript

                    insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList())

    # if GangaDataset is used, check if they want the inputfiles transferred
    inputfiles_list = job.inputfiles
    if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
        inputfiles_list += job.inputdata.files

    for inputFile in inputfiles_list:

        inputfileClassName = getName(inputFile)

        if outputFilePostProcessingOnWN(job, inputfileClassName):
            inputFile.processWildcardMatches()
            if inputFile.subfiles:
                for subfile in inputFile.subfiles:
                    insertScript += subfile.getWNScriptDownloadCommand(indent)
            else:
                insertScript += inputFile.getWNScriptDownloadCommand(indent)

    insertScript = insertScript.replace("###INDENT###", indent)

    return insertScript

    def preparejob(self, jobconfig, master_input_sandbox):

        job = self.getJobObject()
        # print str(job.backend_output_postprocess)
        mon = job.getMonitoringService()
        import Ganga.Core.Sandbox as Sandbox
        subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))

        appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
        if self.nice:
            appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
        if self.nice < 0:
            logger.warning('increasing process priority is often not allowed, your job may fail due to this')

        sharedoutputpath = job.getOutputWorkspace().getPath()
        ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
        outputpatterns = jobconfig.outputbox
        environment = dict() if jobconfig.env is None else jobconfig.env

        import tempfile
        workdir = tempfile.mkdtemp(dir=config['location'])

        import inspect
        script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                        'LocalHostExec.py')

        from Ganga.GPIDev.Lib.File import FileUtils
        script = FileUtils.loadScript(script_location, '')

        script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))

        from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
        from Ganga.Utility.Config import getConfig
        jobidRepr = repr(job.getFQID('.'))


        script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
        script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
        script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
        script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))

        script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
        script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
        script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
        script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
        script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
        script = script.replace('###JOBID###', jobidRepr)
        script = script.replace('###ENVIRONMENT###', repr(environment))
        script = script.replace('###WORKDIR###', repr(workdir))
        script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))

        self.workdir = workdir

        script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))

        wrkspace = job.getInputWorkspace()
        scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)

        return scriptPath

def downloadWrapper(app):
    from os.path import join, split
    from Ganga.GPIDev.Lib.File import FileBuffer
    import string

    from Ganga.GPIDev.Lib.File import getSharedPath

    rootsys = join('.', 'root')
    rootenv = {'ROOTSYS': rootsys}

    script = app.script
    if script == File():
        if not app.usepython:
            script = File(defaultScript())
        else:
            script = File(defaultPyRootScript())
    else:
        script = File(os.path.join(os.path.join(Ganga.GPIDev.Lib.File.getSharedPath(), app.is_prepared.name), os.path.basename(app.script.name)))

    commandline = ''
    scriptPath = join('.', script.subdir, split(script.name)[1])
    if not app.usepython:
        # Arguments to the ROOT script needs to be a comma separated list
        # enclosed in (). Strings should be enclosed in escaped double quotes.
        arglist = []
        for arg in app.args:
            if isinstance(arg, str):
                arglist.append('\\\'' + arg + '\\\'')
            else:
                arglist.append(arg)
        rootarg = '\(\"' + string.join([str(s) for s in arglist], ',') + '\"\)'

        # use root
        commandline = 'root.exe -b -q ' + scriptPath + rootarg + ''
    else:
        # use python
        pyarg = string.join([str(s) for s in app.args], ' ')
        commandline = '\'%(PYTHONCMD)s ' + scriptPath + ' ' + pyarg + ' -b \''

    logger.debug("Command line: %s: ", commandline)

    # Write a wrapper script that installs ROOT and runs script
    script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                                   'wrapperScriptTemplate.py')
    from Ganga.GPIDev.Lib.File import FileUtils
    wrapperscript = FileUtils.loadScript(script_location, '')

    wrapperscript = wrapperscript.replace('###COMMANDLINE###', commandline)
    wrapperscript = wrapperscript.replace('###ROOTVERSION###', app.version)
    wrapperscript = wrapperscript.replace('###SCRIPTPATH###', scriptPath)
    wrapperscript = wrapperscript.replace('###USEPYTHON###', str(app.usepython))

    logger.debug('Script to run on worker node\n' + wrapperscript)
    scriptName = "rootwrapper_generated_%s.py" % randomString()
    runScript = FileBuffer(scriptName, wrapperscript, executable=1)

    inputsandbox = app._getParent().inputsandbox + [script]
    return runScript, inputsandbox, rootenv

def getWNCodeForOutputPostprocessing(job, indent):

    # dict containing the list of outputfiles that need to be processed on the
    # WN for every file type
    outputFilesProcessedOnWN = {}
    patternsToZip = []

    if len(job.outputfiles) == 0:
        return ""
    else:
        for outputFile in job.outputfiles:

            outputfileClassName = getName(outputFile)
            backendClassName = getName(job.backend)

            if outputFile.compressed:
                if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
                    patternsToZip.append(outputFile.namePattern)
                elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
                    patternsToZip.append(outputFile.namePattern)

            if outputfileClassName not in outputFilesProcessedOnWN.keys():
                outputFilesProcessedOnWN[outputfileClassName] = []

            if outputFilePostProcessingOnWN(job, outputfileClassName):
                outputFilesProcessedOnWN[
                    outputfileClassName].append(outputFile)

    shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
    for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
        os.system("gzip %s" % currentFile)

postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')  
"""

    shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

    insertScript = shortScript

    insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
    insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])

    for outputFileName in outputFilesProcessedOnWN.keys():

        if len(outputFilesProcessedOnWN[outputFileName]) > 0:

            insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')

    insertScript += """\n
###INDENT###postprocesslocations.close()
"""
    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

def gaudi_script_template():
    '''Creates the script that will be executed by DIRAC job. '''

    import inspect
    script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                   'GaudiTemplate.py')

    from Ganga.GPIDev.Lib.File import FileUtils
    script_template = FileUtils.loadScript(script_location, '')

    return script_template

def defaultScript():
    tmpdir = tempfile.mktemp()
    os.mkdir(tmpdir)
    fname = os.path.join(tmpdir, 'test.C')
    script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                   'defaultRootScript.C')
    from Ganga.GPIDev.Lib.File import FileUtils
    script = FileUtils.loadScript(script_location, '')
    with open(fname, 'w') as f:
        f.write(script)
    return fname

def diracAPI_script_template():
    """ Generate and return the DiracAPI job submission template """

    import inspect
    import os.path
    script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                   'DiracRTHScript.py')

    from Ganga.GPIDev.Lib.File import FileUtils
    script_template = FileUtils.loadScript(script_location, '')

    return script_template

def getWNCodeForDownloadingInputFiles(job, indent):
    """
    Generate the code to be run on the WN to download input files
    """

    if len(job.inputfiles) == 0 and (not job.inputdata or job.inputdata._name != "GangaDataset" or not job.inputdata.treat_as_inputfiles):
        return ""

    insertScript = """\n
"""

    # first, go over any LocalFiles in GangaDatasets to be transferred
    # The LocalFiles in inputfiles have already been dealt with
    if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
        for inputFile in job.inputdata.files:
            inputfileClassName = stripProxy(inputFile).__class__.__name__

            if inputfileClassName == "LocalFile":

                # special case for LocalFile
                if stripProxy(job.backend).__class__.__name__ in ['Localhost', 'Batch', 'LSF', 'Condor', 'PBS']:
                    # create symlink
                    shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
   os.symlink(f, os.path.basename(f)) 
"""
                    shortScript = FileUtils.indentScript(shortScript, '###INDENT####')

                    insertScript += shortScript

                    insertScript = insertScript.replace('###FILELIST###', "%s" % inputFile.getFilenameList())

    # if GangaDataset is used, check if they want the inputfiles transferred
    inputfiles_list = job.inputfiles
    if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
        inputfiles_list += job.inputdata.files

    for inputFile in inputfiles_list:

        inputfileClassName = stripProxy(inputFile).__class__.__name__

        if outputFilePostProcessingOnWN(job, inputfileClassName):
            inputFile.processWildcardMatches()
            if inputFile.subfiles:
                for subfile in inputFile.subfiles:
                    insertScript += subfile.getWNScriptDownloadCommand(indent)
            else:
                insertScript += inputFile.getWNScriptDownloadCommand(indent)

    insertScript = insertScript.replace('###INDENT###', indent)

    return insertScript

def diracAPI_script_template():
    # NOTE setOutputData(replicate) replicate keyword only for LHCbDirac. must
    # move there when get a chance.

    import inspect
    import os.path
    script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
                                   'DiracRTHScript.py')

    from Ganga.GPIDev.Lib.File import FileUtils
    script_template = FileUtils.loadScript(script_location, '')

    return script_template

    def _WN_wildcard_script(self, namePattern, lfnBase, compressed):
        wildcard_str =  """
for f in glob.glob('###NAME_PATTERN###'):
    processes.append(uploadFile(os.path.basename(f), '###LFN_BASE###', ###COMPRESSED###, '###NAME_PATTERN###'))
"""
        wildcard_str = FileUtils.indentScript(wildcard_str, '###INDENT###')

        replace_dict = { '###NAME_PATTERN###' : namePattern,
                         '###LFN_BASE###' : lfnBase,
                         '###COMPRESSED###' : compressed }

        for k, v in replace_dict.iteritems():
            wildcard_str = wildcard_str.replace(str(k), str(v))

        return wildcard_str

    def getWNScriptDownloadCommand(self, indent):

        # create symlink
        shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
    if not os.path.exists(os.path.basename(f)):
        os.symlink(f, os.path.basename(f))
"""
        from Ganga.GPIDev.Lib.File import FileUtils
        shortScript = FileUtils.indentScript(shortScript, '###INDENT###')

        shortScript = shortScript.replace('###FILELIST###', "%s" % self.getFilenameList())

        return shortScript