Python DataManager.getFile方法代码示例

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
def getFile(lfn, se=""):
    dm = DataManager()

    download_ok = 0
    get_active_replicas_ok = False
    lfn_on_se = False
    error_msg = ""
    if se:
        for i in range(0, 5):
            result = dm.getActiveReplicas(lfn)
            if result["OK"] and result["Value"]["Successful"]:
                get_active_replicas_ok = True
                lfnReplicas = result["Value"]["Successful"]
                if se in lfnReplicas[lfn]:
                    lfn_on_se = True
                    break
            time.sleep(3)
            print "- Get replicas for %s failed, try again" % lfn

        if not get_active_replicas_ok:
            return S_ERROR("Get replicas error: %s" % lfn)

    if lfn_on_se:
        se = StorageElement(se)
        # try 5 times
        for j in range(0, 5):
            result = se.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- %s getStorageFile(%s) failed, try again" % (lfn, se)
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 1
            else:
                error_msg = "Downloading %s from SE %s error!" % (lfn, se)
        else:
            error_msg = result["Message"]
    else:
        if se:
            print 'File %s not found on SE "%s" after %s tries, trying other SE' % (lfn, se, i + 1)
        # try 5 times
        for j in range(0, 5):
            result = dm.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- getFile(%s) failed, try again" % lfn
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 2
            else:
                error_msg = "Downloading %s from random SE error!" % lfn
        else:
            error_msg = result["Message"]

    if download_ok:
        return S_OK({lfn: {"DownloadOK": download_ok, "Retry": j + 1}})

    return S_ERROR(error_msg)

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
    def _get_file(self, lfn):
        dm = DataManager()
        result = dm.getFile(lfn, "")
        if not result['OK']:
            return S_ERROR(result['Message'])

        if result['Value']['Failed']:
            return S_ERROR(result['Value'])
        return result

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
  def getProcessList(self): 
    """ Get the :mod:`ProcessList <ILCDIRAC.Core.Utilities.ProcessList.ProcessList>`
    needed by :mod:`Whizard <ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard>`.

    :return: process list object
    """
    processlistpath = gConfig.getValue("/LocalSite/ProcessListPath", "")
    if not processlistpath:
      gLogger.info('Will download the process list locally. To gain time, please put it somewhere and add to \
      your dirac.cfg the entry /LocalSite/ProcessListPath pointing to the file')
      pathtofile = self.ops.getValue("/ProcessList/Location", "")
      if not pathtofile:
        gLogger.error("Could not get path to process list")
        processlist = ""
      else:
        datMan = DataManager()
        datMan.getFile(pathtofile)
        processlist = os.path.basename(pathtofile)   
    else:
      processlist = processlistpath
    self.processList = ProcessList(processlist)
    return self.processList

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
 def __prepareFileForHTTP( self, lfn, key ):
   """ proxied preapre file for HTTP """
   global HTTP_PATH
   
   res = self.__prepareSecurityDetails()
   if not res['OK']:
     return res
 
   # Clear the local cache
   getFileDir = "%s/%s" % ( HTTP_PATH, key )
   os.makedirs(getFileDir)
  
   # Get the file to the cache 
   from DIRAC.DataManagementSystem.Client.DataManager import DataManager
   dataMgr = DataManager()
   result = dataMgr.getFile( lfn, destinationDir = getFileDir )
   result['CachePath'] = getFileDir
   return result  

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
def downloadFile(tarballURL, app_tar, folder_name):
  """ Get the file locally.
  """
  #need to make sure the url ends with /, other wise concatenation below returns bad url
  if tarballURL[-1] != "/":
    tarballURL += "/"

  app_tar_base = os.path.basename(app_tar)
  if tarballURL.find("http://")>-1:
    try :
      gLogger.debug("Downloading software", '%s' % (folder_name))
      #Copy the file locally, don't try to read from remote, soooo slow
      #Use string conversion %s%s to set the address, makes the system more stable
      urllib.urlretrieve("%s%s" % (tarballURL, app_tar), app_tar_base)
    except IOError as err:
      gLogger.exception(str(err))
      return S_ERROR('Exception during url retrieve: %s' % str(err))
  else:
    datMan = DataManager()
    resget = datMan.getFile("%s%s" % (tarballURL, app_tar))
    if not resget['OK']:
      gLogger.error("File could not be downloaded from the grid")
      return resget
  return S_OK()

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
class Whizard2Analysis(ModuleBase):
  """
  Specific Module to run a Whizard2 job.
  """
  def __init__(self):
    super(Whizard2Analysis, self).__init__()
    self.enable = True
    self.STEP_NUMBER = ''
    self.result = S_ERROR()
    self.applicationName = 'whizard2'
    self.startFrom = 0
    self.randomSeed = -1
    self.whizard2SinFile = ''
    self.eventstring = ['+++ Generating event']
    self.decayProc = ['decay_proc']
    self.integratedProcess = ''
    self.datMan = DataManager()

  def applicationSpecificInputs(self):
    """ Resolve all input variables for the module here.

    :return: S_OK()
    """
    self.randomSeed = self._determineRandomSeed()

    if "IS_PROD" in self.workflow_commons and self.workflow_commons["IS_PROD"]:
      self.OutputFile = getProdFilename(self.OutputFile,
                                        int(self.workflow_commons["PRODUCTION_ID"]),
                                        int(self.workflow_commons["JOB_ID"]),
                                        self.workflow_commons,
                                       )

    return S_OK('Parameters resolved')

  def resolveIntegratedProcess(self):
    """Check if integrated process is set and act accordingly.

    If the integrated process was given as a tarball it should already be available in the working directory and we do
    nothing.
    """
    if not self.integratedProcess:
      return S_OK()

    # integratedProcess is set, check CVMFS or filecatalog
    processes = self.ops.getOptionsDict('/AvailableTarBalls/%s/whizard2/%s/integrated_processes/processes' %
                                        ('x86_64-slc5-gcc43-opt', self.applicationVersion))

    if not processes['OK']:
      LOG.error('Could not resolve known integrated processes', processes['Message'])
      return processes

    options = self.ops.getOptionsDict('/AvailableTarBalls/%s/whizard2/%s/integrated_processes' %
                                      ('x86_64-slc5-gcc43-opt', self.applicationVersion))
    if not options['OK']:
      LOG.error('Failed to get integrated processes options', options['Message'])
      return options

    cvmfsPath = options['Value'].get('CVMFSPath', '')
    tarballURL = options['Value'].get('TarBallURL', '')
    processTarball = processes['Value'].get(self.integratedProcess, '')

    localTarball = os.path.join(cvmfsPath, processTarball)
    if os.path.exists(localTarball):
      LOG.info('Tarball found on cvmfs: %r' % localTarball)
      return extractTarball(localTarball, os.getcwd())

    tarballLFN = os.path.join(tarballURL, processTarball)
    LOG.info('Trying to download tarball', tarballLFN)
    getFile = self.datMan.getFile(tarballLFN)
    if not getFile['OK']:
      LOG.error('Failed to download tarball', getFile['Message'])
      return getFile
    return extractTarball(os.path.split(tarballLFN)[1], os.getcwd())

  def runIt(self):
    """
    Called by JobAgent

    Execute the following:
      - get the environment variables that should have been set during installation
      - prepare the steering file and command line parameters
      - run Whizard2 on this steering file and catch the exit status

    :rtype: :func:`~DIRAC.Core.Utilities.ReturnValues.S_OK`, :func:`~DIRAC.Core.Utilities.ReturnValues.S_ERROR`
    """
    self.result = S_OK()
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      LOG.error("Failed to resolve input parameters:", self.result['Message'])
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('Whizard2 should not proceed as previous step did not end properly')

    resIntProc = self.resolveIntegratedProcess()
    if not resIntProc['OK']:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]

#.........这里部分代码省略.........
    self.assert_(removeRes['OK'])
    self.assert_(removeRes['Value'].has_key('Successful'))
    self.assert_(removeRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeRes['Value']['Successful'][lfn])

  def test_registerFile(self):
    lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time()
    physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
    fileSize = 10000
    storageElementName = 'CERN-RAW'
    fileGuid = makeGuid()
    fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
    registerRes = self.dataManager.registerFile(fileTuple)
    removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn)
    removeFileRes = self.dataManager.removeFile(lfn)

    # Check that the file registration was done correctly
    self.assert_(registerRes['OK'])
    self.assert_(registerRes['Value'].has_key('Successful'))
    self.assert_(registerRes['Value']['Successful'].has_key(lfn))
    self.assert_(registerRes['Value']['Successful'][lfn])
    # Check that the replica removal was successful
    self.assert_(removeCatalogReplicaRes['OK'])
    self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
    self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
    # Check that the removal was successful
    self.assert_(removeFileRes['OK'])
    self.assert_(removeFileRes['Value'].has_key('Successful'))
    self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeFileRes['Value']['Successful'][lfn])

  def test_registerReplica(self):
    print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time()
    physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
    fileSize = 10000
    storageElementName = 'CERN-RAW'
    fileGuid = makeGuid()
    fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
    registerRes = self.dataManager.registerFile(fileTuple)
    seName = 'GRIDKA-RAW'
    replicaTuple = (lfn,physicalFile,seName)
    registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
    removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn)
    removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn)
    removeFileRes = self.dataManager.removeFile(lfn)

    # Check that the file registration was done correctly
    self.assert_(registerRes['OK'])
    self.assert_(registerRes['Value'].has_key('Successful'))
    self.assert_(registerRes['Value']['Successful'].has_key(lfn))
    self.assert_(registerRes['Value']['Successful'][lfn])
    # Check that the replica registration was successful
    self.assert_(registerReplicaRes['OK'])
    self.assert_(registerReplicaRes['Value'].has_key('Successful'))
    self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
    self.assert_(registerReplicaRes['Value']['Successful'][lfn])
    # Check that the replica removal was successful
    self.assert_(removeCatalogReplicaRes1['OK'])
    self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
    self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
    self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
    # Check that the replica removal was successful
    self.assert_(removeCatalogReplicaRes2['OK'])
    self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
    self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
    self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
    # Check that the removal was successful
    self.assert_(removeFileRes['OK'])
    self.assert_(removeFileRes['Value'].has_key('Successful'))
    self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeFileRes['Value']['Successful'][lfn])

  def test_putAndRegisterGet(self):
    print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
    lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time()
    diracSE = 'GRIDKA-RAW'
    putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
    getRes = self.dataManager.getFile(lfn)
    removeRes = self.dataManager.removeFile(lfn)
    localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn))
    if os.path.exists(localFilePath):
      os.remove(localFilePath)

    # Check that the put was successful
    self.assert_(putRes['OK'])
    self.assert_(putRes['Value'].has_key('Successful'))
    self.assert_(putRes['Value']['Successful'].has_key(lfn))
    self.assert_(putRes['Value']['Successful'][lfn])
    # Check that the replica removal was successful
    self.assert_(getRes['OK'])
    self.assert_(getRes['Value'].has_key('Successful'))
    self.assert_(getRes['Value']['Successful'].has_key(lfn))
    self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath)
    # Check that the removal was successful
    self.assert_(removeRes['OK'])
    self.assert_(removeRes['Value'].has_key('Successful'))
    self.assert_(removeRes['Value']['Successful'].has_key(lfn))
    self.assert_(removeRes['Value']['Successful'][lfn])

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
class WhizardAnalysis(ModuleBase):
  """
  Specific Module to run a Whizard job.
  """
  def __init__(self):
    super(WhizardAnalysis, self).__init__()
    self.enable = True
    self.STEP_NUMBER = ''
    self.debug = True
    self.SteeringFile = ''
    self.OutputFile = ''
    self.NumberOfEvents = 1
    self.Lumi = 0
    self.applicationName = 'whizard'
    self.evttype = ""
    self.RandomSeed = 0
    self.getProcessInFile = False
    self.datMan = DataManager()
    self.processlist = None
    self.parameters = {}
    self.susymodel = 0
    self.Model = ''
    self.genmodel = GeneratorModels()
    self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
                        'Floating point exception', 'Event generation finished.', " n_events","luminosity", 
                        "  sum            "]
    self.excludeAllButEventString = False
    self.steeringparameters = ''
    self.options = None
    self.optionsdict = {}
    self.OptionsDictStr = ''
    self.GenLevelCutDictStr = ''
    self.genlevelcuts = {}
    self.willCut = False
    self.useGridFiles = False
    
    
  def obtainProcessList(self):
    """Internal function
    
    Get the process list from storage if whizard.in was not provided

    :return: S_OK(), S_ERROR()
    """
    
    res = self.ops.getValue("/ProcessList/Location", "")
    if not res:
      return S_ERROR("No process list found")
    processlistloc = res
    if not os.path.exists(os.path.basename(processlistloc)):
      res = self.datMan.getFile(processlistloc)
      if not res['OK']:
        LOG.error('Could not get processlist: %s' % res['Message'])
        return res
    self.processlist = ProcessList(os.path.basename(processlistloc))
    return S_OK()
    
  def applicationSpecificInputs(self):
    """Resolve module input

    :return: S_OK()
    """
    self.parameters['ENERGY'] = self.energy

    if not self.RandomSeed and self.jobID:
      self.RandomSeed = self.jobID
    if 'IS_PROD' in self.workflow_commons or 'IS_DBD_GEN_PROD' in self.workflow_commons:
      self.RandomSeed = int(str(int(self.workflow_commons["PRODUCTION_ID"])) + str(int(self.workflow_commons["JOB_ID"])))  

    self.parameters['SEED'] = self.RandomSeed
    self.parameters['NBEVTS'] = self.NumberOfEvents
    self.parameters['LUMI'] = self.Lumi

    ##EVER USED???
    if 'SusyModel' in self.step_commons:
      self.susymodel = self.step_commons['SusyModel']

    self.SteeringFile = os.path.basename(self.step_commons.get("InputFile", self.SteeringFile))
    if self.SteeringFile == "whizard.in":
      os.rename(self.SteeringFile, "whizardnew.in")
      self.SteeringFile = "whizardnew.in"

    self.parameters['PROCESS'] = self.evttype

    listofparams = self.steeringparameters.split(";")
    for param in listofparams:
      if param.count("="):
        self.parameters[param.split("=")[0]] = param.split("=")[1]
 
    if self.OptionsDictStr:
      LOG.info("Will use whizard.in definition from WhizardOptions.")
      try:
        self.optionsdict = eval(self.OptionsDictStr)
        if 'integration_input' not in self.optionsdict:
          self.optionsdict['integration_input'] = {}
        if 'seed' not in self.optionsdict['integration_input']:
          self.optionsdict['integration_input']['seed'] = int(self.RandomSeed)
        if 'process_input' in self.optionsdict:
          if 'sqrts' in self.optionsdict['process_input']:
            self.energy = self.optionsdict['process_input']['sqrts']
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
def doTheWhizardInstallation():
  """Do the instalation for new whizard version Copy libraries, create tarball,
  upload processList file add entry in configuration system

  """
  res = checkSLCVersion()
  if not res['OK']:
    gLogger.error(res['Message'])
    dexit(1)

  res = checkGFortranVersion()
  if not res['OK']:
    gLogger.error(res['Message'])
    dexit(1)

  cliParams = Params()
  cliParams.registerSwitches()
  Script.parseCommandLine( ignoreErrors= False)
  
  whizardResultFolder = cliParams.path
  platform = cliParams.platform
  whizard_version = cliParams.version
  appVersion = whizard_version
  beam_spectra_version = cliParams.beam_spectra

  if not whizardResultFolder or not whizard_version or not beam_spectra_version:
    Script.showHelp()
    dexit(2)
  
  from ILCDIRAC.Core.Utilities.ProcessList                     import ProcessList
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations     import Operations 
  from DIRAC.Interfaces.API.DiracAdmin                         import DiracAdmin
  from ILCDIRAC.Core.Utilities.FileUtils                       import upload
  from DIRAC.DataManagementSystem.Client.DataManager           import DataManager
  diracAdmin = DiracAdmin()

  modifiedCS = False

  softwareSection = "/Operations/Defaults/AvailableTarBalls"
  processlistLocation = "ProcessList/Location"

  appName = "whizard"

  ops = Operations()
  path_to_process_list = ops.getValue(processlistLocation, "")
  if not path_to_process_list:
    gLogger.error("Could not find process list location in CS")
    dexit(2)
    
  gLogger.verbose("Getting process list from file catalog")
  datMan = DataManager()
  res = datMan.getFile(path_to_process_list)
  if not res['OK']:
    gLogger.error("Error while getting process list from storage")
    dexit(2)
  gLogger.verbose("done")

  ##just the name of the local file in current working directory
  processlist = os.path.basename(path_to_process_list)
  if not os.path.exists(processlist):
    gLogger.error("Process list does not exist locally")
    dexit(2)


  pl = ProcessList(processlist)
  
  startDir = os.getcwd()
  inputlist = {}
  os.chdir(whizardResultFolder)
  folderlist = os.listdir(whizardResultFolder)

  whiz_here = folderlist.count("whizard")
  if whiz_here == 0:
    gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder)
    os.chdir(startDir)
    dexit(2)

  whizprc_here = folderlist.count("whizard.prc")
  if whizprc_here == 0:
    gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder)
    os.chdir(startDir)
    dexit(2)

  whizmdl_here = folderlist.count("whizard.mdl")
  if whizmdl_here == 0:
    gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder)
    os.chdir(startDir)
    dexit(2)
   
    
  gLogger.verbose("Preparing process list")

  ## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS
  for f in folderlist:
    if f.count(".in"):
      infile = open(f, "r")
      found_detail = False
      
      for line in infile:
        if line.count("decay_description"):
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
def getOutputData(baseDir, logLevel="INFO"):
    gLogger.setLevel(logLevel)
    exitCode = 0
    res = getProxyInfo(False, False)
    if not res["OK"]:
        gLogger.error("Failed to get client proxy information.", res["Message"])
        DIRAC.exit(71)

    print "Will search for files in %s" % baseDir
    activeDirs = [baseDir]
    # ######################################################################################################## #
    # before is from dirac-dms-user-lfns
    rm = DataManager()
    allFiles = []
    while len(activeDirs) > 0:
        currentDir = activeDirs[0]
        res = rm.getFilesFromDirectory(currentDir)
        activeDirs.remove(currentDir)
        if not res["OK"]:
            gLogger.error("Error retrieving directory contents", "%s %s" % (currentDir, res["Message"]))
        else:
            allFiles = res["Value"]
    # ######################################################################################################## #
    # get file
    ntries = 5
    # getFile supports bulk requests
    files_to_transfer = sortList(allFiles)
    successful_files = []
    failed_files = []
    while ntries > 0:
        if len(failed_files):
            files_to_transfer = failed_files
        gLogger.info("getting the following *list* of files %s" % str(files_to_transfer))
        result = rm.getFile(files_to_transfer)
        if not result["OK"]:
            gLogger.error("Could not complete DataManager request")
            gLogger.error(str(result["Message"]))
            gLogger.info("sleep for 10s and re-try")
            time.sleep(10)
            break
        # next is to check what files we got
        successful_files = result["Value"]["Successful"].keys()
        failed_files = result["Value"]["Failed"].keys()
        if len(failed_files):
            gLogger.info("Could not retrieve one or more files")
            for key in failed_files:
                gLogger.error("%s:%s" % (key, result["Value"]["Failed"][key]))
            for s in successful_files:
                files_to_transfer.remove(s)
            for f in failed_files:
                gLogger.verbose("could not retrieve: %s" % f)
        else:
            break
        ntries -= 1
    if len(failed_files):
        gLogger.error("ERROR could not get all files after %i trials. Giving up :(" % ntries)
        exitCode = 23

    if exitCode:
        return {"OK": False, "Message": "Failed to finish operations.", "RC": exitCode}
    return S_OK(successful_files)

# 需要导入模块: from DIRAC.DataManagementSystem.Client.DataManager import DataManager [as 别名]
# 或者: from DIRAC.DataManagementSystem.Client.DataManager.DataManager import getFile [as 别名]
class OverlayInput (ModuleBase):
  """ Download the files for overlay.
  """
  def __init__(self):
    super(OverlayInput, self).__init__()
    self.enable = True
    self.STEP_NUMBER = ''
    self.applicationName = 'OverlayInput'
    self.curdir = os.getcwd()
    self.applicationLog = ''
    self.printoutflag = ''
    self.prodid = 0
    self.detector = '' ##needed for backward compatibility
    self.detectormodel = ""
    self.energytouse = ''
    self.energy = 0
    self.nbofeventsperfile = 100
    self.lfns = []
    self.nbfilestoget = 0
    self.BkgEvtType = 'gghad'
    self.metaEventType = self.BkgEvtType
    self.BXOverlay = 0
    self.ggtohadint = 3.2
    self.nbsigeventsperfile = 0
    self.nbinputsigfile = 1
    self.NbSigEvtsPerJob = 0
    self.datMan = DataManager()
    self.fcc = FileCatalogClient()
    self.site = DIRAC.siteName()
    self.useEnergyForFileLookup = True
    self.machine = 'clic_cdr'
    self.pathToOverlayFiles = ''
    self.processorName = ''

  def applicationSpecificInputs(self):

    self.pathToOverlayFiles = self.step_commons.get("pathToOverlayFiles", self.pathToOverlayFiles)

    if 'Detector' in self.step_commons:
      self.detectormodel = self.step_commons['Detector']
    if not self.detectormodel and not self.detector and not self.pathToOverlayFiles:
      return S_ERROR('Detector model not defined')

    if 'Energy' in self.step_commons:
      self.energytouse = self.step_commons['Energy']

    if self.energy:
      self.energytouse = energyWithLowerCaseUnit( self.energy )
            
    if not self.energytouse and not self.pathToOverlayFiles:
      return S_ERROR("Energy not set anywhere!")

    if 'BXOverlay' in self.step_commons:
      self.BXOverlay = self.step_commons['BXOverlay']
    if not self.BXOverlay:
      return S_ERROR("BXOverlay parameter not defined")

    if 'ggtohadint' in self.step_commons:
      self.ggtohadint = self.step_commons['ggtohadint']

    if 'ProdID' in self.step_commons:
      self.prodid = self.step_commons['ProdID']

    if 'NbSigEvtsPerJob' in self.step_commons:
      self.NbSigEvtsPerJob = self.step_commons['NbSigEvtsPerJob']

    if 'BkgEvtType' in self.step_commons:
      self.BkgEvtType = self.step_commons['BkgEvtType']
    self.metaEventType = self.BkgEvtType
      
    res =  allowedBkg(self.BkgEvtType, self.energytouse, detector = self.detector, 
                      detectormodel = self.detectormodel, machine = self.machine) 
    if not res['OK']:
      return res
    if res['Value'] < 0 and not self.pathToOverlayFiles:
      return S_ERROR("No suitable ProdID") 
    #if 'Site' in self.workflow_commons:
    #  self.site = self.workflow_commons['Site']

    self.useEnergyForFileLookup = self.step_commons.get("useEnergyForFileLookup", self.useEnergyForFileLookup)

    if self.InputData:
      if self.NumberOfEvents:
        self.nbsigeventsperfile = self.NumberOfEvents
      else:
        return S_ERROR("Number of events in the signal file is missing")
      self.nbinputsigfile = len(self.InputData)

    LOG.info("Signal Events Per Job: %d " % self.NbSigEvtsPerJob)
    LOG.info("Background Event Type: %s " % self.BkgEvtType)
    LOG.info("Meta Event Type: %s " % self.metaEventType)
    LOG.info("Background Events per bunch crossing: %3.2f" % self.ggtohadint)
    LOG.info("SignalEventsPerFile: %d " % self.nbsigeventsperfile)
      
    if not self.NbSigEvtsPerJob and not self.nbsigeventsperfile:
      return S_ERROR("Could not determine the number of signal events per input file")
    return S_OK("Input variables resolved")

  def __getFilesFromFC(self):
    """ Get the list of files from the FileCatalog.
#.........这里部分代码省略.........