Python ComputingElementFactory.getCE方法代码示例

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def test_CreateAndSubmit( self ):

    jobParams = {'JobID': '1',
                 'JobType': 'Merge',
                 'CPUTime': '1000000',
                 'Executable': '$DIRACROOT/scripts/dirac-jobexec',
                 'Arguments': "helloWorld.xml -o LogLevel=DEBUG pilot.cfg",
                 'ExtraOptions': 'pilot.cfg',
                 'InputSandbox': ['helloWorld.xml', 'exe-script.py']}
    resourceParams = {}
    optimizerParams = {}

#     res = createJobWrapper( 1, jobParams, resourceParams, optimizerParams, logLevel = 'DEBUG' )
#     self.assert_( res['OK'] )
#     wrapperFile = res['Value']

    ceFactory = ComputingElementFactory()
    ceInstance = ceFactory.getCE( 'InProcess' )
    self.assert_( ceInstance['OK'] )
    computingElement = ceInstance['Value']

#     res = computingElement.submitJob( wrapperFile, self.payloadProxy )
#     self.assert_( res['OK'] )

    res = createJobWrapper( 2, jobParams, resourceParams, optimizerParams, extraOptions = 'pilot.cfg', logLevel = 'DEBUG' )
    self.assert_( res['OK'] )
    wrapperFile = res['Value']

    res = computingElement.submitJob( wrapperFile, self.payloadProxy )
    self.assert_( res['OK'] )

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def export_killPilot(self, pilotRefList ):
    """ Kill the specified pilots
    """
    # Make a list if it is not yet
    pilotRefs = list( pilotRefList )
    if type( pilotRefList ) in StringTypes:
      pilotRefs = [pilotRefList]
    
    # Regroup pilots per site and per owner
    pilotRefDict = {}
    for pilotReference in pilotRefs:
      result = pilotDB.getPilotInfo(pilotReference)
      if not result['OK'] or not result[ 'Value' ]:
        return S_ERROR('Failed to get info for pilot ' + pilotReference)
  
      pilotDict = result['Value'][pilotReference]
      owner = pilotDict['OwnerDN']
      group = pilotDict['OwnerGroup']
      queue = '@@@'.join( [owner, group, pilotDict['GridSite'], pilotDict['DestinationSite'], pilotDict['Queue']] )
      gridType = pilotDict['GridType']
      pilotRefDict.setdefault( queue, {} )
      pilotRefDict[queue].setdefault( 'PilotList', [] )
      pilotRefDict[queue]['PilotList'].append( pilotReference )
      pilotRefDict[queue]['GridType'] = gridType
      
    # Do the work now queue by queue  
    ceFactory = ComputingElementFactory()
    failed = []
    for key, pilotDict in pilotRefDict.items():
      
      owner,group,site,ce,queue = key.split( '@@@' )
      result = getQueue( site, ce, queue )
      if not result['OK']:
        return result
      queueDict = result['Value']
      gridType = pilotDict['GridType']
      result = ceFactory.getCE( gridType, ce, queueDict )
      if not result['OK']:
        return result
      ce = result['Value']
  
      if gridType in ["LCG","gLite","CREAM"]:
        group = getGroupOption(group,'VOMSRole',group)
        ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
        if not ret['OK']:
          gLogger.error( ret['Message'] )
          gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
          return S_ERROR("Failed to get the pilot's owner proxy")
        proxy = ret['Value']
        ce.setProxy( proxy )

      pilotList = pilotDict['PilotList']
      result = ce.killJob( pilotList )
      if not result['OK']:
        failed.extend( pilotList )
      
    if failed:
      return S_ERROR('Failed to kill at least some pilots')
    
    return S_OK()  

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def getQueues( self, resourceDict ):
    """ Get the list of relevant CEs and their descriptions
    """

    self.queueDict = {}
    ceFactory = ComputingElementFactory()

    for site in resourceDict:
      for ce in resourceDict[site]:
        ceDict = resourceDict[site][ce]
        qDict = ceDict.pop( 'Queues' )
        for queue in qDict:
          queueName = '%s_%s' % ( ce, queue )
          self.queueDict[queueName] = {}
          self.queueDict[queueName]['ParametersDict'] = qDict[queue]
          self.queueDict[queueName]['ParametersDict']['Queue'] = queue
          self.queueDict[queueName]['ParametersDict']['Site'] = site
          self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
          self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
          # Evaluate the CPU limit of the queue according to the Glue convention
          # To Do: should be a utility
          if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
             "SI00" in self.queueDict[queueName]['ParametersDict']:
            maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
            # For some sites there are crazy values in the CS
            maxCPUTime = max( maxCPUTime, 0 )
            maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
            si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
            queueCPUTime = 60. / 250. * maxCPUTime * si00
            self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
          qwDir = os.path.join( self.workingDirectory, queue )
          if not os.path.exists( qwDir ):
            os.makedirs( qwDir )
          self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
          ceQueueDict = dict( ceDict )
          ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
          result = ceFactory.getCE( ceName = ce,
                                    ceType = ceDict['CEType'],
                                    ceParametersDict = ceQueueDict )
          if not result['OK']:
            return result
          self.queueDict[queueName]['CE'] = result['Value']
          self.queueDict[queueName]['CEName'] = ce
          self.queueDict[queueName]['CEType'] = ceDict['CEType']
          self.queueDict[queueName]['Site'] = site
          self.queueDict[queueName]['QueueName'] = queue
          result = self.queueDict[queueName]['CE'].isValid()
          if not result['OK']:
            self.log.fatal( result['Message'] )
            return result
          if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
            self.queueDict[queueName]['BundleProxy'] = True

    return S_OK()

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def __createCE( self, ceName ):
    """
    return a CE object for the given ceName
    """
    self.log.info( "Creating %s CE" % ( ceName ) )

    ceFactory = ComputingElementFactory()
    ret = ceFactory.getCE( ceName )
    if not ret['OK']:
      self.log.warn( ret['Message'] )
      return ret
    return ret

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def initialize( self, loops = 0 ):
    """Sets default parameters and creates CE instance
    """
    # Disable monitoring
    self.am_setOption( 'MonitoringEnabled', False )
    # self.log.setLevel('debug') #temporary for debugging
    self.am_setOption( 'MaxCycles', loops )

    ceType = self.am_getOption( 'CEType', 'InProcess' )
    localCE = gConfig.getValue( '/LocalSite/LocalCE', '' )
    if localCE:
      self.log.info( 'Defining CE from local configuration = %s' % localCE )
      ceType = localCE

    # Create backend Computing Element
    ceFactory = ComputingElementFactory()
    self.ceName = ceType
    ceInstance = ceFactory.getCE( ceType )
    if not ceInstance['OK']:
      self.log.warn( ceInstance['Message'] )
      return ceInstance
    self.computingElement = ceInstance['Value']

    result = self.computingElement.getDescription()
    if not result['OK']:
      self.log.warn( "Can not get the CE description" )
      return result
    ceDict = result['Value']
    self.timeLeft = ceDict.get( 'CPUTime', 0.0 )
    self.timeLeft = gConfig.getValue( '/Resources/Computing/CEDefaults/MaxCPUTime', self.timeLeft )

    self.initTimes = os.times()
    # Localsite options
    self.siteName = gConfig.getValue( '/LocalSite/Site', 'Unknown' )
    self.pilotReference = gConfig.getValue( '/LocalSite/PilotReference', 'Unknown' )
    self.defaultProxyLength = gConfig.getValue( '/Registry/DefaultProxyLifeTime', 86400 * 5 )
    # Agent options
    # This is the factor to convert raw CPU to Normalized units (based on the CPU Model)
    self.cpuFactor = gConfig.getValue( '/LocalSite/CPUNormalizationFactor', 0.0 )
    self.jobSubmissionDelay = self.am_getOption( 'SubmissionDelay', 10 )
    self.fillingMode = self.am_getOption( 'FillingModeFlag', False )
    self.minimumTimeLeft = self.am_getOption( 'MinimumTimeLeft', 1000 )
    self.stopOnApplicationFailure = self.am_getOption( 'StopOnApplicationFailure', True )
    self.stopAfterFailedMatches = self.am_getOption( 'StopAfterFailedMatches', 10 )
    self.jobCount = 0
    self.matchFailedCount = 0
    self.extraOptions = gConfig.getValue( '/AgentJobRequirements/ExtraOptions', '' )
    # Timeleft
    self.timeLeftUtil = TimeLeft()
    self.timeLeftError = ''
    self.scaledCPUTime = 0.0
    self.pilotInfoReportedFlag = False
    return S_OK()

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def initialize( self, loops = 0 ):
    """Sets default parameters and creates CE instance
    """
    # Disable monitoring
    self.am_setOption( 'MonitoringEnabled', False )
    # self.log.setLevel('debug') #temporary for debugging
    self.am_setOption( 'MaxCycles', loops )

    ceType = self.am_getOption( 'CEType', 'InProcess' )
    localCE = gConfig.getValue( '/LocalSite/LocalCE', '' )
    if localCE:
      self.log.info( 'Defining CE from local configuration = %s' % localCE )
      ceType = localCE

    ceFactory = ComputingElementFactory()
    self.ceName = ceType
    ceInstance = ceFactory.getCE( ceType )
    if not ceInstance['OK']:
      self.log.warn( ceInstance['Message'] )
      return ceInstance

    self.initTimes = os.times()

    self.computingElement = ceInstance['Value']
    self.diracRoot = os.path.dirname( os.path.dirname( os.path.dirname( os.path.dirname( __file__ ) ) ) )
    #Localsite options
    self.siteRoot = gConfig.getValue( '/LocalSite/Root', os.getcwd() )
    self.siteName = gConfig.getValue( '/LocalSite/Site', 'Unknown' )
    self.pilotReference = gConfig.getValue( '/LocalSite/PilotReference', 'Unknown' )
    self.defaultProxyLength = gConfig.getValue( '/Registry/DefaultProxyLifeTime', 86400 * 5 )
    #Agent options
    # This is the factor to convert raw CPU to Normalized units (based on the CPU Model)
    self.cpuFactor = gConfig.getValue( '/LocalSite/CPUNormalizationFactor', 0.0 )
    defaultWrapperLocation = 'DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapperTemplate.py'
    self.jobWrapperTemplate = os.path.join( self.diracRoot,
                                            self.am_getOption( 'JobWrapperTemplate',
                                                                defaultWrapperLocation ) )
    self.jobSubmissionDelay = self.am_getOption( 'SubmissionDelay', 10 )
    self.defaultLogLevel = self.am_getOption( 'DefaultLogLevel', 'info' )
    self.fillingMode = self.am_getOption( 'FillingModeFlag', False )
    self.stopOnApplicationFailure = self.am_getOption( 'StopOnApplicationFailure', True )
    self.stopAfterFailedMatches = self.am_getOption( 'StopAfterFailedMatches', 10 )
    self.jobCount = 0
    self.matchFailedCount = 0
    #Timeleft
    self.timeLeftUtil = TimeLeft()
    self.timeLeft = gConfig.getValue( '/Resources/Computing/CEDefaults/MaxCPUTime', 0.0 )
    self.gridCEQueue = gConfig.getValue( '/Resources/Computing/CEDefaults/GridCEQueue', '' )
    self.timeLeftError = ''
    self.scaledCPUTime = 0.0
    self.pilotInfoReportedFlag = False
    return S_OK()

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
 def addComputingElement(self, ceList):
   """
     Check if a CE object for the current CE is available,
     instantiate one if necessary
   """
   for CE in ceList:
     if CE not in self.computingElementDict:
       ceFactory = ComputingElementFactory( )
       ceInstance = ceFactory.getCE( ceName = CE )
       if not ceInstance['OK']:
         self.log.error('Can not create CE object:', ceInstance['Message'])
         return
       self.computingElementDict[CE] = ceInstance['Value'].ceConfigDict
       # add the 'CE' instance at the end to avoid being overwritten
       self.computingElementDict[CE]['CE'] = ceInstance['Value']

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def getQueues( self, resourceDict ):
    """ Get the list of relevant CEs and their descriptions
    """

    self.queueDict = {}
    ceFactory = ComputingElementFactory()

    for site in resourceDict:
      for ce in resourceDict[site]:
        ceDict = resourceDict[site][ce]
        ceTags = ceDict.get( 'Tag' )
        if isinstance( ceTags, basestring ):
          ceTags = fromChar( ceTags )
        qDict = ceDict.pop( 'Queues' )
        for queue in qDict:
          queueName = '%s_%s' % ( ce, queue )
          self.queueDict[queueName] = {}
          self.queueDict[queueName]['ParametersDict'] = qDict[queue]
          self.queueDict[queueName]['ParametersDict']['Queue'] = queue
          self.queueDict[queueName]['ParametersDict']['Site'] = site
          self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
          self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
          # Evaluate the CPU limit of the queue according to the Glue convention
          # To Do: should be a utility
          if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
             "SI00" in self.queueDict[queueName]['ParametersDict']:
            maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
            # For some sites there are crazy values in the CS
            maxCPUTime = max( maxCPUTime, 0 )
            maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
            si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
            queueCPUTime = 60. / 250. * maxCPUTime * si00
            self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
          queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' )
          if queueTags and isinstance( queueTags, basestring ):
            queueTags = fromChar( queueTags )
            self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags
          if ceTags:
            if queueTags:
              allTags = list( set( ceTags + queueTags ) )
              self.queueDict[queueName]['ParametersDict']['Tag'] = allTags
            else:
              self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags

          maxMemory = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM', None )
          if maxMemory:
            # MaxRAM value is supposed to be in MB
            maxMemoryList = range( 1, int( maxMemory )/1000 + 1 )
            memoryTags = [ '%dGB' % mem for mem in maxMemoryList ]
            if memoryTags:
              self.queueDict[queueName]['ParametersDict'].setdefault( 'Tag', [] )
              self.queueDict[queueName]['ParametersDict']['Tag'] += memoryTags
          qwDir = os.path.join( self.workingDirectory, queue )
          if not os.path.exists( qwDir ):
            os.makedirs( qwDir )
          self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir

          platform = ''
          if "Platform" in self.queueDict[queueName]['ParametersDict']:
            platform = self.queueDict[queueName]['ParametersDict']['Platform']
          elif "Platform" in ceDict:
            platform = ceDict['Platform']
          elif "OS" in ceDict:
            architecture = ceDict.get( 'architecture', 'x86_64' )
            OS = ceDict['OS']
            platform = '_'.join( [architecture, OS] )
          if platform and not platform in self.platforms:
            self.platforms.append( platform )

          if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
            result = Resources.getDIRACPlatform( platform )
            if result['OK']:
              self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]

          ceQueueDict = dict( ceDict )
          ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )

          # Generate the CE object for the queue or pick the already existing one
          # if the queue definition did not change
          queueHash = self.__generateQueueHash( ceQueueDict )
          if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash:
            queueCE = self.queueCECache[queueName]['CE']
          else:
            result = ceFactory.getCE( ceName = ce,
                                      ceType = ceDict['CEType'],
                                      ceParametersDict = ceQueueDict )
            if not result['OK']:
              return result
            self.queueCECache.setdefault( queueName, {} )
            self.queueCECache[queueName]['Hash'] = queueHash
            self.queueCECache[queueName]['CE'] = result['Value']
            queueCE = self.queueCECache[queueName]['CE']

          self.queueDict[queueName]['CE'] = queueCE
          self.queueDict[queueName]['CEName'] = ce
          self.queueDict[queueName]['CEType'] = ceDict['CEType']
          self.queueDict[queueName]['Site'] = site
          self.queueDict[queueName]['QueueName'] = queue
          self.queueDict[queueName]['Platform'] = platform
          result = self.queueDict[queueName]['CE'].isValid()
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def __getGridJobOutput(self,pilotReference):
    """ Get the pilot job standard output and standard error files for the Grid
        job reference
    """

    result = pilotDB.getPilotInfo(pilotReference)
    if not result['OK'] or not result[ 'Value' ]:
      return S_ERROR('Failed to get info for pilot ' + pilotReference)

    pilotDict = result['Value'][pilotReference]
    owner = pilotDict['OwnerDN']
    group = pilotDict['OwnerGroup']

    # FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
    result = pilotDB.getPilotOutput(pilotReference)
    if result['OK']:
      stdout = result['Value']['StdOut']
      error = result['Value']['StdErr']
      if stdout or error:
        resultDict = {}
        resultDict['StdOut'] = stdout
        resultDict['StdErr'] = error
        resultDict['OwnerDN'] = owner
        resultDict['OwnerGroup'] = group
        resultDict['FileList'] = []
        return S_OK(resultDict)
      else:
        gLogger.warn( 'Empty pilot output found for %s' % pilotReference )

    gridType = pilotDict['GridType']
    if gridType in ["LCG","gLite","CREAM"]:
      group = getGroupOption(group,'VOMSRole',group)
      ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
      if not ret['OK']:
        gLogger.error( ret['Message'] )
        gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
        return S_ERROR("Failed to get the pilot's owner proxy")
      proxy = ret['Value']

      pilotStamp = pilotDict['PilotStamp']
      result = getPilotOutput( proxy, gridType, pilotReference, pilotStamp )
      if not result['OK']:
        return S_ERROR('Failed to get pilot output: '+result['Message'])
      # FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
      stdout = result['StdOut']
      error = result['StdErr']
      fileList = result['FileList']
      if stdout:
        result = pilotDB.storePilotOutput(pilotReference,stdout,error)
        if not result['OK']:
          gLogger.error('Failed to store pilot output:',result['Message'])

      resultDict = {}
      resultDict['StdOut'] = stdout
      resultDict['StdErr'] = error
      resultDict['OwnerDN'] = owner
      resultDict['OwnerGroup'] = group
      resultDict['FileList'] = fileList
      return S_OK(resultDict)
    else:
      # Instantiate the appropriate CE
      ceFactory = ComputingElementFactory()
      result = getQueue( pilotDict['GridSite'], pilotDict['DestinationSite'], pilotDict['Queue'] )
      if not result['OK']:
        return result
      queueDict = result['Value']
      result = ceFactory.getCE( gridType, pilotDict['DestinationSite'], queueDict )
      if not result['OK']:
        return result
      ce = result['Value']
      pilotStamp = pilotDict['PilotStamp']
      pRef = pilotReference
      if pilotStamp:
        pRef = pRef + ':::' + pilotStamp
      result = ce.getJobOutput( pRef )
      if not result['OK']:
        return result
      stdout,error = result['Value']
      if stdout:
        result = pilotDB.storePilotOutput(pilotReference,stdout,error)
        if not result['OK']:
          gLogger.error('Failed to store pilot output:',result['Message'])

      resultDict = {}
      resultDict['StdOut'] = stdout
      resultDict['StdErr'] = error
      resultDict['OwnerDN'] = owner
      resultDict['OwnerGroup'] = group
      resultDict['FileList'] = []
      return S_OK( resultDict )

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
  def getQueues( self ):
    """ Get the list of relevant CEs and their descriptions
    """

    ceFactory = ComputingElementFactory()
    ceTypes = self.am_getOption( 'CETypes', [] )
    ceConfList = self.am_getOption( 'CEs', [] )

    for siteName in self.siteNames:
      # Look up CE definitions in the site CS description
      ceList = []
      gridType = siteName.split( '.' )[0]
      result = gConfig.getSections( '/Resources/Sites/%s/%s/CEs' % ( gridType, siteName ) )
      if not result['OK']:
        return S_ERROR( 'Failed to look up the CS for the site %s CEs' % siteName )
      if not result['Value']:
        return S_ERROR( 'No CEs found for site %s' % siteName )
      ceTotalList = result['Value']
      for ce in ceTotalList:
        if ( ceConfList and ce in ceConfList ) or not ceConfList:
          ceType = gConfig.getValue( '/Resources/Sites/%s/%s/CEs/%s/CEType' % ( gridType, siteName, ce ), 'Unknown' )
          result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( gridType, siteName, ce ) )
          if not result['OK']:
            return S_ERROR( 'Failed to look up the CS for ce %s' % ce )
          ceDict = result['Value']
          if "SubmissionMode" in ceDict and ceDict['SubmissionMode'].lower() == "direct":
            if ceType in ceTypes:
              ceList.append( ( ce, ceType, ceDict ) )

      for ce, ceType, ceDict in ceList:
        section = '/Resources/Sites/%s/%s/CEs/%s/Queues' % ( gridType, siteName, ce )
        result = gConfig.getSections( section )
        if not result['OK']:
          return S_ERROR( 'Failed to look up the CS for queues' )
        if not result['Value']:
          return S_ERROR( 'No Queues found for site %s, ce %s' % ( siteName, ce ) )

        queues = result['Value']
        for queue in queues:
          result = gConfig.getOptionsDict( '%s/%s' % ( section, queue ) )
          if not result['OK']:
            return S_ERROR( 'Failed to look up the CS for ce,queue %s,%s' % ( ce, queue ) )

          queueName = '%s_%s' % ( ce, queue )
          self.queueDict[queueName] = {}
          self.queueDict[queueName]['ParametersDict'] = result['Value']
          self.queueDict[queueName]['ParametersDict']['Queue'] = queue
          self.queueDict[queueName]['ParametersDict']['Site'] = siteName
          self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
          self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
          # Evaluate the CPU limit of the queue according to the Glue convention
          # To Do: should be a utility
          if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
             "SI00" in self.queueDict[queueName]['ParametersDict']:
            maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
            # For some sites there are crazy values in the CS
            maxCPUTime = max( maxCPUTime, 0 )
            maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
            si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
            queueCPUTime = 60. / 250. * maxCPUTime * si00
            self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
          qwDir = os.path.join( self.workingDirectory, queue )
          if not os.path.exists( qwDir ):
            os.mkdir( qwDir )
          self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
          queueDict = dict( ceDict )
          queueDict.update( self.queueDict[queueName]['ParametersDict'] )
          result = ceFactory.getCE( ceName = ce,
                                   ceType = ceType,
                                   ceParametersDict = queueDict )
          if not result['OK']:
            return result
          self.queueDict[queueName]['CE'] = result['Value']
          self.queueDict[queueName]['CEName'] = ce
          self.queueDict[queueName]['CEType'] = ceType
          self.queueDict[queueName]['Site'] = siteName
          self.queueDict[queueName]['QueueName'] = queue
          result = self.queueDict[queueName]['CE'].isValid()
          if not result['OK']:
            self.log.fatal( result['Message'] )
            return result
          if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
            self.queueDict[queueName]['BundleProxy'] = True

    return S_OK()

# 需要导入模块: from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory [as 别名]
# 或者: from DIRAC.Resources.Computing.ComputingElementFactory.ComputingElementFactory import getCE [as 别名]
    def getQueues(self, resourceDict):
        """ Get the list of relevant CEs and their descriptions
    """

        self.queueDict = {}
        ceFactory = ComputingElementFactory()

        for site in resourceDict:
            for ce in resourceDict[site]:
                ceDict = resourceDict[site][ce]
                qDict = ceDict.pop("Queues")
                for queue in qDict:
                    queueName = "%s_%s" % (ce, queue)
                    self.queueDict[queueName] = {}
                    self.queueDict[queueName]["ParametersDict"] = qDict[queue]
                    self.queueDict[queueName]["ParametersDict"]["Queue"] = queue
                    self.queueDict[queueName]["ParametersDict"]["Site"] = site
                    self.queueDict[queueName]["ParametersDict"]["GridEnv"] = self.gridEnv
                    self.queueDict[queueName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown")
                    # Evaluate the CPU limit of the queue according to the Glue convention
                    # To Do: should be a utility
                    if (
                        "maxCPUTime" in self.queueDict[queueName]["ParametersDict"]
                        and "SI00" in self.queueDict[queueName]["ParametersDict"]
                    ):
                        maxCPUTime = float(self.queueDict[queueName]["ParametersDict"]["maxCPUTime"])
                        # For some sites there are crazy values in the CS
                        maxCPUTime = max(maxCPUTime, 0)
                        maxCPUTime = min(maxCPUTime, 86400 * 12.5)
                        si00 = float(self.queueDict[queueName]["ParametersDict"]["SI00"])
                        queueCPUTime = 60.0 / 250.0 * maxCPUTime * si00
                        self.queueDict[queueName]["ParametersDict"]["CPUTime"] = int(queueCPUTime)
                    qwDir = os.path.join(self.workingDirectory, queue)
                    if not os.path.exists(qwDir):
                        os.makedirs(qwDir)
                    self.queueDict[queueName]["ParametersDict"]["WorkingDirectory"] = qwDir

                    platform = ""
                    if "Platform" in self.queueDict[queueName]["ParametersDict"]:
                        platform = self.queueDict[queueName]["ParametersDict"]["Platform"]
                    elif "Platform" in ceDict:
                        platform = ceDict["Platform"]
                    elif "OS" in ceDict:
                        architecture = ceDict.get("architecture", "x86_64")
                        OS = ceDict["OS"]
                        platform = "_".join([architecture, OS])
                    if platform and not platform in self.platforms:
                        self.platforms.append(platform)

                    if not "Platform" in self.queueDict[queueName]["ParametersDict"] and platform:
                        result = Resources.getDIRACPlatform(platform)
                        if result["OK"]:
                            self.queueDict[queueName]["ParametersDict"]["Platform"] = result["Value"]

                    ceQueueDict = dict(ceDict)
                    ceQueueDict.update(self.queueDict[queueName]["ParametersDict"])
                    result = ceFactory.getCE(ceName=ce, ceType=ceDict["CEType"], ceParametersDict=ceQueueDict)
                    if not result["OK"]:
                        return result
                    self.queueDict[queueName]["CE"] = result["Value"]
                    self.queueDict[queueName]["CEName"] = ce
                    self.queueDict[queueName]["CEType"] = ceDict["CEType"]
                    self.queueDict[queueName]["Site"] = site
                    self.queueDict[queueName]["QueueName"] = queue
                    result = self.queueDict[queueName]["CE"].isValid()
                    if not result["OK"]:
                        self.log.fatal(result["Message"])
                        return result
                    if "BundleProxy" in self.queueDict[queueName]["ParametersDict"]:
                        self.queueDict[queueName]["BundleProxy"] = True
                    elif "BundleProxy" in ceDict:
                        self.queueDict[queueName]["BundleProxy"] = True

                    if site not in self.sites:
                        self.sites.append(site)

        return S_OK()