Python CFG.setOption方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
  def execute( self ):
    """ execute
    
    """

    stopAgents = self.findStopAgents()[ 'Value' ]
    if stopAgents:
      self.log.info( 'Aborting, there are stop_agents to be picked' )
      return S_OK()

    pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' )
    if not pilotVersion:
      self.log.error( 'There is no pilot version on the CS' )
      return S_OK()

    pilotVersion = self.getNewestPilotVersion()
    if not pilotVersion[ 'OK' ]:
      self.log.error( pilotVersion[ 'Message' ] )
      return S_ERROR( pilotVersion[ 'Message' ] )
    pilotVersion = pilotVersion[ 'Value' ]
      
    localCFG = CFG()
    
    #load local CFG
    localCFG.loadFromFile( self.cfgToUpdate )
    releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ]
    
    self.log.info( 'PilotVersion : %s' % pilotVersion )
    self.log.info( 'ReleaseVersion : %s' % releaseVersion )
            
    if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ):
    
      self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) )
    
      localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion )
      localCFG.writeToFile( self.cfgToUpdate )  
      
      self.touchStopAgents()
    
    else:
      
      self.log.info( 'Nothing to do' )
    
    return S_OK()     

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
def checkFunction():
  """ gets CPU normalisation from MFJ or calculate itself """
  from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getPowerFromMJF
  from ILCDIRAC.Core.Utilities.CPUNormalization import getCPUNormalization
  from DIRAC import gLogger, gConfig

  result = getCPUNormalization()

  if not result['OK']:
    gLogger.error( result['Message'] )

  norm = round( result['Value']['NORM'], 1 )

  gLogger.notice( 'Estimated CPU power is %.1f %s' % ( norm, result['Value']['UNIT'] ) )

  mjfPower = getPowerFromMJF()
  if mjfPower:
    gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
  else:
    gLogger.notice( 'MJF not available on this node' )

  if update and not configFile:
    gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )

    gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
  if configFile:
    from DIRAC.Core.Utilities.CFG import CFG
    cfg = CFG()
    try:
      # Attempt to open the given file
      cfg.loadFromFile( configFile )
    except:
      pass
    # Create the section if it does not exist
    if not cfg.existsKey( 'LocalSite' ):
      cfg.createNewSection( 'LocalSite' )
    cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )

    cfg.writeToFile( configFile )


  DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
 def toCFG(self):
   """ Get the full description of the file in CFG format
   """
   oCFG = CFG()
   strippedLFN = self.lfn.replace('/','&&')
   oCFG.createNewSection(strippedLFN)
   oCFG.setOption('%s/Status' % (strippedLFN), self.status)    
   oCFG.setOption('%s/Size' % (strippedLFN), self.size)    
   oCFG.setOption('%s/GUID' % (strippedLFN), self.guid)    
   oCFG.setOption('%s/Checksum' % (strippedLFN), self.checksum)
   #TODO: still have to include the CFG from the replica objects 
   if self.catalogReplicas:
     oCFG.createNewSection('%s/CatalogReplicas' % strippedLFN)
     for replica in self.catalogReplicas:
       pass
       #  rCFG.mergeWith(CFG().loadFromBuffer(replica.toCFG()['Value']))
   return S_OK(str(oCFG))

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
class JobManifest(object):
    def __init__(self, manifest=""):
        self.__manifest = CFG()
        self.__dirty = False
        self.__ops = False
        if manifest:
            result = self.load(manifest)
            if not result["OK"]:
                raise Exception(result["Message"])

    def isDirty(self):
        return self.__dirty

    def setDirty(self):
        self.__dirty = True

    def clearDirty(self):
        self.__dirty = False

    def load(self, dataString):
        """
    Auto discover format type based on [ .. ] of JDL
    """
        dataString = dataString.strip()
        if dataString[0] == "[" and dataString[-1] == "]":
            return self.loadJDL(dataString)
        else:
            return self.loadCFG(dataString)

    def loadJDL(self, jdlString):
        """
    Load job manifest from JDL format
    """
        result = loadJDLAsCFG(jdlString.strip())
        if not result["OK"]:
            self.__manifest = CFG()
            return result
        self.__manifest = result["Value"][0]
        return S_OK()

    def loadCFG(self, cfgString):
        """
    Load job manifest from CFG format
    """
        try:
            self.__manifest.loadFromBuffer(cfgString)
        except Exception as e:
            return S_ERROR("Can't load manifest from cfg: %s" % str(e))
        return S_OK()

    def dumpAsCFG(self):
        return str(self.__manifest)

    def getAsCFG(self):
        return self.__manifest.clone()

    def dumpAsJDL(self):
        return dumpCFGAsJDL(self.__manifest)

    def __getCSValue(self, varName, defaultVal=None):
        if not self.__ops:
            self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
        if varName[0] != "/":
            varName = "JobDescription/%s" % varName
        return self.__ops.getValue(varName, defaultVal)

    def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
        """
    Check a numerical var
    """
        initialVal = False
        if varName not in self.__manifest:
            varValue = self.__getCSValue("Default%s" % varName, defaultVal)
        else:
            varValue = self.__manifest[varName]
            initialVal = varValue
        try:
            varValue = long(varValue)
        except:
            return S_ERROR("%s must be a number" % varName)
        minVal = self.__getCSValue("Min%s" % varName, minVal)
        maxVal = self.__getCSValue("Max%s" % varName, maxVal)
        varValue = max(minVal, min(varValue, maxVal))
        if initialVal != varValue:
            self.__manifest.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkChoiceVar(self, varName, defaultVal, choices):
        """
    Check a choice var
    """
        initialVal = False
        if varName not in self.__manifest:
            varValue = self.__getCSValue("Default%s" % varName, defaultVal)
        else:
            varValue = self.__manifest[varName]
            initialVal = varValue
        if varValue not in self.__getCSValue("Choices%s" % varName, choices):
            return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
        if initialVal != varValue:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
  mjfPower = getPowerFromMJF()
  if mjfPower:
    gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
  else:
    gLogger.notice( 'MJF not available on this node' )

  if update and not configFile:
    gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )

    gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
  if configFile:
    from DIRAC.Core.Utilities.CFG import CFG
    cfg = CFG()
    try:
      # Attempt to open the given file
      cfg.loadFromFile( configFile )
    except:
      pass
    # Create the section if it does not exist
    if not cfg.existsKey( 'LocalSite' ):
      cfg.createNewSection( 'LocalSite' )
    cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )

    cfg.writeToFile( configFile )


  DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
  def execute( self ):
    """The JobAgent execution method.
    """
    if self.jobCount:
      #Only call timeLeft utility after a job has been picked up
      self.log.info( 'Attempting to check CPU time left for filling mode' )
      if self.fillingMode:
        if self.timeLeftError:
          self.log.warn( self.timeLeftError )
          return self.__finish( self.timeLeftError )
        self.log.info( '%s normalized CPU units remaining in slot' % ( self.timeLeft ) )
        # Need to update the Configuration so that the new value is published in the next matching request
        result = self.computingElement.setCPUTimeLeft( cpuTimeLeft = self.timeLeft )
        if not result['OK']:
          return self.__finish( result['Message'] )
        
        # Update local configuration to be used by submitted job wrappers
        localCfg = CFG()
        if self.extraOptions:
          localConfigFile = os.path.join( '.', self.extraOptions )
        else:
          localConfigFile = os.path.join( rootPath, "etc", "dirac.cfg" )
        localCfg.loadFromFile( localConfigFile )
        if not localCfg.isSection('/LocalSite'):
          localCfg.createNewSection('/LocalSite')
        localCfg.setOption( '/LocalSite/CPUTimeLeft', self.timeLeft )
        localCfg.writeToFile( localConfigFile )
        
      else:
        return self.__finish( 'Filling Mode is Disabled' )

    self.log.verbose( 'Job Agent execution loop' )
    available = self.computingElement.available()
    if not available['OK'] or not available['Value']:
      self.log.info( 'Resource is not available' )
      self.log.info( available['Message'] )
      return self.__finish( 'CE Not Available' )

    self.log.info( available['Message'] )

    result = self.computingElement.getDescription()
    if not result['OK']:
      return result
    ceDict = result['Value']

    # Add pilot information
    gridCE = gConfig.getValue( 'LocalSite/GridCE', 'Unknown' )
    if gridCE != 'Unknown':
      ceDict['GridCE'] = gridCE
    if not 'PilotReference' in ceDict:
      ceDict['PilotReference'] = str( self.pilotReference )
    ceDict['PilotBenchmark'] = self.cpuFactor
    ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag

    # Add possible job requirements
    result = gConfig.getOptionsDict( '/AgentJobRequirements' )
    if result['OK']:
      requirementsDict = result['Value']
      ceDict.update( requirementsDict )

    self.log.verbose( ceDict )
    start = time.time()
    jobRequest = self.__requestJob( ceDict )
    matchTime = time.time() - start
    self.log.info( 'MatcherTime = %.2f (s)' % ( matchTime ) )

    self.stopAfterFailedMatches = self.am_getOption( 'StopAfterFailedMatches', self.stopAfterFailedMatches )

    if not jobRequest['OK']:
      if re.search( 'No match found', jobRequest['Message'] ):
        self.log.notice( 'Job request OK: %s' % ( jobRequest['Message'] ) )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )
      elif jobRequest['Message'].find( "seconds timeout" ) != -1:
        self.log.error( jobRequest['Message'] )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )
      elif jobRequest['Message'].find( "Pilot version does not match" ) != -1 :
        self.log.error( jobRequest['Message'] )
        return S_ERROR( jobRequest['Message'] )
      else:
        self.log.notice( 'Failed to get jobs: %s' % ( jobRequest['Message'] ) )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )

    # Reset the Counter
    self.matchFailedCount = 0

    matcherInfo = jobRequest['Value']
    jobID = matcherInfo['JobID']
    if not self.pilotInfoReportedFlag:
      # Check the flag after the first access to the Matcher
      self.pilotInfoReportedFlag = matcherInfo.get( 'PilotInfoReportedFlag', False )
    matcherParams = ['JDL', 'DN', 'Group']
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
else:
  print "WORKSPACE: %s" % os.path.expandvars('$WORKSPACE')
  if os.path.isfile( os.path.expandvars('$WORKSPACE')+'/PilotInstallDIR/etc/dirac.cfg' ):
    localConfigFile = os.path.expandvars('$WORKSPACE')+'/PilotInstallDIR/etc/dirac.cfg'
  elif os.path.isfile( os.path.expandvars('$WORKSPACE')+'/ServerInstallDIR/etc/dirac.cfg' ):
    localConfigFile = os.path.expandvars('$WORKSPACE')+'/ServerInstallDIR/etc/dirac.cfg'
  elif os.path.isfile( './etc/dirac.cfg' ):
    localConfigFile = './etc/dirac.cfg'
  else:
    print "Local CFG file not found"
    exit( 2 )

localCfg.loadFromFile( localConfigFile )
if not localCfg.isSection( '/LocalSite' ):
  localCfg.createNewSection( '/LocalSite' )
localCfg.setOption( '/LocalSite/CPUTimeLeft', 5000 )
localCfg.setOption( '/DIRAC/Security/UseServerCertificate', False )

if not sMod:
  if not setup:
    setup = gConfig.getValue('/DIRAC/Setup')
    if not setup:
      setup = 'JenkinsSetup'
  if not vo:
    vo = gConfig.getValue('/DIRAC/VirtualOrganization')
    if not vo:
      vo = 'dirac'

  if not localCfg.isSection( '/DIRAC/VOPolicy' ):
    localCfg.createNewSection( '/DIRAC/VOPolicy' )
  if not localCfg.isSection( '/DIRAC/VOPolicy/%s' % vo ):

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]

#.........这里部分代码省略.........
    written = self._writeRepository( destination )
    if not written:
      return S_ERROR( "Failed to write repository" )
    return S_OK( destination )

  def resetRepository( self, jobIDs = [] ):
    if not jobIDs:
      jobs = self.readRepository()['Value']
      jobIDs = jobs.keys()
    paramDict = {'State'       : 'Submitted',
                 'Retrieved'   : 0,
                 'OutputData'  : 0}
    for jobID in jobIDs:
      self._writeJob( jobID, paramDict, True )
    self._writeRepository( self.location )
    return S_OK()

  def _writeRepository( self, path ):
    handle, tmpName = tempfile.mkstemp()
    written = self.repo.writeToFile( tmpName )
    os.close( handle )
    if not written:
      if os.path.exists( tmpName ):
        os.remove( tmpName )
      return written
    if os.path.exists( path ):
      gLogger.debug( "Replacing %s" % path )
    try:
      shutil.move( tmpName, path )
      return True
    except Exception as x:
      gLogger.error( "Failed to overwrite repository.", x )
      gLogger.info( "If your repository is corrupted a backup can be found %s" % tmpName )
      return False

  def appendToRepository( self, repoLocation ):
    if not os.path.exists( repoLocation ):
      gLogger.error( "Secondary repository does not exist", repoLocation )
      return S_ERROR( "Secondary repository does not exist" )
    self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo )
    self._writeRepository( self.location )
    return S_OK()

  def addJob( self, jobID, state = 'Submitted', retrieved = 0, outputData = 0, update = False ):
    paramDict = { 'State'       : state,
                  'Time'        : self._getTime(),
                  'Retrieved'   : int( retrieved ),
                  'OutputData'  : outputData}
    self._writeJob( jobID, paramDict, update )
    self._writeRepository( self.location )
    return S_OK( jobID )

  def updateJob( self, jobID, paramDict ):
    if self._existsJob( jobID ):
      paramDict['Time'] = self._getTime()
      self._writeJob( jobID, paramDict, True )
      self._writeRepository( self.location )
    return S_OK()

  def updateJobs( self, jobDict ):
    for jobID, paramDict in jobDict.items():
      if self._existsJob( jobID ):
        paramDict['Time'] = self._getTime()
        self._writeJob( jobID, paramDict, True )
    self._writeRepository( self.location )
    return S_OK()

  def _getTime( self ):
    runtime = time.ctime()
    return runtime.replace( " ", "_" )

  def _writeJob( self, jobID, paramDict, update ):
    jobID = str( jobID )
    jobExists = self._existsJob( jobID )
    if jobExists and ( not update ):
      gLogger.warn( "Job exists and not overwriting" )
      return S_ERROR( "Job exists and not overwriting" )
    if not jobExists:
      self.repo.createNewSection( 'Jobs/%s' % jobID )
    for key, value in paramDict.items():
      self.repo.setOption( 'Jobs/%s/%s' % ( jobID, key ), value )
    return S_OK()

  def removeJob( self, jobID ):
    res = self.repo['Jobs'].deleteKey( str( jobID ) ) #pylint: disable=no-member
    if res:
      self._writeRepository( self.location )
    return S_OK()

  def existsJob( self, jobID ):
    return S_OK( self._existsJob( jobID ) )

  def _existsJob( self, jobID ):
    return self.repo.isSection( 'Jobs/%s' % jobID )

  def getLocation( self ):
    return S_OK( self.location )

  def getSize( self ):
    return S_OK( len( self.repo.getAsDict( 'Jobs' ) ) )

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
  if not result['OK']:
    DIRAC.gLogger.error( result['Message'] )

  norm = int( ( result['Value']['NORM'] + 0.05 ) * 10 ) / 10.

  DIRAC.gLogger.notice( 'Normalization for current CPU is %.1f %s' % ( norm, result['Value']['UNIT'] ) )

  if update and not configFile:
    DIRAC.gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', norm )
    DIRAC.gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )

    DIRAC.gConfig.dumpLocalCFGToFile( DIRAC.gConfig.diracConfigFilePath )
  if configFile:
    from DIRAC.Core.Utilities.CFG import CFG
    cfg = CFG()
    try:
      # Attempt to open the given file
      cfg.loadFromFile( configFile )
    except:
      pass
    # Create the section if it does not exist
    if not cfg.existsKey( 'LocalSite' ):
      cfg.createNewSection( 'LocalSite' )
    cfg.setOption( '/LocalSite/CPUScalingFactor', norm )
    cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )

    cfg.writeToFile( configFile )


  DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
 def toCFG(self):
   oCFG = CFG()
   oCFG.createNewSection(self.se)
   oCFG.setOption('%s/Status' % (self.se), self.status)
   oCFG.setOption('%s/PFN' % (self.se), self.pfn)
   return S_OK(str(oCFG))

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
  def execute(self):
    """The JobAgent execution method.
    """
    if self.jobCount:
      # Temporary mechanism to pass a shutdown message to the agent
      if os.path.exists('/var/lib/dirac_drain'):
        return self.__finish('Node is being drained by an operator')
      # Only call timeLeft utility after a job has been picked up
      self.log.info('Attempting to check CPU time left for filling mode')
      if self.fillingMode:
        if self.timeLeftError:
          self.log.warn(self.timeLeftError)
          return self.__finish(self.timeLeftError)
        self.log.info('%s normalized CPU units remaining in slot' % (self.timeLeft))
        if self.timeLeft <= self.minimumTimeLeft:
          return self.__finish('No more time left')
        # Need to update the Configuration so that the new value is published in the next matching request
        result = self.computingElement.setCPUTimeLeft(cpuTimeLeft=self.timeLeft)
        if not result['OK']:
          return self.__finish(result['Message'])

        # Update local configuration to be used by submitted job wrappers
        localCfg = CFG()
        if self.extraOptions:
          localConfigFile = os.path.join('.', self.extraOptions)
        else:
          localConfigFile = os.path.join(rootPath, "etc", "dirac.cfg")
        localCfg.loadFromFile(localConfigFile)
        if not localCfg.isSection('/LocalSite'):
          localCfg.createNewSection('/LocalSite')
        localCfg.setOption('/LocalSite/CPUTimeLeft', self.timeLeft)
        localCfg.writeToFile(localConfigFile)

      else:
        return self.__finish('Filling Mode is Disabled')

    self.log.verbose('Job Agent execution loop')
    result = self.computingElement.available()
    if not result['OK']:
      self.log.info('Resource is not available')
      self.log.info(result['Message'])
      return self.__finish('CE Not Available')

    self.log.info(result['Message'])

    ceInfoDict = result['CEInfoDict']
    runningJobs = ceInfoDict.get("RunningJobs")
    availableSlots = result['Value']

    if not availableSlots:
      if runningJobs:
        self.log.info('No available slots with %d running jobs' % runningJobs)
        return S_OK('Job Agent cycle complete with %d running jobs' % runningJobs)
      else:
        self.log.info('CE is not available')
        return self.__finish('CE Not Available')

    result = self.computingElement.getDescription()
    if not result['OK']:
      return result
    ceDict = result['Value']
    # Add pilot information
    gridCE = gConfig.getValue('LocalSite/GridCE', 'Unknown')
    if gridCE != 'Unknown':
      ceDict['GridCE'] = gridCE
    if 'PilotReference' not in ceDict:
      ceDict['PilotReference'] = str(self.pilotReference)
    ceDict['PilotBenchmark'] = self.cpuFactor
    ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag

    # Add possible job requirements
    result = gConfig.getOptionsDict('/AgentJobRequirements')
    if result['OK']:
      requirementsDict = result['Value']
      ceDict.update(requirementsDict)
      self.log.info('Requirements:', requirementsDict)

    self.log.verbose(ceDict)
    start = time.time()
    jobRequest = MatcherClient().requestJob(ceDict)
    matchTime = time.time() - start
    self.log.info('MatcherTime = %.2f (s)' % (matchTime))

    self.stopAfterFailedMatches = self.am_getOption('StopAfterFailedMatches', self.stopAfterFailedMatches)

    if not jobRequest['OK']:
      if re.search('No match found', jobRequest['Message']):
        self.log.notice('Job request OK: %s' % (jobRequest['Message']))
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish('Nothing to do for more than %d cycles' % self.stopAfterFailedMatches)
        return S_OK(jobRequest['Message'])
      elif jobRequest['Message'].find("seconds timeout") != -1:
        self.log.error('Timeout while requesting job', jobRequest['Message'])
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish('Nothing to do for more than %d cycles' % self.stopAfterFailedMatches)
        return S_OK(jobRequest['Message'])
      elif jobRequest['Message'].find("Pilot version does not match") != -1:
        errorMsg = 'Pilot version does not match the production version'
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
class JobDescription( object ):

  def __init__( self ):
    self.__description = CFG()
    self.__dirty = False

  def isDirty( self ):
    return self.__dirty

  def loadDescription( self, dataString ):
    """
    Auto discover format type based on [ .. ] of JDL
    """
    dataString = dataString.strip()
    if dataString[0] == "[" and dataString[-1] == "]":
      return self.loadDescriptionFromJDL( dataString )
    else:
      return self.loadDescriptionFromCFG( dataString )

  def loadDescriptionFromJDL( self, jdlString ):
    """
    Load job description from JDL format
    """
    result = loadJDLAsCFG( jdlString.strip() )
    if not result[ 'OK' ]:
      self.__description = CFG()
      return result
    self.__description = result[ 'Value' ][0]
    return S_OK()

  def loadDescriptionFromCFG( self, cfgString ):
    """
    Load job description from CFG format
    """
    try:
      self.__description.loadFromBuffer( cfgString )
    except Exception as e:
      return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
    return S_OK()

  def dumpDescriptionAsCFG( self ):
    return str( self.__description )

  def dumpDescriptionAsJDL( self ):
    return dumpCFGAsJDL( self.__description )

  def __checkNumericalVarInDescription( self, varName, defaultVal, minVal, maxVal ):
    """
    Check a numerical var
    """
    initialVal = 0
    if varName not in self.__description:
      varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
    else:
      varValue = self.__description[ varName ]
      initialVal = varValue
    try:
      varValue = long( varValue )
    except:
      return S_ERROR( "%s must be a number" % varName )
    minVal = Operations().getValue( "JobDescription/Min%s" % varName, minVal )
    maxVal = Operations().getValue( "JobDescription/Max%s" % varName, maxVal )
    varValue = max( minVal, min( varValue, maxVal ) )
    if initialVal != varValue:
      self.__description.setOption( varName, varValue )
    return S_OK( varValue )

  def __checkChoiceVarInDescription( self, varName, defaultVal, choices ):
    """
    Check a choice var
    """
    initialVal = False
    if varName not in self.__description:
      varValue = Operations().getValue( "JobDescription/Default%s" % varName , defaultVal )
    else:
      varValue = self.__description[ varName ]
      initialVal = varValue
    if varValue not in Operations().getValue( "JobDescription/Choices%s" % varName , choices ):
      return S_ERROR( "%s is not a valid value for %s" % ( varValue, varName ) )
    if initialVal != varValue:
      self.__description.setOption( varName, varValue )
    return S_OK( varValue )

  def __checkMultiChoiceInDescription( self, varName, choices ):
    """
    Check a multi choice var
    """
    initialVal = False
    if varName not in self.__description:
      return S_OK()
    else:
      varValue = self.__description[ varName ]
      initialVal = varValue
    choices = Operations().getValue( "JobDescription/Choices%s" % varName , choices )
    for v in List.fromChar( varValue ):
      if v not in choices:
        return S_ERROR( "%s is not a valid value for %s" % ( v, varName ) )
    if initialVal != varValue:
      self.__description.setOption( varName, varValue )
    return S_OK( varValue )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
    result = getCPUNormalization()

    if not result["OK"]:
        DIRAC.gLogger.error(result["Message"])

    norm = int((result["Value"]["NORM"] + 0.05) * 10) / 10.0

    DIRAC.gLogger.notice("Normalization for current CPU is %.1f %s" % (norm, result["Value"]["UNIT"]))

    if update:
        DIRAC.gConfig.setOptionValue("/LocalSite/CPUNormalizationFactor", norm)
        DIRAC.gConfig.dumpLocalCFGToFile(DIRAC.gConfig.diracConfigFilePath)
    if configFile:
        from DIRAC.Core.Utilities.CFG import CFG

        cfg = CFG()
        try:
            # Attempt to open the given file
            cfg.loadFromFile(configFile)
        except:
            pass
        # Create the section if it does not exist
        if not cfg.existsKey("LocalSite"):
            cfg.createNewSection("LocalSite")
        cfg.setOption("/LocalSite/CPUNormalizationFactor", norm)

        cfg.writeToFile(configFile)

    DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
    def execute(self):
        """The JobAgent execution method.
    """
        if self.jobCount:
            # Only call timeLeft utility after a job has been picked up
            self.log.info("Attempting to check CPU time left for filling mode")
            if self.fillingMode:
                if self.timeLeftError:
                    self.log.warn(self.timeLeftError)
                    return self.__finish(self.timeLeftError)
                self.log.info("%s normalized CPU units remaining in slot" % (self.timeLeft))
                if self.timeLeft <= self.minimumTimeLeft:
                    return self.__finish("No more time left")
                # Need to update the Configuration so that the new value is published in the next matching request
                result = self.computingElement.setCPUTimeLeft(cpuTimeLeft=self.timeLeft)
                if not result["OK"]:
                    return self.__finish(result["Message"])

                # Update local configuration to be used by submitted job wrappers
                localCfg = CFG()
                if self.extraOptions:
                    localConfigFile = os.path.join(".", self.extraOptions)
                else:
                    localConfigFile = os.path.join(rootPath, "etc", "dirac.cfg")
                localCfg.loadFromFile(localConfigFile)
                if not localCfg.isSection("/LocalSite"):
                    localCfg.createNewSection("/LocalSite")
                localCfg.setOption("/LocalSite/CPUTimeLeft", self.timeLeft)
                localCfg.writeToFile(localConfigFile)

            else:
                return self.__finish("Filling Mode is Disabled")

        self.log.verbose("Job Agent execution loop")
        available = self.computingElement.available()
        if not available["OK"] or not available["Value"]:
            self.log.info("Resource is not available")
            self.log.info(available["Message"])
            return self.__finish("CE Not Available")

        self.log.info(available["Message"])

        result = self.computingElement.getDescription()
        if not result["OK"]:
            return result
        ceDict = result["Value"]

        # Add pilot information
        gridCE = gConfig.getValue("LocalSite/GridCE", "Unknown")
        if gridCE != "Unknown":
            ceDict["GridCE"] = gridCE
        if not "PilotReference" in ceDict:
            ceDict["PilotReference"] = str(self.pilotReference)
        ceDict["PilotBenchmark"] = self.cpuFactor
        ceDict["PilotInfoReportedFlag"] = self.pilotInfoReportedFlag

        # Add possible job requirements
        result = gConfig.getOptionsDict("/AgentJobRequirements")
        if result["OK"]:
            requirementsDict = result["Value"]
            ceDict.update(requirementsDict)

        self.log.verbose(ceDict)
        start = time.time()
        jobRequest = self.__requestJob(ceDict)
        matchTime = time.time() - start
        self.log.info("MatcherTime = %.2f (s)" % (matchTime))

        self.stopAfterFailedMatches = self.am_getOption("StopAfterFailedMatches", self.stopAfterFailedMatches)

        if not jobRequest["OK"]:
            if re.search("No match found", jobRequest["Message"]):
                self.log.notice("Job request OK: %s" % (jobRequest["Message"]))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])
            elif jobRequest["Message"].find("seconds timeout") != -1:
                self.log.error("Timeout while requesting job", jobRequest["Message"])
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])
            elif jobRequest["Message"].find("Pilot version does not match") != -1:
                errorMsg = "Pilot version does not match the production version"
                self.log.error(errorMsg, jobRequest["Message"].replace(errorMsg, ""))
                return S_ERROR(jobRequest["Message"])
            else:
                self.log.notice("Failed to get jobs: %s" % (jobRequest["Message"]))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])

        # Reset the Counter
        self.matchFailedCount = 0

        matcherInfo = jobRequest["Value"]
        if not self.pilotInfoReportedFlag:
            # Check the flag after the first access to the Matcher
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import setOption [as 别名]
setupName = args[0]

import os.path

from DIRAC.Core.Utilities.CFG import CFG

localCfg = CFG()

# Which file
localConfigFile = os.path.join( '.', 'etc', 'Production.cfg' )
localCfg.loadFromFile( localConfigFile )

# Where to store outputs
if not os.path.isdir( '%s/sandboxes' % setupName ):
  os.makedirs( '%s/sandboxes' % setupName )
localCfg.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/BasePath',
                    '%s/sandboxes' % setupName )
localCfg.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/LogLevel', 'DEBUG' )

# Now setting a SandboxSE as the following:
#     ProductionSandboxSE
#     {
#       BackendType = DISET
#       AccessProtocol.1
#       {
#         Host = localhost
#         Port = 9196
#         ProtocolName = DIP
#         Protocol = dips
#         Path = /scratch/workspace/%s/sandboxes % setupName
#         Access = remote
#         SpaceToken =