Python CFG.writeToFile方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
class ProcessList(object):
  """ The ProcessList uses internally the CFG utility to store the processes and their properties.
  """
  def __init__(self, location):
    self.cfg = CFG()
    self.location = location
    self.goodProcessList = True
    if os.path.exists(self.location):
      self.cfg.loadFromFile(self.location)
      if not self.cfg.existsKey('Processes'):
        self.cfg.createNewSection('Processes')
    else:
      self.goodProcessList = False  
      
  def _writeProcessList(self, path):
    """ Write to text
    """
    handle, tmpName = tempfile.mkstemp()
    written = self.cfg.writeToFile(tmpName)
    os.close(handle)
    if not written:
      if os.path.exists(tmpName):
        os.remove(tmpName)
      return written
    if os.path.exists(path):
      LOG.debug("Replacing %s" % path)
    try:
      shutil.move(tmpName, path)
      return True
    except OSError, err:
      LOG.error("Failed to overwrite process list.", err)
      LOG.info("If your process list is corrupted a backup can be found %s" % tmpName)
      return False

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
def checkFunction():
  """ gets CPU normalisation from MFJ or calculate itself """
  from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getPowerFromMJF
  from ILCDIRAC.Core.Utilities.CPUNormalization import getCPUNormalization
  from DIRAC import gLogger, gConfig

  result = getCPUNormalization()

  if not result['OK']:
    gLogger.error( result['Message'] )

  norm = round( result['Value']['NORM'], 1 )

  gLogger.notice( 'Estimated CPU power is %.1f %s' % ( norm, result['Value']['UNIT'] ) )

  mjfPower = getPowerFromMJF()
  if mjfPower:
    gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
  else:
    gLogger.notice( 'MJF not available on this node' )

  if update and not configFile:
    gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )

    gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
  if configFile:
    from DIRAC.Core.Utilities.CFG import CFG
    cfg = CFG()
    try:
      # Attempt to open the given file
      cfg.loadFromFile( configFile )
    except:
      pass
    # Create the section if it does not exist
    if not cfg.existsKey( 'LocalSite' ):
      cfg.createNewSection( 'LocalSite' )
    cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )

    cfg.writeToFile( configFile )


  DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
  def execute( self ):
    """ execute
    
    """

    stopAgents = self.findStopAgents()[ 'Value' ]
    if stopAgents:
      self.log.info( 'Aborting, there are stop_agents to be picked' )
      return S_OK()

    pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' )
    if not pilotVersion:
      self.log.error( 'There is no pilot version on the CS' )
      return S_OK()

    pilotVersion = self.getNewestPilotVersion()
    if not pilotVersion[ 'OK' ]:
      self.log.error( pilotVersion[ 'Message' ] )
      return S_ERROR( pilotVersion[ 'Message' ] )
    pilotVersion = pilotVersion[ 'Value' ]
      
    localCFG = CFG()
    
    #load local CFG
    localCFG.loadFromFile( self.cfgToUpdate )
    releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ]
    
    self.log.info( 'PilotVersion : %s' % pilotVersion )
    self.log.info( 'ReleaseVersion : %s' % releaseVersion )
            
    if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ):
    
      self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) )
    
      localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion )
      localCFG.writeToFile( self.cfgToUpdate )  
      
      self.touchStopAgents()
    
    else:
      
      self.log.info( 'Nothing to do' )
    
    return S_OK()     

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
  mjfPower = getPowerFromMJF()
  if mjfPower:
    gLogger.notice( 'CPU power from MJF is %.1f HS06' % mjfPower )
  else:
    gLogger.notice( 'MJF not available on this node' )

  if update and not configFile:
    gConfig.setOptionValue( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    gConfig.setOptionValue( '/LocalSite/CPUNormalizationFactor', norm )

    gConfig.dumpLocalCFGToFile( gConfig.diracConfigFilePath )
  if configFile:
    from DIRAC.Core.Utilities.CFG import CFG
    cfg = CFG()
    try:
      # Attempt to open the given file
      cfg.loadFromFile( configFile )
    except:
      pass
    # Create the section if it does not exist
    if not cfg.existsKey( 'LocalSite' ):
      cfg.createNewSection( 'LocalSite' )
    cfg.setOption( '/LocalSite/CPUScalingFactor', mjfPower if mjfPower else norm )
    cfg.setOption( '/LocalSite/CPUNormalizationFactor', norm )

    cfg.writeToFile( configFile )


  DIRAC.exit()

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
  def execute( self ):
    """The JobAgent execution method.
    """
    if self.jobCount:
      #Only call timeLeft utility after a job has been picked up
      self.log.info( 'Attempting to check CPU time left for filling mode' )
      if self.fillingMode:
        if self.timeLeftError:
          self.log.warn( self.timeLeftError )
          return self.__finish( self.timeLeftError )
        self.log.info( '%s normalized CPU units remaining in slot' % ( self.timeLeft ) )
        # Need to update the Configuration so that the new value is published in the next matching request
        result = self.computingElement.setCPUTimeLeft( cpuTimeLeft = self.timeLeft )
        if not result['OK']:
          return self.__finish( result['Message'] )
        
        # Update local configuration to be used by submitted job wrappers
        localCfg = CFG()
        if self.extraOptions:
          localConfigFile = os.path.join( '.', self.extraOptions )
        else:
          localConfigFile = os.path.join( rootPath, "etc", "dirac.cfg" )
        localCfg.loadFromFile( localConfigFile )
        if not localCfg.isSection('/LocalSite'):
          localCfg.createNewSection('/LocalSite')
        localCfg.setOption( '/LocalSite/CPUTimeLeft', self.timeLeft )
        localCfg.writeToFile( localConfigFile )
        
      else:
        return self.__finish( 'Filling Mode is Disabled' )

    self.log.verbose( 'Job Agent execution loop' )
    available = self.computingElement.available()
    if not available['OK'] or not available['Value']:
      self.log.info( 'Resource is not available' )
      self.log.info( available['Message'] )
      return self.__finish( 'CE Not Available' )

    self.log.info( available['Message'] )

    result = self.computingElement.getDescription()
    if not result['OK']:
      return result
    ceDict = result['Value']

    # Add pilot information
    gridCE = gConfig.getValue( 'LocalSite/GridCE', 'Unknown' )
    if gridCE != 'Unknown':
      ceDict['GridCE'] = gridCE
    if not 'PilotReference' in ceDict:
      ceDict['PilotReference'] = str( self.pilotReference )
    ceDict['PilotBenchmark'] = self.cpuFactor
    ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag

    # Add possible job requirements
    result = gConfig.getOptionsDict( '/AgentJobRequirements' )
    if result['OK']:
      requirementsDict = result['Value']
      ceDict.update( requirementsDict )

    self.log.verbose( ceDict )
    start = time.time()
    jobRequest = self.__requestJob( ceDict )
    matchTime = time.time() - start
    self.log.info( 'MatcherTime = %.2f (s)' % ( matchTime ) )

    self.stopAfterFailedMatches = self.am_getOption( 'StopAfterFailedMatches', self.stopAfterFailedMatches )

    if not jobRequest['OK']:
      if re.search( 'No match found', jobRequest['Message'] ):
        self.log.notice( 'Job request OK: %s' % ( jobRequest['Message'] ) )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )
      elif jobRequest['Message'].find( "seconds timeout" ) != -1:
        self.log.error( jobRequest['Message'] )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )
      elif jobRequest['Message'].find( "Pilot version does not match" ) != -1 :
        self.log.error( jobRequest['Message'] )
        return S_ERROR( jobRequest['Message'] )
      else:
        self.log.notice( 'Failed to get jobs: %s' % ( jobRequest['Message'] ) )
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish( 'Nothing to do for more than %d cycles' % self.stopAfterFailedMatches )
        return S_OK( jobRequest['Message'] )

    # Reset the Counter
    self.matchFailedCount = 0

    matcherInfo = jobRequest['Value']
    jobID = matcherInfo['JobID']
    if not self.pilotInfoReportedFlag:
      # Check the flag after the first access to the Matcher
      self.pilotInfoReportedFlag = matcherInfo.get( 'PilotInfoReportedFlag', False )
    matcherParams = ['JDL', 'DN', 'Group']
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
    localConfigFile = './etc/dirac.cfg'
  else:
    print "Local CFG file not found"
    exit( 2 )

localCfg.loadFromFile( localConfigFile )
if not localCfg.isSection( '/LocalSite' ):
  localCfg.createNewSection( '/LocalSite' )
localCfg.setOption( '/LocalSite/CPUTimeLeft', 5000 )
localCfg.setOption( '/DIRAC/Security/UseServerCertificate', False )

if not sMod:
  if not setup:
    setup = gConfig.getValue('/DIRAC/Setup')
    if not setup:
      setup = 'JenkinsSetup'
  if not vo:
    vo = gConfig.getValue('/DIRAC/VirtualOrganization')
    if not vo:
      vo = 'dirac'

  if not localCfg.isSection( '/DIRAC/VOPolicy' ):
    localCfg.createNewSection( '/DIRAC/VOPolicy' )
  if not localCfg.isSection( '/DIRAC/VOPolicy/%s' % vo ):
    localCfg.createNewSection( '/DIRAC/VOPolicy/%s' % vo )
  if not localCfg.isSection( '/DIRAC/VOPolicy/%s/%s' % ( vo, setup ) ):
    localCfg.createNewSection( '/DIRAC/VOPolicy/%s/%s' % ( vo, setup ) )
  localCfg.setOption( '/DIRAC/VOPolicy/%s/%s/SoftwareDistModule' % ( vo, setup ), '' )

localCfg.writeToFile( localConfigFile )

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
class JobRepository( object ):

  def __init__( self, repository = None ):
    self.location = repository
    if not self.location:
      if "HOME" in os.environ:
        self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
      else:
        self.location = '%s/.dirac.repo.rep' % os.getcwd()
    self.repo = CFG()
    if os.path.exists( self.location ):
      self.repo.loadFromFile( self.location )
      if not self.repo.existsKey( 'Jobs' ):
        self.repo.createNewSection( 'Jobs' )
    else:
      self.repo.createNewSection( 'Jobs' )
    self.OK = True
    written = self._writeRepository( self.location )
    if not written:
      self.OK = False

  def isOK( self ):
    return self.OK

  def readRepository( self ):
    return S_OK( self.repo.getAsDict( 'Jobs' ) )

  def writeRepository( self, alternativePath = None ):
    destination = self.location
    if alternativePath:
      destination = alternativePath
    written = self._writeRepository( destination )
    if not written:
      return S_ERROR( "Failed to write repository" )
    return S_OK( destination )

  def resetRepository( self, jobIDs = [] ):
    if not jobIDs:
      jobs = self.readRepository()['Value']
      jobIDs = jobs.keys()
    paramDict = {'State'       : 'Submitted',
                 'Retrieved'   : 0,
                 'OutputData'  : 0}
    for jobID in jobIDs:
      self._writeJob( jobID, paramDict, True )
    self._writeRepository( self.location )
    return S_OK()

  def _writeRepository( self, path ):
    handle, tmpName = tempfile.mkstemp()
    written = self.repo.writeToFile( tmpName )
    os.close( handle )
    if not written:
      if os.path.exists( tmpName ):
        os.remove( tmpName )
      return written
    if os.path.exists( path ):
      gLogger.debug( "Replacing %s" % path )
    try:
      shutil.move( tmpName, path )
      return True
    except Exception as x:
      gLogger.error( "Failed to overwrite repository.", x )
      gLogger.info( "If your repository is corrupted a backup can be found %s" % tmpName )
      return False

  def appendToRepository( self, repoLocation ):
    if not os.path.exists( repoLocation ):
      gLogger.error( "Secondary repository does not exist", repoLocation )
      return S_ERROR( "Secondary repository does not exist" )
    self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo )
    self._writeRepository( self.location )
    return S_OK()

  def addJob( self, jobID, state = 'Submitted', retrieved = 0, outputData = 0, update = False ):
    paramDict = { 'State'       : state,
                  'Time'        : self._getTime(),
                  'Retrieved'   : int( retrieved ),
                  'OutputData'  : outputData}
    self._writeJob( jobID, paramDict, update )
    self._writeRepository( self.location )
    return S_OK( jobID )

  def updateJob( self, jobID, paramDict ):
    if self._existsJob( jobID ):
      paramDict['Time'] = self._getTime()
      self._writeJob( jobID, paramDict, True )
      self._writeRepository( self.location )
    return S_OK()

  def updateJobs( self, jobDict ):
    for jobID, paramDict in jobDict.items():
      if self._existsJob( jobID ):
        paramDict['Time'] = self._getTime()
        self._writeJob( jobID, paramDict, True )
    self._writeRepository( self.location )
    return S_OK()

  def _getTime( self ):
    runtime = time.ctime()
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
  def execute(self):
    """The JobAgent execution method.
    """
    if self.jobCount:
      # Temporary mechanism to pass a shutdown message to the agent
      if os.path.exists('/var/lib/dirac_drain'):
        return self.__finish('Node is being drained by an operator')
      # Only call timeLeft utility after a job has been picked up
      self.log.info('Attempting to check CPU time left for filling mode')
      if self.fillingMode:
        if self.timeLeftError:
          self.log.warn(self.timeLeftError)
          return self.__finish(self.timeLeftError)
        self.log.info('%s normalized CPU units remaining in slot' % (self.timeLeft))
        if self.timeLeft <= self.minimumTimeLeft:
          return self.__finish('No more time left')
        # Need to update the Configuration so that the new value is published in the next matching request
        result = self.computingElement.setCPUTimeLeft(cpuTimeLeft=self.timeLeft)
        if not result['OK']:
          return self.__finish(result['Message'])

        # Update local configuration to be used by submitted job wrappers
        localCfg = CFG()
        if self.extraOptions:
          localConfigFile = os.path.join('.', self.extraOptions)
        else:
          localConfigFile = os.path.join(rootPath, "etc", "dirac.cfg")
        localCfg.loadFromFile(localConfigFile)
        if not localCfg.isSection('/LocalSite'):
          localCfg.createNewSection('/LocalSite')
        localCfg.setOption('/LocalSite/CPUTimeLeft', self.timeLeft)
        localCfg.writeToFile(localConfigFile)

      else:
        return self.__finish('Filling Mode is Disabled')

    self.log.verbose('Job Agent execution loop')
    result = self.computingElement.available()
    if not result['OK']:
      self.log.info('Resource is not available')
      self.log.info(result['Message'])
      return self.__finish('CE Not Available')

    self.log.info(result['Message'])

    ceInfoDict = result['CEInfoDict']
    runningJobs = ceInfoDict.get("RunningJobs")
    availableSlots = result['Value']

    if not availableSlots:
      if runningJobs:
        self.log.info('No available slots with %d running jobs' % runningJobs)
        return S_OK('Job Agent cycle complete with %d running jobs' % runningJobs)
      else:
        self.log.info('CE is not available')
        return self.__finish('CE Not Available')

    result = self.computingElement.getDescription()
    if not result['OK']:
      return result
    ceDict = result['Value']
    # Add pilot information
    gridCE = gConfig.getValue('LocalSite/GridCE', 'Unknown')
    if gridCE != 'Unknown':
      ceDict['GridCE'] = gridCE
    if 'PilotReference' not in ceDict:
      ceDict['PilotReference'] = str(self.pilotReference)
    ceDict['PilotBenchmark'] = self.cpuFactor
    ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag

    # Add possible job requirements
    result = gConfig.getOptionsDict('/AgentJobRequirements')
    if result['OK']:
      requirementsDict = result['Value']
      ceDict.update(requirementsDict)
      self.log.info('Requirements:', requirementsDict)

    self.log.verbose(ceDict)
    start = time.time()
    jobRequest = MatcherClient().requestJob(ceDict)
    matchTime = time.time() - start
    self.log.info('MatcherTime = %.2f (s)' % (matchTime))

    self.stopAfterFailedMatches = self.am_getOption('StopAfterFailedMatches', self.stopAfterFailedMatches)

    if not jobRequest['OK']:
      if re.search('No match found', jobRequest['Message']):
        self.log.notice('Job request OK: %s' % (jobRequest['Message']))
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish('Nothing to do for more than %d cycles' % self.stopAfterFailedMatches)
        return S_OK(jobRequest['Message'])
      elif jobRequest['Message'].find("seconds timeout") != -1:
        self.log.error('Timeout while requesting job', jobRequest['Message'])
        self.matchFailedCount += 1
        if self.matchFailedCount > self.stopAfterFailedMatches:
          return self.__finish('Nothing to do for more than %d cycles' % self.stopAfterFailedMatches)
        return S_OK(jobRequest['Message'])
      elif jobRequest['Message'].find("Pilot version does not match") != -1:
        errorMsg = 'Pilot version does not match the production version'
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.CFG import CFG [as 别名]
# 或者: from DIRAC.Core.Utilities.CFG.CFG import writeToFile [as 别名]
    def execute(self):
        """The JobAgent execution method.
    """
        if self.jobCount:
            # Only call timeLeft utility after a job has been picked up
            self.log.info("Attempting to check CPU time left for filling mode")
            if self.fillingMode:
                if self.timeLeftError:
                    self.log.warn(self.timeLeftError)
                    return self.__finish(self.timeLeftError)
                self.log.info("%s normalized CPU units remaining in slot" % (self.timeLeft))
                if self.timeLeft <= self.minimumTimeLeft:
                    return self.__finish("No more time left")
                # Need to update the Configuration so that the new value is published in the next matching request
                result = self.computingElement.setCPUTimeLeft(cpuTimeLeft=self.timeLeft)
                if not result["OK"]:
                    return self.__finish(result["Message"])

                # Update local configuration to be used by submitted job wrappers
                localCfg = CFG()
                if self.extraOptions:
                    localConfigFile = os.path.join(".", self.extraOptions)
                else:
                    localConfigFile = os.path.join(rootPath, "etc", "dirac.cfg")
                localCfg.loadFromFile(localConfigFile)
                if not localCfg.isSection("/LocalSite"):
                    localCfg.createNewSection("/LocalSite")
                localCfg.setOption("/LocalSite/CPUTimeLeft", self.timeLeft)
                localCfg.writeToFile(localConfigFile)

            else:
                return self.__finish("Filling Mode is Disabled")

        self.log.verbose("Job Agent execution loop")
        available = self.computingElement.available()
        if not available["OK"] or not available["Value"]:
            self.log.info("Resource is not available")
            self.log.info(available["Message"])
            return self.__finish("CE Not Available")

        self.log.info(available["Message"])

        result = self.computingElement.getDescription()
        if not result["OK"]:
            return result
        ceDict = result["Value"]

        # Add pilot information
        gridCE = gConfig.getValue("LocalSite/GridCE", "Unknown")
        if gridCE != "Unknown":
            ceDict["GridCE"] = gridCE
        if not "PilotReference" in ceDict:
            ceDict["PilotReference"] = str(self.pilotReference)
        ceDict["PilotBenchmark"] = self.cpuFactor
        ceDict["PilotInfoReportedFlag"] = self.pilotInfoReportedFlag

        # Add possible job requirements
        result = gConfig.getOptionsDict("/AgentJobRequirements")
        if result["OK"]:
            requirementsDict = result["Value"]
            ceDict.update(requirementsDict)

        self.log.verbose(ceDict)
        start = time.time()
        jobRequest = self.__requestJob(ceDict)
        matchTime = time.time() - start
        self.log.info("MatcherTime = %.2f (s)" % (matchTime))

        self.stopAfterFailedMatches = self.am_getOption("StopAfterFailedMatches", self.stopAfterFailedMatches)

        if not jobRequest["OK"]:
            if re.search("No match found", jobRequest["Message"]):
                self.log.notice("Job request OK: %s" % (jobRequest["Message"]))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])
            elif jobRequest["Message"].find("seconds timeout") != -1:
                self.log.error("Timeout while requesting job", jobRequest["Message"])
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])
            elif jobRequest["Message"].find("Pilot version does not match") != -1:
                errorMsg = "Pilot version does not match the production version"
                self.log.error(errorMsg, jobRequest["Message"].replace(errorMsg, ""))
                return S_ERROR(jobRequest["Message"])
            else:
                self.log.notice("Failed to get jobs: %s" % (jobRequest["Message"]))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish("Nothing to do for more than %d cycles" % self.stopAfterFailedMatches)
                return S_OK(jobRequest["Message"])

        # Reset the Counter
        self.matchFailedCount = 0

        matcherInfo = jobRequest["Value"]
        if not self.pilotInfoReportedFlag:
            # Check the flag after the first access to the Matcher
#.........这里部分代码省略.........