Python CSAPI.commit方法代码示例

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
def updateCS( changeSet ):
  
  global vo, dry, ceBdiiDict
  
  changeList = list( changeSet )
  changeList.sort()
  if dry:
    gLogger.notice( 'The following needed changes are detected:\n' )
  else:  
    gLogger.notice( 'We are about to make the following changes to CS:\n' )
  for entry in changeList:
    gLogger.notice( "%s/%s %s -> %s" % entry )

  if not dry:
    csAPI = CSAPI()
    csAPI.initialize()
    result = csAPI.downloadCSData()
    if not result['OK']:
      gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
      DIRACExit( -1 )
    for section, option, value, new_value in changeSet:
      if value == 'Unknown' or not value:
        csAPI.setOption( cfgPath( section, option ), new_value )
      else:
        csAPI.modifyValue( cfgPath( section, option ), new_value )
        
    yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
    if yn == '' or yn.lower().startswith( 'y' ):    
      result = csAPI.commit()
      if not result['OK']:
        gLogger.error( "Error while commit to CS", result['Message'] )
      else:
        gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )  

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
    def commit(self):
        """Commit the changes to the configuration system."""
        # Perform all the appending operations at the end to only get from current config once.
        for path, value in chain(self._append_dict.iteritems(),
                                 self._append_unique_dict.iteritems()):
            section, option = path.rsplit('/', 1)
            self.add(section, option, value)
            self._num_changes += 1

        result = CSAPI.commit(self)
        if not result['OK']:
            gLogger.error("Error while commit to CS", result['Message'])
            raise RuntimeError("Error while commit to CS")
        if self._num_changes:
            gLogger.notice("Successfully committed %d changes to CS\n"
                           % self._num_changes)
            self._num_changes = 0
            self._append_dict.clear()
            self._append_unique_dict.clear()
        else:
            gLogger.notice("No changes to commit")

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
res = csAPI.createSection( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite' )
if not res['OK']:
  print res['Message']
  exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/matchParams' )
csAPI.setOption( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/policyType', 'AlwaysBanned' )
csAPI.setOption( 'Operations/Defaults/ResourceStatus/Policies/AlwaysBannedForSite/matchParams/element', 'Site' )


# Now setting the catalog list in Operations/Defults/Services/Catalogs/CatalogList

res = csAPI.createSection( 'Operations/Defaults/Services' )
if not res['OK']:
  print res['Message']
  exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/Services/Catalogs' )
if not res['OK']:
  print res['Message']
  exit( 1 )
res = csAPI.createSection( 'Operations/Defaults/Services/Catalogs/CatalogList' )
if not res['OK']:
  print res['Message']
  exit( 1 )
csAPI.setOption('Operations/Defaults/Services/Catalogs/CatalogList', 'FileCatalog')



# Final action: commit in CS
csAPI.commit()

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........

  def __updateCS( self, bdiiChangeSet ):
    
    queueVODict = {}
    changeSet = set()
    for entry in bdiiChangeSet:
      section, option , _value, new_value = entry
      if option == "VO":
        queueVODict.setdefault( section, set() )
        queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) )  
      else:
        changeSet.add( entry )  
    for section, VOs in queueVODict.items():
      changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) )    

    if changeSet:
      changeList = list( changeSet )
      changeList.sort()
      body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] )
      if body and self.addressTo and self.addressFrom:
        notification = NotificationClient()
        result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
        
      if body:  
        self.log.info( 'The following configuration changes were detected:' )  
        self.log.info( body )

      for section, option, value, new_value in changeSet:
        if value == 'Unknown' or not value:
          self.csAPI.setOption( cfgPath( section, option ), new_value )
        else:
          self.csAPI.modifyValue( cfgPath( section, option ), new_value )

      result = self.csAPI.commit()
      if not result['OK']:
        self.log.error( "Error while committing to CS", result['Message'] )
      else:
        self.log.info( "Successfully committed %d changes to CS" % len( changeList ) )
      return result
    else:
      self.log.info( "No changes found" )
      return S_OK()

  def __lookForNewSEs( self ):
    """ Look up BDII for SEs not yet present in the DIRAC CS
    """
    
    bannedSEs = self.am_getOption( 'BannedSEs', [] )
    result = getSEsFromCS()
    if not result['OK']:
      return result
    knownSEs = set( result['Value'] )
    knownSEs = knownSEs.union( set( bannedSEs ) )

    for vo in self.voName:
      result = self.__getBdiiSEInfo( vo )
      if not result['OK']:
        continue
      bdiiInfo = result['Value']
      result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs )
      if not result['OK']:
        continue
      siteDict = result['Value']  
      body = ''
      for site in siteDict:
        newSEs = set( siteDict[site].keys() )

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
                self.csAPI.modifyValue( section, newpilot )
              changed = True

          result = ldapCEState( ce, vo = self.voName )        #getBDIICEVOView
          if not result['OK']:
            self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
            result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
            continue
          try:
            queues = result['Value']
          except:
            self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
            continue

          newcetype = 'Unknown'
          for queue in queues:
            try:
              queuetype = queue['GlueCEImplementationName']
            except:
              queuetype = 'Unknown'
            if newcetype == 'Unknown':
              newcetype = queuetype
            else:
              if queuetype != newcetype:
                self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) )

          if newcetype=='ARC-CE':
            newcetype = 'ARC'

          if cetype != newcetype and newcetype != 'Unknown':
            section = cfgPath( ceSection, 'CEType' )
            self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
            if cetype == 'Unknown':
              self.csAPI.setOption( section, newcetype )
            else:
              self.csAPI.modifyValue( section, newcetype )
            changed = True

          for queue in queues:
            try:
              queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
            except:
              self.log.warn( 'error in queuename ', queue )
              continue

            try:
              newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
            except:
              newmaxCPUTime = None

            newsi00 = None
            try:
              caps = queue['GlueCECapability']
              if type( caps ) == type( '' ):
                caps = [caps]
              for cap in caps:
                if cap.count( 'CPUScalingReferenceSI00' ):
                  newsi00 = cap.split( '=' )[-1]
            except:
              newsi00 = None

            queueSection = cfgPath( ceSection, 'Queues', queueName )
            result = gConfig.getOptionsDict( queueSection )
            if not result['OK']:
              self.log.warn( "Section Queues", result['Message'] )
              maxCPUTime = 'Unknown'
              si00 = 'Unknown'
            else:
              queueopt = result['Value']
              maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
              si00 = queueopt.get( 'SI00', 'Unknown' )

            if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
              section = cfgPath( queueSection, 'maxCPUTime' )
              self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
              if maxCPUTime == 'Unknown':
                self.csAPI.setOption( section, newmaxCPUTime )
              else:
                self.csAPI.modifyValue( section, newmaxCPUTime )
              changed = True

            if newsi00 and ( si00 != newsi00 ):
              section = cfgPath( queueSection, 'SI00' )
              self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
              if si00 == 'Unknown':
                self.csAPI.setOption( section, newsi00 )
              else:
                self.csAPI.modifyValue( section, newsi00 )
              changed = True

    if changed:
      self.log.info( body )
      if body and self.addressTo and self.addressFrom:
        notification = NotificationClient()
        result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )

      return self.csAPI.commit()
    else:
      self.log.info( "No changes found" )
      return S_OK()

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
      diracSites = result['Value']
      yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
      if not yn or yn.lower().startswith( 'y' ):
        if len( diracSites ) > 1:
          prompt = 'Which DIRAC site the new SE should be attached to ?'
          for i, s in enumerate( diracSites ):
            prompt += '\n[%d] %s' % ( i, s )
          prompt += '\nEnter your choice number: '    
          inp = raw_input( prompt )
          try:
            ind = int( inp )
          except:
            gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
            continue
          diracSite = diracSites[ind]
        else:
          diracSite = diracSites[0]       
      
        domain, siteName, country = diracSite.split( '.' )
        recName = '%s-disk' % siteName
        inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
        diracSEName = inp
        if not inp:
          diracSEName = recName  
          
        gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )        
        seSection = cfgPath( '/Resources/StorageElements', diracSEName )
        changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
        changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )  
        bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
        seVOs = csVOs.intersection( bdiiVOs )  
        changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
        accessSection = cfgPath( seSection, 'AccessProtocol.1' )
        changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
        changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
        endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
        result = pfnparse( endPoint )
        if not result['OK']:
          gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
          continue
        host = result['Value']['Host']
        port = result['Value']['Port']
        changeSet.add( ( accessSection, 'Host', host ) ) 
        changeSet.add( ( accessSection, 'Port', port ) ) 
        changeSet.add( ( accessSection, 'Access', 'remote' ) )
        voPathSection = cfgPath( accessSection, 'VOPath' )
        if 'VOPath' in seDict:
          path = seDict['VOPath']
          voFromPath = os.path.basename( path )
          if voFromPath != diracVO:
            gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
            changeSet.add( ( voPathSection, diracVO, path ) )
          path = os.path.dirname( path )  
        else:  
          # Try to guess the Path
          domain = '.'.join( host.split( '.' )[-2:] )
          path = '/dpm/%s/home' % domain
        changeSet.add( ( accessSection, 'Path', path ) ) 
        changeSet.add( ( accessSection, 'SpaceToken', '' ) )
        changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) ) 
        
        gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
        changeList = list( changeSet )
        changeList.sort()
        for entry in changeList:
          gLogger.notice( entry )
        yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )   
        if not yn or yn.lower().startswith( 'y' ):
          changeSetFull = changeSetFull.union( changeSet )        
          
  if dry:
    if changeSetFull:
      gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
    else:
      gLogger.notice( "No new SE to be added" )
    return S_OK()

  if changeSetFull:
    csAPI = CSAPI()
    csAPI.initialize()
    result = csAPI.downloadCSData()
    if not result['OK']:
      gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
      DIRACExit( -1 )
    changeList = list( changeSetFull )
    changeList.sort()
    for section, option, value in changeList:
      csAPI.setOption( cfgPath( section, option ), value )

    yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
    if not yn or yn.lower().startswith( 'y' ):
      result = csAPI.commit()
      if not result['OK']:
        gLogger.error( "Error while commit to CS", result['Message'] )
      else:
        gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
  else:
    gLogger.notice( "No new SE to be added" )

  return S_OK()

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
    queueVODict = {}
    changeSet = set()
    for entry in bdiiChangeSet:
      section, option, _value, new_value = entry
      if option == "VO":
        queueVODict.setdefault(section, set())
        queueVODict[section] = queueVODict[section].union(set(new_value.split(',')))
      else:
        changeSet.add(entry)
    for section, VOs in queueVODict.items():
      changeSet.add((section, 'VO', '', ','.join(VOs)))

    if changeSet:
      changeList = sorted(changeSet)
      body = '\n'.join(["%s/%s %s -> %s" % entry for entry in changeList])
      if body and self.addressTo and self.addressFrom:
        notification = NotificationClient()
        result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)

      if body:
        self.log.info('The following configuration changes were detected:')
        self.log.info(body)

      for section, option, value, new_value in changeSet:
        if value == 'Unknown' or not value:
          self.csAPI.setOption(cfgPath(section, option), new_value)
        else:
          self.csAPI.modifyValue(cfgPath(section, option), new_value)

      if self.dryRun:
        self.log.info("Dry Run: CS won't be updated")
        self.csAPI.showDiff()
      else:
        result = self.csAPI.commit()
        if not result['OK']:
          self.log.error("Error while committing to CS", result['Message'])
        else:
          self.log.info("Successfully committed %d changes to CS" % len(changeList))
        return result
    else:
      self.log.info("No changes found")
      return S_OK()

  def __lookForNewSEs(self):
    """ Look up BDII for SEs not yet present in the DIRAC CS
    """

    bannedSEs = self.am_getOption('BannedSEs', [])
    result = getSEsFromCS()
    if not result['OK']:
      return result
    knownSEs = set(result['Value'])
    knownSEs = knownSEs.union(set(bannedSEs))

    for vo in self.voName:
      result = self.__getBdiiSEInfo(vo)
      if not result['OK']:
        continue
      bdiiInfo = result['Value']
      result = getGridSRMs(vo, bdiiInfo=bdiiInfo, srmBlackList=knownSEs)
      if not result['OK']:
        continue
      siteDict = result['Value']
      body = ''
      for site in siteDict:
        newSEs = set(siteDict[site].keys())  # pylint: disable=no-member

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
class CVMFSAdder(object):
  """Container for all the objects and functions to add software to ILCDirac"""
  def __init__(self, cliParams ):
    from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
    self.modifiedCS = False
    self.softSec = "/Operations/Defaults/AvailableTarBalls"
    self.mailadress = '[email protected]'
    self.cliParams = cliParams
    self.parameter = dict( softSec = self.softSec,
                           platform = cliParams.platform,
                           version = cliParams.version,
                           basepath = cliParams.basePath,
                           initsctipt = cliParams.initScriptLocation
                         )
    self.applications = cliParams.applicationSet
    self.detmodels = {}
    self.csAPI = CSAPI()

  def findDDSimDetectorModels( self ):
    """ find all detector models in lcgeo and
    fill the self.detmodels dictionary with Detmodel as key and path as value

    :returns: None
    """
    for root,dirs,_files in os.walk( os.path.join( self.parameter["basepath"], "lcgeo" ) ):
      for direct in dirs:
        if root.endswith("compact"):
          ## the main xml file must have the same name as the folder
          xmlPath = os.path.join( root, direct, direct+".xml")
          if os.path.exists( xmlPath ):
            self.detmodels[direct] = xmlPath

  def checkConsistency(self):
    """checks if platform is defined, application exists, etc."""
    gLogger.notice("Checking consistency")
    av_platforms = gConfig.getSections(self.softSec, [])
    if av_platforms['OK']:
      if not self.parameter['platform'] in av_platforms['Value']:
        gLogger.error("Platform %s unknown, available are %s." % (self.parameter['platform'], ", ".join(av_platforms['Value'])))
        gLogger.error("If yours is missing, add it in CS")
        return S_ERROR()
    else:
      gLogger.error("Could not find all platforms available in CS")
      return S_ERROR()

    for application in self.applications:
      av_apps = gConfig.getSections("%(softSec)s/%(platform)s/" % self.parameter + str(application), [])
      if not av_apps['OK']:
        gLogger.error("Could not find this application in the CS: '%s'" % application)
        gLogger.error("Add its section to the CS, if it is missing")
        return S_ERROR()

    gLogger.notice("All OK, continuing...")
    return S_OK()


  def commitToCS(self):
    """write changes to the CS to the server"""
    if self.modifiedCS and not self.cliParams.dryRun:
      gLogger.notice("Commiting changes to the CS")
      result = self.csAPI.commit()
      if not result[ 'OK' ]:
        gLogger.error('Commit failed with message = %s' % (result[ 'Message' ]))
        return S_ERROR("Failed to commit to CS")
      gLogger.info('Successfully committed changes to CS')
    else:
      gLogger.info('No modifications to CS required')
    return S_OK()

  def addAllToCS(self):
    """add all the applications to the CS, take care of special cases (mokka, ildconfig, ddsim,...)"""
    from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import insertCSSection

    for application in self.applications:
      csParameter = dict( CVMFSEnvScript = self.cliParams.initScriptLocation,
                          CVMFSPath      = self.parameter['basepath']
                        )

      if application == 'mokka':
        csParameter['CVMFSDBSlice'] = self.cliParams.dbSliceLocation

      if application == 'ddsim':
        self.findDDSimDetectorModels()

        csPathModels = "Operations/Defaults/DDSimDetectorModels"
        csModels = { self.parameter["version"] : self.detmodels }
        insertCSSection( self.csAPI, csPathModels, csModels )
        self.modifiedCS = True

      elif application.endswith('config'):
        del csParameter['CVMFSEnvScript']
        csParameter['CVMFSPath'] = self.cliParams.configPath
        if self.cliParams.dbSliceLocation:
          csParameter['CVMFSDBSlice'] = self.cliParams.dbSliceLocation

      resInsert = self.insertApplicationToCS(application, csParameter)
      if not resInsert['OK']:
        return resInsert

    return S_OK()
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
class MonitorAgents(AgentModule):
  """MonitorAgents class."""

  def __init__(self, *args, **kwargs):
    """Initialize the agent, clients, default values."""
    AgentModule.__init__(self, *args, **kwargs)
    self.name = 'MonitorAgents'
    self.setup = "Production"
    self.enabled = False
    self.restartAgents = False
    self.restartExecutors = False
    self.restartServices = False
    self.controlComponents = False
    self.commitURLs = False
    self.diracLocation = "/opt/dirac/pro"

    self.sysAdminClient = SystemAdministratorClient(socket.gethostname())
    self.jobMonClient = JobMonitoringClient()
    self.nClient = NotificationClient()
    self.csAPI = None
    self.agents = dict()
    self.executors = dict()
    self.services = dict()
    self.errors = list()
    self.accounting = defaultdict(dict)

    self.addressTo = ["[email protected]"]
    self.addressFrom = "[email protected]"
    self.emailSubject = "MonitorAgents on %s" % socket.gethostname()

  def logError(self, errStr, varMsg=''):
    """Append errors to a list, which is sent in email notification."""
    self.log.error(errStr, varMsg)
    self.errors.append(errStr + " " + varMsg)

  def beginExecution(self):
    """Reload the configurations before every cycle."""
    self.setup = self.am_getOption("Setup", self.setup)
    self.enabled = self.am_getOption("EnableFlag", self.enabled)
    self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents)
    self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors)
    self.restartServices = self.am_getOption("RestartServices", self.restartServices)
    self.diracLocation = os.environ.get("DIRAC", self.diracLocation)
    self.addressTo = self.am_getOption('MailTo', self.addressTo)
    self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
    self.controlComponents = self.am_getOption('ControlComponents', self.controlComponents)
    self.commitURLs = self.am_getOption('CommitURLs', self.commitURLs)

    self.csAPI = CSAPI()

    res = self.getRunningInstances(instanceType='Agents')
    if not res["OK"]:
      return S_ERROR("Failure to get running agents")
    self.agents = res["Value"]

    res = self.getRunningInstances(instanceType='Executors')
    if not res["OK"]:
      return S_ERROR("Failure to get running executors")
    self.executors = res["Value"]

    res = self.getRunningInstances(instanceType='Services')
    if not res["OK"]:
      return S_ERROR("Failure to get running services")
    self.services = res["Value"]

    self.accounting.clear()
    return S_OK()

  def sendNotification(self):
    """Send email notification about changes done in the last cycle."""
    if not(self.errors or self.accounting):
      return S_OK()

    emailBody = ""
    rows = []
    for instanceName, val in self.accounting.iteritems():
      rows.append([[instanceName],
                   [val.get('Treatment', 'No Treatment')],
                   [str(val.get('LogAge', 'Not Relevant'))]])

    if rows:
      columns = ["Instance", "Treatment", "Log File Age (Minutes)"]
      emailBody += printTable(columns, rows, printOut=False, numbering=False, columnSeparator=' | ')

    if self.errors:
      emailBody += "\n\nErrors:"
      emailBody += "\n".join(self.errors)

    self.log.notice("Sending Email:\n" + emailBody)
    for address in self.addressTo:
      res = self.nClient.sendMail(address, self.emailSubject, emailBody, self.addressFrom, localAttempt=False)
      if not res['OK']:
        self.log.error("Failure to send Email notification to ", address)
        continue

    self.errors = []
    self.accounting.clear()

    return S_OK()

#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
                        "G4LEVELGAMMADATA",
                        "G4RADIOACTIVEDATA",
                        "G4NEUTRONXSDATA",
                        "G4PIIDATA",
                        "G4REALSURFACEDATA",
                        "G4SAIDXSDATA",
                        "G4ABLADATA",
                        "G4ENSDFSTATEDATA",
                      ]
    for g4data in  g4datavariables:
      if os.environ.get(g4data) :
        csParameter["AdditionalEnvVar"][g4data] = os.path.realpath( os.environ.get(g4data) )

    pars = dict( platform=self.platform,
                 name="ddsim",
                 version=self.version
               )

    csPath = os.path.join( self.softSec , "%(platform)s/%(name)s/%(version)s/" % pars )
    pprint(csParameter)
    result = insertCSSection( self.csapi, csPath, csParameter )

    csPathModels = "Operations/Defaults/DDSimDetectorModels"
    csModels = { self.version : self.detmodels }
    pprint(csModels)

    result = insertCSSection( self.csapi, csPathModels, csModels )

    if self.csapi is not None:
      resProxy = checkOrGetGroupProxy( "diracAdmin" )
      if not resProxy['OK']:
        gLogger.error( "Failed to get AdminProxy", resProxy['Message'] )
        raise RuntimeError( "Failed to get diracAdminProxy" )
      self.csapi.commit()

    if not result['OK']:
      gLogger.error( "Failed to create CS Section", result['Message'] )
      raise RuntimeError( "Failed to create CS Section" )

  def createDDSimTarBall( self ):
    """ do everything to create the DDSim tarball"""

    from ILCDIRAC.ILCTransformationSystem.Utilities.ReleaseHelper import copyFolder, getLibraryPath, getFiles, getDependentLibraries, copyLibraries, getPythonStuff, killRPath, resolveLinks, removeSystemLibraries

    self.parseArgs()
    ddBase, lcgeoBase, _rootsys = self.checkEnvironment()

    realTargetFolder = os.path.join( os.getcwd(), self.name+self.version )
    targetFolder = os.path.join( os.getcwd(), "temp", self.name+self.version )
    for folder in (targetFolder, targetFolder+"/lib"):
      try:
        os.makedirs( folder )
      except OSError:
        pass

    libraries = set()
    rootmaps = set()

    dd4hepLibPath = getLibraryPath( ddBase )
    lcgeoPath = getLibraryPath( lcgeoBase )

    ## FIXME: Automatically pick up folders with /compact/ in them
    self.copyDetectorModels( lcgeoBase, "CLIC" , targetFolder+"/detectors" )
    self.copyDetectorModels( lcgeoBase, "ILD"  , targetFolder+"/detectors" )
    self.copyDetectorModels( lcgeoBase, "SiD"  , targetFolder+"/detectors" )

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
for i in resultList:
    if not cfg.isSection("Resources/Sites/%s" % i):
        gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
        continue
    hasRun = True
    if SORTBYNAME:
        dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(ascending=not REVERSE)
    else:
        dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE)
    if dirty:
        isDirty = True

if not hasRun:
    gLogger.notice("Failed to find suitable subsections with site names to sort")
    DIRAC.exit(0)

if not isDirty:
    gLogger.notice("Nothing to do, site names are already sorted")
    DIRAC.exit(0)

timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp)
cs.setOptionComment("/Resources/Sites", stamp)

result = cs.commit()
if not result["OK"]:
    gLogger.error("Failed to commit changes to CS", result["Message"])
    DIRAC.exit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRAC.exit(0)

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........

  def __addDIRACSiteName(self, inputList):
    """
    Extend given list of GOCDB endpoints with DIRAC site name, i.e.
    add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
    If given site name could not be found "DIRACSITENAME" is set to 'None'.

    :return: List of perfSONAR endpoints (dictionaries).
    """

    log = self.log.getSubLogger('__addDIRACSiteName')
    log.debug('Begin function ...')

    # get site name dictionary
    result = getDIRACGOCDictionary()
    if not result['OK']:
      log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message'])
      return S_ERROR('Could not get site name dictionary')

    # reverse the dictionary (assume 1 to 1 relation)
    DIRACGOCDict = result['Value']
    GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))

    # add DIRAC site names
    outputList = []
    for entry in inputList:
      try:
        entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
      except KeyError:
        self.log.warn("No dictionary entry for %s. " % entry['SITENAME'])
        entry['DIRACSITENAME'] = None
      outputList.append(entry)

    log.debug('End function.')
    return S_OK(outputList)

  def __updateConfiguration(self, setElements=None, delElements=None):
    """
    Update configuration stored by CS.
    """
    if setElements is None:
      setElements = {}
    if delElements is None:
      delElements = []

    log = self.log.getSubLogger('__updateConfiguration')
    log.debug('Begin function ...')

    # assure existence and proper value of a section or an option
    for path, value in setElements.iteritems():

      if value is None:
        section = path
      else:
        split = path.rsplit('/', 1)
        section = split[0]

      try:
        result = self.csAPI.createSection(section)
        if not result['OK']:
          log.error("createSection() failed with message: %s" % result['Message'])
      except Exception as e:
        log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')'))

      if value is not None:
        try:
          result = self.csAPI.setOption(path, value)
          if not result['OK']:
            log.error("setOption() failed with message: %s" % result['Message'])
        except Exception as e:
          log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')'))

    # delete elements in the configuration
    for path in delElements:
      result = self.csAPI.delOption(path)
      if not result['OK']:
        log.warn("delOption() failed with message: %s" % result['Message'])

        result = self.csAPI.delSection(path)
        if not result['OK']:
          log.warn("delSection() failed with message: %s" % result['Message'])

    if self.dryRun:
      log.info("Dry Run: CS won't be updated")
      self.csAPI.showDiff()
    else:
      # update configuration stored by CS
      result = self.csAPI.commit()
      if not result['OK']:
        log.error("commit() failed with message: %s" % result['Message'])
        return S_ERROR("Could not commit changes to CS.")
      else:
        log.info("Committed changes to CS")

    log.debug('End function.')
    return S_OK()

  # define mapping between an agent option in the configuration and a function call
  __functionMap = {'UpdatePerfSONARS': updatePerfSONARConfiguration,
                   }

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]
  Script.setUsageMessage( '\n'.join( [__doc__.split( '\n' )[1],
                                      'Usage:',
                                      '  %s [option|cfgfile] ... ShifterRole UserName DIRACGroup ...' % Script.scriptName,
                                      'Arguments:',
                                      '  ShifterRole: Name of the shifter role, e.g. DataManager',
                                      '  UserName: A user name, as registered in Registry section',
                                      '  DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)'] ) )
  Script.parseCommandLine( ignoreErrors = True )
  args = Script.getPositionalArgs( )

  csAPI = CSAPI( )

  if len( args ) < 3:
    Script.showHelp( )
    DIRACExit( -1 )

  shifterRole = args[0]
  userName = args[1]
  diracGroup = args[2]

  res = csAPI.addShifter( {shifterRole: {'User': userName, 'Group': diracGroup}} )
  if not res['OK']:
    gLogger.error( "Could not add shifter", ": " + res['Message'] )
    DIRACExit( 1 )
  res = csAPI.commit()
  if not res['OK']:
    gLogger.error( "Could not add shifter", ": " + res['Message'] )
    DIRACExit( 1 )
  gLogger.notice( "Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup) )

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
                        if newCEType == "ARC-CE":
                            newCEType = "ARC"

                        if ceType != newCEType and newCEType != "Unknown":
                            section = cfgPath(ceSection, "CEType")
                            self.log.info(section, " -> ".join((ceType, newCEType)))
                            if ceType == "Unknown":
                                self.csAPI.setOption(section, newCEType)
                            else:
                                self.csAPI.modifyValue(section, newCEType)
                            changed = True

                        for queue in queues:
                            try:
                                queueName = queue["GlueCEUniqueID"].split("/")[-1]
                            except:
                                self.log.warn("Error in queueName ", queue)
                                continue

                            try:
                                newMaxCPUTime = queue["GlueCEPolicyMaxCPUTime"]
                            except:
                                newMaxCPUTime = None

                            newSI00 = None
                            try:
                                caps = queue["GlueCECapability"]
                                if type(caps) == type(""):
                                    caps = [caps]
                                for cap in caps:
                                    if cap.count("CPUScalingReferenceSI00"):
                                        newSI00 = cap.split("=")[-1]
                            except:
                                newSI00 = None

                            queueSection = cfgPath(ceSection, "Queues", queueName)
                            result = gConfig.getOptionsDict(queueSection)
                            if not result["OK"]:
                                self.log.warn("Section Queues", result["Message"])
                                maxCPUTime = "Unknown"
                                si00 = "Unknown"
                                allowedVOs = [""]
                            else:
                                queueOpt = result["Value"]
                                maxCPUTime = queueOpt.get("maxCPUTime", "Unknown")
                                si00 = queueOpt.get("SI00", "Unknown")
                                if newVO == "":  # Remember previous iteration, if none - read from conf
                                    allowedVOs = queueOpt.get("VO", "").split(",")
                                else:  # Else use newVO, as it can contain changes, which aren't in conf yet
                                    allowedVOs = newVO.split(",")
                            if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
                                section = cfgPath(queueSection, "maxCPUTime")
                                self.log.info(section, " -> ".join((maxCPUTime, newMaxCPUTime)))
                                if maxCPUTime == "Unknown":
                                    self.csAPI.setOption(section, newMaxCPUTime)
                                else:
                                    self.csAPI.modifyValue(section, newMaxCPUTime)
                                changed = True

                            if newSI00 and (si00 != newSI00):
                                section = cfgPath(queueSection, "SI00")
                                self.log.info(section, " -> ".join((si00, newSI00)))
                                if si00 == "Unknown":
                                    self.csAPI.setOption(section, newSI00)
                                else:
                                    self.csAPI.modifyValue(section, newSI00)
                                changed = True

                            modifyVO = True  # Flag saying if we need VO option to change
                            newVO = ""
                            if allowedVOs != [""]:
                                for allowedVO in allowedVOs:
                                    allowedVO = allowedVO.strip()  # Get rid of spaces
                                    newVO += allowedVO
                                    if allowedVO == vo:  # Current VO has been already in list
                                        newVO = ""
                                        modifyVO = False  # Don't change anything
                                        break  # Skip next 'if', proceed to next VO
                                    newVO += ", "

                            if modifyVO:
                                section = cfgPath(queueSection, "VO")
                                newVO += vo
                                self.log.info(section, " -> ".join(("%s" % allowedVOs, newVO)))
                                if allowedVOs == [""]:
                                    self.csAPI.setOption(section, newVO)
                                else:
                                    self.csAPI.modifyValue(section, newVO)
                                changed = True

        if changed:
            self.log.info(body)
            if body and self.addressTo and self.addressFrom:
                notification = NotificationClient()
                result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)

            return self.csAPI.commit()
        else:
            self.log.info("No changes found")
            return S_OK()

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import commit [as 别名]

#.........这里部分代码省略.........
            diracSites = result["Value"]
            yn = raw_input(
                "\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
            )
            if not yn or yn.lower().startswith("y"):
                if len(diracSites) > 1:
                    prompt = "Which DIRAC site the new SE should be attached to ?"
                    for i, s in enumerate(diracSites):
                        prompt += "\n[%d] %s" % (i, s)
                    prompt += "\nEnter your choice number: "
                    inp = raw_input(prompt)
                    try:
                        ind = int(inp)
                    except:
                        gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
                        continue
                    diracSite = diracSites[ind]
                else:
                    diracSite = diracSites[0]

                domain, siteName, country = diracSite.split(".")
                recName = "%s-disk" % siteName
                inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
                diracSEName = inp
                if not inp:
                    diracSEName = recName

                gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
                seSection = cfgPath("/Resources/StorageElements", diracSEName)
                changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
                changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
                bdiiVOs = set(
                    [re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
                )
                seVOs = csVOs.intersection(bdiiVOs)
                changeSet.add((seSection, "VO", ",".join(seVOs)))
                accessSection = cfgPath(seSection, "AccessProtocol.1")
                changeSet.add((accessSection, "Protocol", "srm"))
                changeSet.add((accessSection, "PluginName", "SRM2"))
                endPoint = srmDict.get("GlueServiceEndpoint", "")
                host = urlparse(endPoint).hostname
                port = urlparse(endPoint).port
                changeSet.add((accessSection, "Host", host))
                changeSet.add((accessSection, "Port", port))
                changeSet.add((accessSection, "Access", "remote"))
                voPathSection = cfgPath(accessSection, "VOPath")
                if "VOPath" in seDict:
                    path = seDict["VOPath"]
                    voFromPath = os.path.basename(path)
                    if voFromPath != diracVO:
                        gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
                        changeSet.add((voPathSection, diracVO, path))
                    path = os.path.dirname(path)
                else:
                    # Try to guess the Path
                    domain = ".".join(host.split(".")[-2:])
                    path = "/dpm/%s/home" % domain
                changeSet.add((accessSection, "Path", path))
                changeSet.add((accessSection, "SpaceToken", ""))
                changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))

                gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
                changeList = list(changeSet)
                changeList.sort()
                for entry in changeList:
                    gLogger.notice(entry)
                yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
                if not yn or yn.lower().startswith("y"):
                    changeSetFull = changeSetFull.union(changeSet)

    if dry:
        if changeSetFull:
            gLogger.notice("Skipping commit of the new SE data in a dry run")
        else:
            gLogger.notice("No new SE to be added")
        return S_OK()

    if changeSetFull:
        csAPI = CSAPI()
        csAPI.initialize()
        result = csAPI.downloadCSData()
        if not result["OK"]:
            gLogger.error("Failed to initialize CSAPI object", result["Message"])
            DIRACExit(-1)
        changeList = list(changeSetFull)
        changeList.sort()
        for section, option, value in changeList:
            csAPI.setOption(cfgPath(section, option), value)

        yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
        if not yn or yn.lower().startswith("y"):
            result = csAPI.commit()
            if not result["OK"]:
                gLogger.error("Error while commit to CS", result["Message"])
            else:
                gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
    else:
        gLogger.notice("No new SE to be added")

    return S_OK()