Python CSAPI.downloadCSData方法代码示例

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class TestComponentInstallation( unittest.TestCase ):
  """
  Contains methods for testing of separate elements
  """

  def setUp( self ):
    self.host = 'localhost'
    self.notificationPort = 9154
    self.rootPwd = ''
    self.csClient = CSAPI()
    self.monitoringClient = ComponentMonitoringClient()
    self.client = SystemAdministratorClientCLI( self.host )

    self.csClient.downloadCSData()
    result = self.csClient.getCurrentCFG()
    if not result[ 'OK' ]:
      raise Exception( result[ 'Message' ] )
    cfg = result[ 'Value' ]

    setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' )

    self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' )
    self.rootPwd = cfg.getOption( 'Systems/Databases/Password' )
    self.diracPwd = self.rootPwd

  def tearDown( self ):
    pass

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
def updateCS( changeSet ):
  
  global vo, dry, ceBdiiDict
  
  changeList = list( changeSet )
  changeList.sort()
  if dry:
    gLogger.notice( 'The following needed changes are detected:\n' )
  else:  
    gLogger.notice( 'We are about to make the following changes to CS:\n' )
  for entry in changeList:
    gLogger.notice( "%s/%s %s -> %s" % entry )

  if not dry:
    csAPI = CSAPI()
    csAPI.initialize()
    result = csAPI.downloadCSData()
    if not result['OK']:
      gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
      DIRACExit( -1 )
    for section, option, value, new_value in changeSet:
      if value == 'Unknown' or not value:
        csAPI.setOption( cfgPath( section, option ), new_value )
      else:
        csAPI.modifyValue( cfgPath( section, option ), new_value )
        
    yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
    if yn == '' or yn.lower().startswith( 'y' ):    
      result = csAPI.commit()
      if not result['OK']:
        gLogger.error( "Error while commit to CS", result['Message'] )
      else:
        gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )  

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class TestComponentInstallation(unittest.TestCase):
  """
  Contains methods for testing of separate elements
  """

  def setUp(self):
    self.host = 'localhost'
    self.notificationPort = 9154
    self.rootPwd = ''
    self.csClient = CSAPI()
    self.monitoringClient = ComponentMonitoringClient()
    self.client = SystemAdministratorClientCLI(self.host)

    self.csClient.downloadCSData()
    result = self.csClient.getCurrentCFG()
    if not result['OK']:
      raise Exception(result['Message'])
    cfg = result['Value']

    setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')

    self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
    self.rootPwd = cfg.getOption('Systems/Databases/Password')
    self.diracPwd = self.rootPwd

    result = getProxyInfo()
    if not result['OK']:
      raise Exception(result['Message'])
    chain = result['Value']['chain']
    result = chain.getCertInChain(-1)
    if not result['OK']:
      raise Exception(result['Message'])
    result = result['Value'].getSubjectDN()
    if not result['OK']:
      raise Exception(result['Message'])
    userDN = result['Value']
    result = getUsernameForDN(userDN)
    if not result['OK']:
      raise Exception(result['Message'])
    self.user = result['Value']
    if not self.user:
      self.user = 'unknown'

  def tearDown(self):
    pass

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class Bdii2CSAgent( AgentModule ):

  addressTo = ''
  addressFrom = ''
  voName = ''
  subject = "CE2CSAgent"
  alternativeBDIIs = []

  def initialize( self ):

    self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
    self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
    # Create a list of alternative bdii urls
    self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
    # Check if the bdii url is appended by a port number, if not append the default 2170
    for index, url in enumerate( self.alternativeBDIIs ):
      if not url.split( ':' )[-1].isdigit():
        self.alternativeBDIIs[index] += ':2170'
    if self.addressTo and self.addressFrom:
      self.log.info( "MailTo", self.addressTo )
      self.log.info( "MailFrom", self.addressFrom )
    if self.alternativeBDIIs :
      self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
    self.subject = "CE2CSAgent"
    
    self.processCEs = self.am_getOption( 'ProcessCEs', True )
    self.processSEs = self.am_getOption( 'ProcessSEs', False )

    self.voName = self.am_getOption( 'VirtualOrganization', [] )
    if not self.voName:
      self.voName = self.am_getOption( 'VO', [] )
    if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
      # Get all VOs defined in the configuration
      self.voName = []
      result = getVOs()
      if result['OK']:
        vos = result['Value']
        for vo in vos:
          vomsVO = getVOOption( vo, "VOMSName" )
          if vomsVO:
            self.voName.append( vomsVO )

    if self.voName:
      self.log.info( "Agent will manage VO(s) %s" % self.voName )
    else:
      self.log.fatal( "VirtualOrganization option not defined for agent" )
      return S_ERROR()
    self.voBdiiCEDict = {}
    self.voBdiiSEDict = {}

    self.csAPI = CSAPI()
    return self.csAPI.initialize()

  def execute( self ):
    """ General agent execution method
    """
   
    # Get a "fresh" copy of the CS data
    result = self.csAPI.downloadCSData()
    if not result['OK']:
      self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )

    if self.processCEs:
      self.__lookForNewCEs()
      self.__updateCEs()
    if self.processSEs:  
      self.__lookForNewSEs()
      self.__updateSEs()
    return S_OK()

  def __lookForNewCEs( self ):
    """ Look up BDII for CEs not yet present in the DIRAC CS
    """

    bannedCEs = self.am_getOption( 'BannedCEs', [] )
    result = getCEsFromCS()
    if not result['OK']:
      return result
    knownCEs = set( result['Value'] )
    knownCEs = knownCEs.union( set( bannedCEs ) )

    for vo in self.voName:
      result = self.__getBdiiCEInfo( vo )
      if not result['OK']:
        continue
      bdiiInfo = result['Value']
      result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
      if not result['OK']:
        self.log.error( 'Failed to get unused CEs', result['Message'] )
      siteDict = result['Value']  
      body = ''
      for site in siteDict:
        newCEs = set( siteDict[site].keys() )
        if not newCEs:
          continue
        
        ceString = ''
        for ce in newCEs:
          queueString = ''
          ceInfo = bdiiInfo[site]['CEs'][ce]
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class CE2CSAgent( AgentModule ):

  addressTo = ''
  addressFrom = ''
  voName = ''
  subject = "CE2CSAgent"
  alternativeBDIIs = []

  def initialize( self ):

    # TODO: Have no default and if no mail is found then use the diracAdmin group
    # and resolve all associated mail addresses.
    self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
    self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
    # create a list of alternative bdii urls
    self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
    # check if the bdii url is appended by a port number, if not append the default 2170
    for index, url in enumerate( self.alternativeBDIIs ):
      if not url.split( ':' )[-1].isdigit():
        self.alternativeBDIIs[index] += ':2170'
    if self.addressTo and self.addressFrom:
      self.log.info( "MailTo", self.addressTo )
      self.log.info( "MailFrom", self.addressFrom )
    if self.alternativeBDIIs :
      self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
    self.subject = "CE2CSAgent"

    # This sets the Default Proxy to used as that defined under
    # /Operations/Shifter/TestManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'TestManager' )

    self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
    if not self.voName:
      self.voName = getVO()

    if not self.voName:
      self.log.fatal( "VO option not defined for agent" )
      return S_ERROR()

    self.csAPI = CSAPI()
    return self.csAPI.initialize()

  def execute( self ):

    self.log.info( "Start Execution" )
    result = getProxyInfo()
    if not result[ 'OK' ]:
      return result
    infoDict = result[ 'Value' ]
    self.log.info( formatProxyInfoAsString( infoDict ) )

    #Get a "fresh" copy of the CS data
    result = self.csAPI.downloadCSData()
    if not result[ 'OK' ]:
      self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )

    self.__lookForCE()
    self.__infoFromCE()
    self.log.info( "End Execution" )
    return S_OK()

  def __checkAlternativeBDIISite( self, fun, *args ):
    if self.alternativeBDIIs:
      self.log.warn( "Trying to use alternative bdii sites" )
      for site in self.alternativeBDIIs :
        self.log.info( "Trying to contact alternative bdii ", site )
        if len( args ) == 1 :
          result = fun( args[0], host = site )
        elif len( args ) == 2 :
          result = fun( args[0], vo = args[1], host = site )
        if not result['OK'] :
          self.log.error ( "Problem contacting alternative bddii", result['Message'] )
        elif result['OK'] :
          return result
      self.log.warn( "Also checking alternative BDII sites failed" )
      return result

  def __lookForCE( self ):

    knownces = self.am_getOption( 'BannedCEs', [] )

    result = gConfig.getSections( '/Resources/Sites' )
    if not result['OK']:
      return
    grids = result['Value']

    for grid in grids:

      result = gConfig.getSections( '/Resources/Sites/%s' % grid )
      if not result['OK']:
        return
      sites = result['Value']

      for site in sites:
        opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
        ces = List.fromChar( opt.get( 'CE', '' ) )
        knownces += ces

    response = ldapCEState( '', vo = self.voName )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]

#.........这里部分代码省略.........
      diracSites = result['Value']
      yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
      if not yn or yn.lower().startswith( 'y' ):
        if len( diracSites ) > 1:
          prompt = 'Which DIRAC site the new SE should be attached to ?'
          for i, s in enumerate( diracSites ):
            prompt += '\n[%d] %s' % ( i, s )
          prompt += '\nEnter your choice number: '    
          inp = raw_input( prompt )
          try:
            ind = int( inp )
          except:
            gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
            continue
          diracSite = diracSites[ind]
        else:
          diracSite = diracSites[0]       
      
        domain, siteName, country = diracSite.split( '.' )
        recName = '%s-disk' % siteName
        inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
        diracSEName = inp
        if not inp:
          diracSEName = recName  
          
        gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )        
        seSection = cfgPath( '/Resources/StorageElements', diracSEName )
        changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
        changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )  
        bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
        seVOs = csVOs.intersection( bdiiVOs )  
        changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
        accessSection = cfgPath( seSection, 'AccessProtocol.1' )
        changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
        changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
        endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
        result = pfnparse( endPoint )
        if not result['OK']:
          gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
          continue
        host = result['Value']['Host']
        port = result['Value']['Port']
        changeSet.add( ( accessSection, 'Host', host ) ) 
        changeSet.add( ( accessSection, 'Port', port ) ) 
        changeSet.add( ( accessSection, 'Access', 'remote' ) )
        voPathSection = cfgPath( accessSection, 'VOPath' )
        if 'VOPath' in seDict:
          path = seDict['VOPath']
          voFromPath = os.path.basename( path )
          if voFromPath != diracVO:
            gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
            changeSet.add( ( voPathSection, diracVO, path ) )
          path = os.path.dirname( path )  
        else:  
          # Try to guess the Path
          domain = '.'.join( host.split( '.' )[-2:] )
          path = '/dpm/%s/home' % domain
        changeSet.add( ( accessSection, 'Path', path ) ) 
        changeSet.add( ( accessSection, 'SpaceToken', '' ) )
        changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) ) 
        
        gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
        changeList = list( changeSet )
        changeList.sort()
        for entry in changeList:
          gLogger.notice( entry )
        yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )   
        if not yn or yn.lower().startswith( 'y' ):
          changeSetFull = changeSetFull.union( changeSet )        
          
  if dry:
    if changeSetFull:
      gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
    else:
      gLogger.notice( "No new SE to be added" )
    return S_OK()

  if changeSetFull:
    csAPI = CSAPI()
    csAPI.initialize()
    result = csAPI.downloadCSData()
    if not result['OK']:
      gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
      DIRACExit( -1 )
    changeList = list( changeSetFull )
    changeList.sort()
    for section, option, value in changeList:
      csAPI.setOption( cfgPath( section, option ), value )

    yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
    if not yn or yn.lower().startswith( 'y' ):
      result = csAPI.commit()
      if not result['OK']:
        gLogger.error( "Error while commit to CS", result['Message'] )
      else:
        gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
  else:
    gLogger.notice( "No new SE to be added" )

  return S_OK()

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class Bdii2CSAgent(AgentModule):

  def __init__(self, *args, **kwargs):
    """ Defines default parameters
    """

    super(Bdii2CSAgent, self).__init__(*args, **kwargs)

    self.addressTo = ''
    self.addressFrom = ''
    self.voName = []
    self.subject = "Bdii2CSAgent"
    self.alternativeBDIIs = []
    self.voBdiiCEDict = {}
    self.voBdiiSEDict = {}
    self.host = 'lcg-bdii.cern.ch:2170'
    self.glue2URLs = []
    self.glue2Only = False

    self.csAPI = None

    # What to get
    self.processCEs = True
    self.processSEs = False
    self.selectedSites = []

    # Update the CS or not?
    self.dryRun = False

  def initialize(self):
    """ Gets run paramaters from the configuration
    """

    self.addressTo = self.am_getOption('MailTo', self.addressTo)
    self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
    # Create a list of alternative bdii urls
    self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
    self.host = self.am_getOption('Host', self.host)
    self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
    self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)

    # Check if the bdii url is appended by a port number, if not append the default 2170
    for index, url in enumerate(self.alternativeBDIIs):
      if not url.split(':')[-1].isdigit():
        self.alternativeBDIIs[index] += ':2170'
    if self.addressTo and self.addressFrom:
      self.log.info("MailTo", self.addressTo)
      self.log.info("MailFrom", self.addressFrom)
    if self.alternativeBDIIs:
      self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)

    self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
    self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
    self.selectedSites = self.am_getOption('SelectedSites', [])
    self.dryRun = self.am_getOption('DryRun', self.dryRun)

    self.voName = self.am_getOption('VirtualOrganization', self.voName)
    if not self.voName:
      self.voName = self.am_getOption('VO', [])
    if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
      # Get all VOs defined in the configuration
      self.voName = []
      result = getVOs()
      if result['OK']:
        vos = result['Value']
        for vo in vos:
          vomsVO = getVOOption(vo, "VOMSName")
          if vomsVO:
            self.voName.append(vomsVO)

    if self.voName:
      self.log.info("Agent will manage VO(s) %s" % self.voName)
    else:
      self.log.fatal("VirtualOrganization option not defined for agent")
      return S_ERROR()

    self.csAPI = CSAPI()
    return self.csAPI.initialize()

  def execute(self):
    """ General agent execution method
    """
    self.voBdiiCEDict = {}
    self.voBdiiSEDict = {}

    # Get a "fresh" copy of the CS data
    result = self.csAPI.downloadCSData()
    if not result['OK']:
      self.log.warn("Could not download a fresh copy of the CS data", result['Message'])

    # Refresh the configuration from the master server
    gConfig.forceRefresh(fromMaster=True)

    if self.processCEs:
      self.__lookForNewCEs()
      self.__updateCEs()
    if self.processSEs:
      self.__lookForNewSEs()
      self.__updateSEs()
    return S_OK()
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]

#.........这里部分代码省略.........
      if self.autoDeleteUsers:
        self.log.info('The following users will be deleted: %s' % str(oldUsers))
        result = self.csapi.deleteUsers(oldUsers)
        if result['OK']:
          self.adminMsgs['Info'].append('The following users are deleted from CS:\n  %s\n' % str(oldUsers))
          resultDict['DeletedUsers'] = oldUsers
        else:
          self.adminMsgs['Errors'].append('Error in deleting users from CS:\n  %s' % str(oldUsers))
          self.log.error('Error while user deletion from CS', result)
      else:
        self.adminMsgs['Info'].append('The following users to be checked for deletion:\n  %s' % str(oldUsers))
        self.log.info('The following users to be checked for deletion: %s' % str(oldUsers))

    resultDict['CSAPI'] = self.csapi
    resultDict['AdminMessages'] = self.adminMsgs
    return S_OK(resultDict)

  def getVOUserData(self, refreshFlag=False):
    """ Get a report for users of a given VO

    :param bool refreshFlag: flag to indicate that the configuration must be refreshed
                             before looking up user data
    :return: S_OK/S_ERROR, Value = user description dictionary
    """
    if refreshFlag:
      gConfig.forceRefresh()

    # Get DIRAC users
    diracUsers = getUsersInVO(self.vo)
    if not diracUsers:
      return S_ERROR("No VO users found for %s" % self.vo)

    if refreshFlag:
      result = self.csapi.downloadCSData()
      if not result['OK']:
        return result
    result = self.csapi.describeUsers(diracUsers)
    if not result['OK']:
      self.log.error('Could not retrieve CS User description')
    return result

  def getVOUserReport(self):
    """ Get a report string with the current status of the DIRAC Registry for the
        Virtual Organization

    :return: S_OK with the report string as Value
    """

    result = self.getVOUserData(refreshFlag=True)
    if not result['OK']:
      return result

    userDict = result['Value']

    # Get DIRAC group vs VOMS Role Mappings
    result = getVOMSRoleGroupMapping(self.vo)
    if not result['OK']:
      return result

    diracVOMSMapping = result['Value']['DIRACVOMS']
    records = []
    groupDict = defaultdict(int)
    multiDNUsers = {}
    suspendedUsers = []
    for user in userDict:
      for group in userDict[user]['Groups']:

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]

#.........这里部分代码省略.........
      self.log.info( "  %s" % infoMsg )

    self.log.info( "Checking static groups" )
    staticUsers = []
    for group in staticGroups:
      self.log.info( "  Checking static group %s" % group )
      numUsersInGroup = 0
      result = csapi.listUsers( group )
      if not result[ 'OK' ]:
        self.log.error( "Could not get the list of users in DIRAC group %s" % group , result[ 'Message' ] )
        return result
      for userName in result[ 'Value' ]:
        if userName in usersData:
          numUsersInGroup += 1
          usersData[ userName ][ 'Groups' ].append( group )
        else:
          if group not in vomsGroups and userName not in staticUsers:
            staticUsers.append( userName )
      infoMsg = "There are %s users in group %s" % ( numUsersInGroup, group )
      self.__adminMsgs[ 'Info' ].append( infoMsg )
      self.log.info( "  %s" % infoMsg )
    if staticUsers:
      infoMsg = "There are %s static users: %s" % ( len( staticUsers ) , ', '.join( staticUsers ) )
      self.__adminMsgs[ 'Info' ].append( infoMsg )
      self.log.info( "%s" % infoMsg )

    for user in currentUsers:
      if user not in usersData and user not in staticUsers:
        self.log.info( 'User %s is no longer valid' % user )
        obsoleteUserNames.append( user )

    #Do the CS Sync
    self.log.info( "Updating CS..." )
    ret = csapi.downloadCSData()
    if not ret['OK']:
      self.log.fatal( 'Can not update from CS', ret['Message'] )
      return ret

    usersWithMoreThanOneDN = {}
    for user in usersData:
      csUserData = dict( usersData[ user ] )
      if len( csUserData[ 'DN' ] ) > 1:
        usersWithMoreThanOneDN[ user ] = csUserData[ 'DN' ]
      result = csapi.describeUsers( [ user ] )
      if result[ 'OK' ]:
        if result[ 'Value' ]:
          prevUser = result[ 'Value' ][ user ]
          prevDNs = List.fromChar( prevUser[ 'DN' ] )
          newDNs = csUserData[ 'DN' ]
          for DN in newDNs:
            if DN not in prevDNs:
              self.__adminMsgs[ 'Info' ].append( "User %s has new DN %s" % ( user, DN ) )
          for DN in prevDNs:
            if DN not in newDNs:
              self.__adminMsgs[ 'Info' ].append( "User %s has lost a DN %s" % ( user, DN ) )
        else:
          newDNs = csUserData[ 'DN' ]
          for DN in newDNs:
            self.__adminMsgs[ 'Info' ].append( "New user %s has new DN %s" % ( user, DN ) )
      for k in ( 'DN', 'CA', 'Email' ):
        csUserData[ k ] = ", ".join( csUserData[ k ] )
      result = csapi.modifyUser( user, csUserData, createIfNonExistant = True )
      if not result[ 'OK' ]:
        self.__adminMsgs[ 'Error' ].append( "Cannot modify user %s: %s" % ( user, result[ 'Message' ] ) )
        self.log.error( "Cannot modify user", user )

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class VOMS2CSAgent( AgentModule ):

  def initialize( self ):

    self.__voDict = {}
    voNames = self.am_getOption( 'VO', [] )
    if not voNames[0].lower() == "none":
      if voNames[0].lower() == "any":
        voNames = []
      result = getVOMSVOs( voNames )
      if not result['OK']:
        return result
      self.__voDict = result['Value']

    self.__adminMsgs = {}
    self.csapi = CSAPI()
    self.voChanged = False

    self.log.notice( "VOs: %s" % self.__voDict.keys() )

    self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False )
    self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False )
    self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False )
    return S_OK()

  def execute( self ):

    self.__adminMsgs = {}
    self.csapi.downloadCSData()
    for vo in self.__voDict:
      self.voChanged = False
      voAdminUser = getVOOption( vo, "VOAdmin")
      voAdminMail = None
      if voAdminUser:
        voAdminMail = getUserOption( voAdminUser, "Email")
      voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )

      self.log.info( 'Performing VOMS sync for VO %s with credentials %[email protected]%s' % ( vo, voAdminUser, voAdminGroup ) )

      result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
      if not result['OK']:
        self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
        continue

      if self.voChanged:
        mailMsg = ""
        if self.__adminMsgs[ 'Errors' ]:
          mailMsg += "\nErrors list:\n  %s" % "\n  ".join( self.__adminMsgs[ 'Errors' ] )
        if self.__adminMsgs[ 'Info' ]:
          mailMsg += "\nRun result:\n  %s" % "\n  ".join( self.__adminMsgs[ 'Info' ] )
        NotificationClient().sendMail( self.am_getOption( 'MailTo', voAdminMail ),
                                       "VOMS2CSAgent run log", mailMsg,
                                       self.am_getOption( 'mailFrom', "DIRAC system" ) )

    # We have accumulated all the changes, commit them now
    result = self.csapi.commitChanges()
    if not result[ 'OK' ]:
      self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
      return result
    self.log.info( "Configuration committed" )
    return S_OK()

  @executeWithUserProxy
  def __syncCSWithVOMS( self, vo ):
    self.__adminMsgs = { 'Errors' : [], 'Info' : [] }

    # Get DIRAC group vs VOMS Role Mappings
    result = getVOMSRoleGroupMapping( vo )
    if not result['OK']:
      return result

    vomsDIRACMapping = result['Value']['VOMSDIRAC']
    diracVOMSMapping = result['Value']['DIRACVOMS']
    noVOMSGroups = result['Value']['NoVOMS']

    vomsSrv = VOMSService( vo )

    # Get VOMS VO name
    result = vomsSrv.admGetVOName()
    if not result['OK']:
      self.log.error( 'Could not retrieve VOMS VO name', "for %s" % vo )
      return result
    vomsVOName = result[ 'Value' ].lstrip( '/' )
    self.log.verbose( "VOMS VO Name for %s is %s" % ( vo, vomsVOName ) )

    # Get VOMS users
    result = vomsSrv.getUsers()
    if not result['OK']:
      self.log.error( 'Could not retrieve user information from VOMS', result['Message'] )
      return result
    vomsUserDict = result[ 'Value' ]
    message = "There are %s registered users in VOMS VO %s" % ( len( vomsUserDict ), vomsVOName )
    self.__adminMsgs[ 'Info' ].append( message )
    self.log.info( message )

    # Get DIRAC users
    diracUsers = getUsersInVO( vo )
    if not diracUsers:
      return S_ERROR( "No VO users found for %s" % vo )

#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class VOMS2CSAgent( AgentModule ):

  def __init__( self, *args, **kwargs ):
    """ Defines default parameters
    """
    super(VOMS2CSAgent, self).__init__( *args, **kwargs )

    self.__voDict = {}
    self.__adminMsgs = {}
    self.csapi = None
    self.voChanged = False
    self.dryRun = False

    self.autoAddUsers = False
    self.autoModifyUsers = False
    self.autoDeleteUsers = False
    self.detailedReport = True
    self.makeFCEntry = False

  def initialize( self ):
    """ Initialize the default parameters
    """

    voNames = self.am_getOption( 'VO', [] )
    if not voNames[0].lower() == "none":
      if voNames[0].lower() == "any":
        voNames = []
      result = getVOMSVOs( voNames )
      if not result['OK']:
        return result
      self.__voDict = result['Value']
      self.log.notice( "VOs: %s" % self.__voDict.keys() )

    self.csapi = CSAPI()

    self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
    self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
    self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
    self.autoDeleteUsers = self.am_getOption( 'AutoDeleteUsers', self.autoDeleteUsers )
    self.detailedReport = self.am_getOption( 'DetailedReport', self.detailedReport )
    self.makeFCEntry = self.am_getOption( 'MakeHomeDirectory', self.makeFCEntry )

    return S_OK()

  def execute( self ):

    self.__adminMsgs = {}
    self.csapi.downloadCSData()
    for vo in self.__voDict:
      self.voChanged = False
      voAdminUser = getVOOption( vo, "VOAdmin")
      voAdminMail = None
      if voAdminUser:
        voAdminMail = getUserOption( voAdminUser, "Email")
      voAdminGroup = getVOOption( vo, "VOAdminGroup", getVOOption( vo, "DefaultGroup" ) )

      self.log.info( 'Performing VOMS sync for VO %s with credentials %[email protected]%s' % ( vo, voAdminUser, voAdminGroup ) )

      result = self.__syncCSWithVOMS( vo, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
      if not result['OK']:
        self.log.error( 'Failed to perform VOMS to CS synchronization:', 'VO %s: %s' % ( vo, result["Message"] ) )
        continue
      resultDict = result['Value']
      newUsers = resultDict.get( "NewUsers", [] )
      modUsers = resultDict.get( "ModifiedUsers", [] )
      delUsers = resultDict.get( "DeletedUsers", [] )
      self.log.info( "Run results: new users %d, modified users %d, deleted users %d" % \
                     ( len( newUsers ), len( modUsers ), len( delUsers ) ) )

      if self.csapi.csModified:
        # We have accumulated all the changes, commit them now
        self.log.info( "There are changes to the CS for vo %s ready to be committed" % vo )
        if self.dryRun:
          self.log.info( "Dry Run: CS won't be updated" )
          self.csapi.showDiff()
        else:
          result = self.csapi.commitChanges()
          if not result[ 'OK' ]:
            self.log.error( "Could not commit configuration changes", result[ 'Message' ] )
            return result
          self.log.notice( "Configuration committed for VO %s" % vo )
      else:
        self.log.info( "No changes to the CS for VO %s recorded at this cycle" % vo )

      # Add user home directory in the file catalog
      if self.makeFCEntry and newUsers:
        self.log.info( "Creating home directories for users %s" % str( newUsers ) )
        result = self.__addHomeDirectory( vo, newUsers, proxyUserName = voAdminUser, proxyUserGroup = voAdminGroup ) #pylint: disable=unexpected-keyword-arg
        if not result['OK']:
          self.log.error( 'Failed to create user home directories:', 'VO %s: %s' % ( vo, result["Message"] ) )
        else:
          for user in result['Value']['Failed']:
            self.log.error( "Failed to create home directory", "user: %s, operation: %s" % \
                            ( user, result['Value']['Failed'][user] ) )
            self.__adminMsgs[ 'Errors' ].append( "Failed to create home directory for user %s: operation %s" % \
                                                 ( user, result['Value']['Failed'][user] ) )
          for user in result['Value']['Successful']:
            self.__adminMsgs[ 'Info' ].append( "Created home directory for user %s" % user )

      if self.voChanged or self.detailedReport:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]
class CE2CSAgent(AgentModule):

    addressTo = ""
    addressFrom = ""
    voName = ""
    subject = "CE2CSAgent"
    alternativeBDIIs = []

    def initialize(self):

        # TODO: Have no default and if no mail is found then use the diracAdmin group
        # and resolve all associated mail addresses.
        self.addressTo = self.am_getOption("MailTo", self.addressTo)
        self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
        # Create a list of alternative bdii urls
        self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", [])
        # Check if the bdii url is appended by a port number, if not append the default 2170
        for index, url in enumerate(self.alternativeBDIIs):
            if not url.split(":")[-1].isdigit():
                self.alternativeBDIIs[index] += ":2170"
        if self.addressTo and self.addressFrom:
            self.log.info("MailTo", self.addressTo)
            self.log.info("MailFrom", self.addressFrom)
        if self.alternativeBDIIs:
            self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
        self.subject = "CE2CSAgent"

        # This sets the Default Proxy to used as that defined under
        # /Operations/Shifter/TestManager
        # the shifterProxy option in the Configuration can be used to change this default.
        self.am_setOption("shifterProxy", "TestManager")

        self.voName = self.am_getOption("VirtualOrganization", [])
        if not self.voName:
            vo = getVO()
            if vo:
                self.voName = [vo]

        if self.voName:
            self.log.info("Agent will manage VO(s) %s" % self.voName)
        else:
            self.log.fatal("VirtualOrganization option not defined for agent")
            return S_ERROR()

        self.csAPI = CSAPI()
        return self.csAPI.initialize()

    def execute(self):

        self.log.info("Start Execution")
        result = getProxyInfo()
        if not result["OK"]:
            return result
        infoDict = result["Value"]
        self.log.info(formatProxyInfoAsString(infoDict))

        # Get a "fresh" copy of the CS data
        result = self.csAPI.downloadCSData()
        if not result["OK"]:
            self.log.warn("Could not download a fresh copy of the CS data", result["Message"])

        self.__lookForCE()
        self.__infoFromCE()
        self.log.info("End Execution")
        return S_OK()

    def __checkAlternativeBDIISite(self, fun, *args):
        if self.alternativeBDIIs:
            self.log.warn("Trying to use alternative BDII sites")
            for site in self.alternativeBDIIs:
                self.log.info("Trying to contact alternative BDII", site)
                if len(args) == 1:
                    result = fun(args[0], host=site)
                elif len(args) == 2:
                    result = fun(args[0], vo=args[1], host=site)
                if not result["OK"]:
                    self.log.error("Problem contacting alternative BDII", result["Message"])
                elif result["OK"]:
                    return result
            self.log.warn("Also checking alternative BDII sites failed")
            return result

    def __lookForCE(self):

        knownCEs = self.am_getOption("BannedCEs", [])

        result = gConfig.getSections("/Resources/Sites")
        if not result["OK"]:
            return
        grids = result["Value"]

        for grid in grids:
            result = gConfig.getSections("/Resources/Sites/%s" % grid)
            if not result["OK"]:
                return
            sites = result["Value"]

            for site in sites:
                opt = gConfig.getOptionsDict("/Resources/Sites/%s/%s" % (grid, site))["Value"]
                ces = List.fromChar(opt.get("CE", ""))
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI [as 别名]
# 或者: from DIRAC.ConfigurationSystem.Client.CSAPI.CSAPI import downloadCSData [as 别名]

#.........这里部分代码省略.........
            diracSites = result["Value"]
            yn = raw_input(
                "\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
            )
            if not yn or yn.lower().startswith("y"):
                if len(diracSites) > 1:
                    prompt = "Which DIRAC site the new SE should be attached to ?"
                    for i, s in enumerate(diracSites):
                        prompt += "\n[%d] %s" % (i, s)
                    prompt += "\nEnter your choice number: "
                    inp = raw_input(prompt)
                    try:
                        ind = int(inp)
                    except:
                        gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
                        continue
                    diracSite = diracSites[ind]
                else:
                    diracSite = diracSites[0]

                domain, siteName, country = diracSite.split(".")
                recName = "%s-disk" % siteName
                inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
                diracSEName = inp
                if not inp:
                    diracSEName = recName

                gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
                seSection = cfgPath("/Resources/StorageElements", diracSEName)
                changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
                changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
                bdiiVOs = set(
                    [re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
                )
                seVOs = csVOs.intersection(bdiiVOs)
                changeSet.add((seSection, "VO", ",".join(seVOs)))
                accessSection = cfgPath(seSection, "AccessProtocol.1")
                changeSet.add((accessSection, "Protocol", "srm"))
                changeSet.add((accessSection, "PluginName", "SRM2"))
                endPoint = srmDict.get("GlueServiceEndpoint", "")
                host = urlparse(endPoint).hostname
                port = urlparse(endPoint).port
                changeSet.add((accessSection, "Host", host))
                changeSet.add((accessSection, "Port", port))
                changeSet.add((accessSection, "Access", "remote"))
                voPathSection = cfgPath(accessSection, "VOPath")
                if "VOPath" in seDict:
                    path = seDict["VOPath"]
                    voFromPath = os.path.basename(path)
                    if voFromPath != diracVO:
                        gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
                        changeSet.add((voPathSection, diracVO, path))
                    path = os.path.dirname(path)
                else:
                    # Try to guess the Path
                    domain = ".".join(host.split(".")[-2:])
                    path = "/dpm/%s/home" % domain
                changeSet.add((accessSection, "Path", path))
                changeSet.add((accessSection, "SpaceToken", ""))
                changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))

                gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
                changeList = list(changeSet)
                changeList.sort()
                for entry in changeList:
                    gLogger.notice(entry)
                yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
                if not yn or yn.lower().startswith("y"):
                    changeSetFull = changeSetFull.union(changeSet)

    if dry:
        if changeSetFull:
            gLogger.notice("Skipping commit of the new SE data in a dry run")
        else:
            gLogger.notice("No new SE to be added")
        return S_OK()

    if changeSetFull:
        csAPI = CSAPI()
        csAPI.initialize()
        result = csAPI.downloadCSData()
        if not result["OK"]:
            gLogger.error("Failed to initialize CSAPI object", result["Message"])
            DIRACExit(-1)
        changeList = list(changeSetFull)
        changeList.sort()
        for section, option, value in changeList:
            csAPI.setOption(cfgPath(section, option), value)

        yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
        if not yn or yn.lower().startswith("y"):
            result = csAPI.commit()
            if not result["OK"]:
                gLogger.error("Error while commit to CS", result["Message"])
            else:
                gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
    else:
        gLogger.notice("No new SE to be added")

    return S_OK()