Python DataOperation.DataOperation類代碼示例

  def __sendAccounting( ftsJob, ownerDN ):
    """ prepare and send DataOperation to AccouringDB """

    dataOp = DataOperation()
    dataOp.setStartTime( fromString( ftsJob.SubmitTime ) )
    dataOp.setEndTime( fromString( ftsJob.LastUpdate ) )

    accountingDict = dict()
    accountingDict["OperationType"] = "ReplicateAndRegister"

    username = getUsernameForDN( ownerDN )
    if not username["OK"]:
      username = ownerDN
    else:
      username = username["Value"]

    accountingDict["User"] = username
    accountingDict["Protocol"] = "FTS"

    # accountingDict['RegistrationTime'] = 0
    # accountingDict['RegistrationOK'] = 0
    # accountingDict['RegistrationTotal'] = 0

    accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status == "Finished" ] )
    accountingDict["TransferTotal"] = len( ftsJob )
    accountingDict["TransferSize"] = ftsJob.Size
    accountingDict["FinalStatus"] = ftsJob.Status
    accountingDict["Source"] = ftsJob.SourceSE
    accountingDict["Destination"] = ftsJob.TargetSE

    dt = ftsJob.LastUpdate - ftsJob.SubmitTime
    transferTime = dt.days * 86400 + dt.seconds
    accountingDict["TransferTime"] = transferTime
    dataOp.setValuesFromDict( accountingDict )
    dataOp.commit()

  def __sendAccounting(ftsJob):
    """ prepare and send DataOperation to AccountingDB

        :param ftsJob: the FTS3Job from which we send the accounting info
    """

    dataOp = DataOperation()
    dataOp.setStartTime(fromString(ftsJob.submitTime))
    dataOp.setEndTime(fromString(ftsJob.lastUpdate))

    dataOp.setValuesFromDict(ftsJob.accountingDict)
    dataOp.delayedCommit()

def createAccountingRecord():
  accountingDict = {}
  accountingDict['OperationType'] = 'putAndRegister'
  accountingDict['User'] = 'system'
  accountingDict['Protocol'] = 'DataManager'
  accountingDict['RegistrationTime'] = 0.0
  accountingDict['RegistrationOK'] = 0
  accountingDict['RegistrationTotal'] = 0
  accountingDict['Destination'] = 'se'
  accountingDict['TransferTotal'] = 1
  accountingDict['TransferOK'] = 1
  accountingDict['TransferSize'] = 1
  accountingDict['TransferTime'] = 0.0
  accountingDict['FinalStatus'] = 'Successful'
  accountingDict['Source'] = 'testSite'
  oDataOperation = DataOperation()
  oDataOperation.setValuesFromDict( accountingDict )
  return oDataOperation

  def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
    """ send accounting record

    :param self: self reference
    :param regSuc: number of files successfully registered
    :param regTotal: number of files attepted to register
    :param regTime: time stamp at the end of registration
    :param transEndTime: time stamp at the end of FTS job
    :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
    """

    submitTime = fromString( self.submitTime )
    oAccounting = DataOperation()
    dt = transEndTime - submitTime
    transferTime = dt.days * 86400 + dt.seconds
    if 'fts3' in self.ftsServer and transferTime < 0:
      import datetime
      while transferTime < 0:
        # Shift by one hour until transfer time is positive (ugly fix for FTS3 bug)
        transferTime += 3600
        submitTime -= datetime.timedelta( 0, 3600 )
      self.log.verbose( 'Fixed UTC submit time... Submit: %s, end: %s' % ( submitTime, transEndTime ) )
    oAccounting.setEndTime( transEndTime )
    oAccounting.setStartTime( submitTime )

    accountingDict = {}
    accountingDict['OperationType'] = 'replicateAndRegister'
    result = getProxyInfo()
    if not result['OK']:
      userName = 'system'
    else:
      userName = result['Value'].get( 'username', 'unknown' )
    accountingDict['User'] = userName
    accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
    accountingDict['RegistrationTime'] = regTime
    accountingDict['RegistrationOK'] = regSuc
    accountingDict['RegistrationTotal'] = regTotal
    accountingDict['TransferOK'] = transDict['transOK']
    accountingDict['TransferTotal'] = transDict['transTotal']
    accountingDict['TransferSize'] = transDict['transSize']
    accountingDict['FinalStatus'] = self.requestStatus
    accountingDict['Source'] = self.sourceSE
    accountingDict['Destination'] = self.targetSE
    accountingDict['TransferTime'] = transferTime
    oAccounting.setValuesFromDict( accountingDict )
    self.log.verbose( "Attempting to commit accounting message..." )
    oAccounting.commit()
    self.log.verbose( "...committed." )
    return S_OK()

 def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime ):
   transSuc = 0
   transSize = 0
   missingSize = []
   for lfn in self.fileDict.keys():
     if self.fileDict[lfn].get( 'Status' ) == 'Finished':
       transSuc += 1
       if not self.catalogMetadata.has_key( lfn ):
         missingSize.append( lfn )
   if missingSize:
     self.__updateMetadataCache( missingSize )
   for lfn in self.fileDict.keys():
     if self.fileDict[lfn].get( 'Status' ) == 'Finished':
       transSize += self.catalogMetadata[lfn]['Size']
   transTotal = 0
   for state in ( self.statusSummary.keys() ):
     transTotal += self.statusSummary[state]
   submitTime = fromString( self.submitTime )
   endTime = fromString( transEndTime )
   oAccounting = DataOperation()
   #oAccounting.setEndTime(endTime)
   oAccounting.setEndTime( transEndTime )
   oAccounting.setStartTime( submitTime )
   accountingDict = {}
   accountingDict['OperationType'] = 'replicateAndRegister'
   accountingDict['User'] = 'acsmith'
   accountingDict['Protocol'] = 'FTS'
   accountingDict['RegistrationTime'] = regTime
   accountingDict['RegistrationOK'] = regSuc
   accountingDict['RegistrationTotal'] = regTotal
   accountingDict['TransferOK'] = transSuc
   accountingDict['TransferTotal'] = transTotal
   accountingDict['TransferSize'] = transSize
   accountingDict['FinalStatus'] = self.requestStatus
   accountingDict['Source'] = self.sourceSE
   accountingDict['Destination'] = self.targetSE
   c = transEndTime - submitTime
   transferTime = c.days * 86400 + c.seconds
   accountingDict['TransferTime'] = transferTime
   oAccounting.setValuesFromDict( accountingDict )
   gLogger.verbose( "Attempting to commit accounting message..." )
   oAccounting.commit()
   gLogger.verbose( "...committed." )
   return S_OK()

  def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
    """ send accounting record

    :param self: self reference
    :param regSuc: number of files successfully registered
    :param regTotal: number of files attepted to register
    :param regTime: time stamp at the end of registration
    :param transEndTime: time stamp at the end of FTS job
    :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
    """

    oAccounting = DataOperation()
    oAccounting.setEndTime( transEndTime )
    oAccounting.setStartTime( self.submitTime )

    accountingDict = {}
    accountingDict['OperationType'] = 'replicateAndRegister'
    result = getProxyInfo()
    if not result['OK']:
      userName = 'system'
    else:
      userName = result['Value'].get( 'username', 'unknown' )
    accountingDict['User'] = userName
    accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
    accountingDict['RegistrationTime'] = regTime
    accountingDict['RegistrationOK'] = regSuc
    accountingDict['RegistrationTotal'] = regTotal
    accountingDict['TransferOK'] = transDict['transOK']
    accountingDict['TransferTotal'] = transDict['transTotal']
    accountingDict['TransferSize'] = transDict['transSize']
    accountingDict['FinalStatus'] = self.requestStatus
    accountingDict['Source'] = self.sourceSE
    accountingDict['Destination'] = self.targetSE
    accountingDict['TransferTime'] = self.transferTime
    oAccounting.setValuesFromDict( accountingDict )
    self.log.verbose( "Attempting to commit accounting message..." )
    oAccounting.commit()
    self.log.verbose( "...committed." )
    return S_OK()

  def __sendAccounting( ftsJob, ownerDN ):
    """ prepare and send DataOperation to AccouringDB """

    dataOp = DataOperation()
    dataOp.setStartTime( fromString( ftsJob.SubmitTime ) )
    dataOp.setEndTime( fromString( ftsJob.LastUpdate ) )

    accountingDict = dict()
    accountingDict["OperationType"] = "ReplicateAndRegister"

    username = getUsernameForDN( ownerDN )
    if not username["OK"]:
      username = ownerDN
    else:
      username = username["Value"]

    accountingDict["User"] = username
    accountingDict["Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower() else 'FTS'
    accountingDict['ExecutionSite'] = ftsJob.FTSServer

    accountingDict['RegistrationTime'] = ftsJob._regTime
    accountingDict['RegistrationOK'] = ftsJob._regSuccess
    accountingDict['RegistrationTotal'] = ftsJob._regTotal

    accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] )
    accountingDict["TransferTotal"] = len( ftsJob )
    accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize
    accountingDict["FinalStatus"] = ftsJob.Status
    accountingDict["Source"] = ftsJob.SourceSE
    accountingDict["Destination"] = ftsJob.TargetSE

    # dt = ftsJob.LastUpdate - ftsJob.SubmitTime
    # transferTime = dt.days * 86400 + dt.seconds
    # accountingDict["TransferTime"] = transferTime
    accountingDict['TransferTime'] = sum( [int( f._duration ) for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] )
    dataOp.setValuesFromDict( accountingDict )
    dataOp.commit()

  def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
    """ send accounting record

    :param self: self reference
    :param regSuc: number of files successfully registered
    :param regTotal: number of files attepted to register 
    :param regTime: time stamp at the end of registration 
    :param transEndTime: time stamp at the end of FTS job
    :param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers 
    """

    submitTime = fromString( self.submitTime )
    oAccounting = DataOperation()
    oAccounting.setEndTime( transEndTime )
    oAccounting.setStartTime( submitTime )

    accountingDict = {}
    accountingDict['OperationType'] = 'replicateAndRegister'
    accountingDict['User'] = 'acsmith'
    accountingDict['Protocol'] = 'FTS'
    accountingDict['RegistrationTime'] = regTime
    accountingDict['RegistrationOK'] = regSuc
    accountingDict['RegistrationTotal'] = regTotal
    accountingDict['TransferOK'] = transDict['transOK']
    accountingDict['TransferTotal'] = transDict['transTotal']
    accountingDict['TransferSize'] = transDict['transSize']
    accountingDict['FinalStatus'] = self.requestStatus
    accountingDict['Source'] = self.sourceSE
    accountingDict['Destination'] = self.targetSE
    dt = transEndTime - submitTime
    transferTime = dt.days * 86400 + dt.seconds
    accountingDict['TransferTime'] = transferTime
    oAccounting.setValuesFromDict( accountingDict )
    self.log.verbose( "Attempting to commit accounting message..." )
    oAccounting.commit()
    self.log.verbose( "...committed." )
    return S_OK()

 def __initialiseAccountingObject( self, operation, se, startTime, endTime, size ):
   accountingDict = {}
   accountingDict['OperationType'] = operation
   accountingDict['User'] = self.userName
   accountingDict['Protocol'] = 'SRM'
   accountingDict['RegistrationTime'] = 0.0
   accountingDict['RegistrationOK'] = 0
   accountingDict['RegistrationTotal'] = 0
   accountingDict['TransferTotal'] = 1
   accountingDict['TransferOK'] = 1
   accountingDict['TransferSize'] = size
   timeDiff = endTime - startTime
   transferTime = ( timeDiff.days * 86400 ) + ( timeDiff.seconds ) + ( timeDiff.microseconds / 1000000.0 )
   accountingDict['TransferTime'] = transferTime
   accountingDict['FinalStatus'] = 'Successful'
   accountingDict['Source'] = siteName()
   accountingDict['Destination'] = se
   oDataOperation = DataOperation()
   oDataOperation.setEndTime( endTime )
   oDataOperation.setStartTime( startTime )
   oDataOperation.setValuesFromDict( accountingDict )
   return oDataOperation

  def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters, callRes):
    """
        Generates a DataOperation accounting if needs to be, and adds it to the DataStore client cache

        :param lfns : list of lfns on which we attempted the operation
        :param startDate : datetime, start of the operation
        :param elapsedTime : time (seconds) the operation took
        :param storageParameters : the parameters of the plugins used to perform the operation
        :param callRes : the return of the method call, S_OK or S_ERROR

        The operation is generated with the OperationType "se.methodName"
        The TransferSize and TransferTotal for directory methods actually take into
        account the files inside the directory, and not the amount of directory given
        as parameter


    """

    if self.methodName not in (self.readMethods + self.writeMethods + self.removeMethods):
      return

    baseAccountingDict = {}
    baseAccountingDict['OperationType'] = 'se.%s' % self.methodName
    baseAccountingDict['User'] = getProxyInfo().get('Value', {}).get('username', 'unknown')
    baseAccountingDict['RegistrationTime'] = 0.0
    baseAccountingDict['RegistrationOK'] = 0
    baseAccountingDict['RegistrationTotal'] = 0

    # if it is a get method, then source and destination of the transfer should be inverted
    if self.methodName == 'getFile':
      baseAccountingDict['Destination'] = siteName()
      baseAccountingDict['Source'] = self.name
    else:
      baseAccountingDict['Destination'] = self.name
      baseAccountingDict['Source'] = siteName()

    baseAccountingDict['TransferTotal'] = 0
    baseAccountingDict['TransferOK'] = 0
    baseAccountingDict['TransferSize'] = 0
    baseAccountingDict['TransferTime'] = 0.0
    baseAccountingDict['FinalStatus'] = 'Successful'

    oDataOperation = DataOperation()
    oDataOperation.setValuesFromDict(baseAccountingDict)
    oDataOperation.setStartTime(startDate)
    oDataOperation.setEndTime(startDate + datetime.timedelta(seconds=elapsedTime))
    oDataOperation.setValueByKey('TransferTime', elapsedTime)
    oDataOperation.setValueByKey('Protocol', storageParameters.get('Protocol', 'unknown'))

    if not callRes['OK']:
      # Everything failed
      oDataOperation.setValueByKey('TransferTotal', len(lfns))
      oDataOperation.setValueByKey('FinalStatus', 'Failed')
    else:

      succ = callRes.get('Value', {}).get('Successful', {})
      failed = callRes.get('Value', {}).get('Failed', {})

      totalSize = 0
      # We don't take len(lfns) in order to make two
      # separate entries in case of few failures
      totalSucc = len(succ)

      if self.methodName in ('putFile', 'getFile'):
        # putFile and getFile return for each entry
        # in the successful dir the size of the corresponding file
        totalSize = sum(succ.values())

      elif self.methodName in ('putDirectory', 'getDirectory'):
        # putDirectory and getDirectory return for each dir name
        # a dictionnary with the keys 'Files' and 'Size'
        totalSize = sum(val.get('Size', 0) for val in succ.values() if isinstance(val, dict))
        totalSucc = sum(val.get('Files', 0) for val in succ.values() if isinstance(val, dict))
        oDataOperation.setValueByKey('TransferOK', len(succ))

      oDataOperation.setValueByKey('TransferSize', totalSize)
      oDataOperation.setValueByKey('TransferTotal', totalSucc)
      oDataOperation.setValueByKey('TransferOK', totalSucc)

      if callRes['Value']['Failed']:
        oDataOperationFailed = copy.deepcopy(oDataOperation)
        oDataOperationFailed.setValueByKey('TransferTotal', len(failed))
        oDataOperationFailed.setValueByKey('TransferOK', 0)
        oDataOperationFailed.setValueByKey('TransferSize', 0)
        oDataOperationFailed.setValueByKey('FinalStatus', 'Failed')

        accRes = gDataStoreClient.addRegister(oDataOperationFailed)
        if not accRes['OK']:
          self.log.error("Could not send failed accounting report", accRes['Message'])

    accRes = gDataStoreClient.addRegister(oDataOperation)
    if not accRes['OK']:
      self.log.error("Could not send accounting report", accRes['Message'])

  def __monitorStorageElementStageRequests( self, storageElement, seReplicaIDs, replicaIDs ):
    terminalReplicaIDs = {}
    oldRequests = []
    stagedReplicas = []
    pfnRepIDs = {}
    pfnReqIDs = {}
    for replicaID in seReplicaIDs:
      pfn = replicaIDs[replicaID]['PFN']
      pfnRepIDs[pfn] = replicaID
      requestID = replicaIDs[replicaID].get( 'RequestID', None )
      if requestID:
        pfnReqIDs[pfn] = replicaIDs[replicaID]['RequestID']

    gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: Monitoring %s stage requests for %s." % ( len( pfnRepIDs ), storageElement ) )
    oAccounting = DataOperation()
    oAccounting.setStartTime()

    res = self.replicaManager.getStorageFileMetadata( pfnReqIDs.keys(), storageElement )
    if not res['OK']:
      gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Completely failed to monitor stage requests for replicas.", res['Message'] )
      return
    prestageStatus = res['Value']

    accountingDict = self.__newAccountingDict( storageElement )

    for pfn, reason in prestageStatus['Failed'].items():
      accountingDict['TransferTotal'] += 1
      if re.search( 'File does not exist', reason ):
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: PFN did not exist in the StorageElement", pfn )
        terminalReplicaIDs[pfnRepIDs[pfn]] = 'PFN did not exist in the StorageElement'
    for pfn, staged in prestageStatus['Successful'].items():
      if staged and 'Cached' in staged and staged['Cached']:
        accountingDict['TransferTotal'] += 1
        accountingDict['TransferOK'] += 1
        accountingDict['TransferSize'] += staged['Size']
        stagedReplicas.append( pfnRepIDs[pfn] )
      if staged and 'Cached' in staged and not staged['Cached']:
        oldRequests.append( pfnRepIDs[pfn] ); #only ReplicaIDs

    oAccounting.setValuesFromDict( accountingDict )
    oAccounting.setEndTime()
    gDataStoreClient.addRegister( oAccounting )

    # Update the states of the replicas in the database
    if terminalReplicaIDs:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s replicas are terminally failed." % len( terminalReplicaIDs ) )
      res = self.stagerClient.updateReplicaFailure( terminalReplicaIDs )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to update replica failures.", res['Message'] )
    if stagedReplicas:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s staged replicas to be updated." % len( stagedReplicas ) )
      res = self.stagerClient.setStageComplete( stagedReplicas )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to updated staged replicas.", res['Message'] )
      res = self.stagerClient.updateReplicaStatus( stagedReplicas, 'Staged' )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to insert replica status.", res['Message'] )
    if oldRequests:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s old requests will be retried." % len( oldRequests ) )
      res = self.__wakeupOldRequests( oldRequests )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to wakeup old requests.", res['Message'] )
    return

  def __monitorStorageElementStageRequests( self, storageElement, seReplicaIDs, replicaIDs ):
    terminalReplicaIDs = {}
    oldRequests = []
    stagedReplicas = []

    # Since we are in a given SE, the LFN is a unique key
    lfnRepIDs = {}
    for replicaID in seReplicaIDs:
      lfn = replicaIDs[replicaID]['LFN']
      lfnRepIDs[lfn] = replicaID

    if lfnRepIDs:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: Monitoring %s stage requests for %s." % ( len( lfnRepIDs ),
                                                                                                                  storageElement ) )
    else:
      gLogger.warn( "StageMonitor.__monitorStorageElementStageRequests: No requests to monitor for %s." % storageElement )
      return
    oAccounting = DataOperation()
    oAccounting.setStartTime()

    res = StorageElement( storageElement ).getFileMetadata( lfnRepIDs )
    if not res['OK']:
      gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Completely failed to monitor stage requests for replicas.", res['Message'] )
      return
    prestageStatus = res['Value']

    accountingDict = self.__newAccountingDict( storageElement )

    for lfn, reason in prestageStatus['Failed'].iteritems():
      accountingDict['TransferTotal'] += 1
      if re.search( 'File does not exist', reason ):
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: LFN did not exist in the StorageElement", lfn )
        terminalReplicaIDs[lfnRepIDs[lfn]] = 'LFN did not exist in the StorageElement'
    for lfn, metadata in prestageStatus['Successful'].iteritems():
      if not metadata:
        continue
      staged = metadata.get( 'Cached', metadata['Accessible'] )
      if staged:
        accountingDict['TransferTotal'] += 1
        accountingDict['TransferOK'] += 1
        accountingDict['TransferSize'] += metadata['Size']
        stagedReplicas.append( lfnRepIDs[lfn] )
      elif staged is not None:
        oldRequests.append( lfnRepIDs[lfn] )  # only ReplicaIDs

    oAccounting.setValuesFromDict( accountingDict )
    oAccounting.setEndTime()
    gDataStoreClient.addRegister( oAccounting )

    # Update the states of the replicas in the database
    if terminalReplicaIDs:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s replicas are terminally failed." % len( terminalReplicaIDs ) )
      res = self.stagerClient.updateReplicaFailure( terminalReplicaIDs )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to update replica failures.", res['Message'] )
    if stagedReplicas:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s staged replicas to be updated." % len( stagedReplicas ) )
      res = self.stagerClient.setStageComplete( stagedReplicas )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to updated staged replicas.", res['Message'] )
      res = self.stagerClient.updateReplicaStatus( stagedReplicas, 'Staged' )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to insert replica status.", res['Message'] )
    if oldRequests:
      gLogger.info( "StageMonitor.__monitorStorageElementStageRequests: %s old requests will be retried." % len( oldRequests ) )
      res = self.__wakeupOldRequests( oldRequests )
      if not res['OK']:
        gLogger.error( "StageMonitor.__monitorStorageElementStageRequests: Failed to wakeup old requests.", res['Message'] )
    return