Python DataManager.DataManager类代码示例

  def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
    """Upload the new processList to the FileCatalog
    """
    from ILCDIRAC.Core.Utilities.FileUtils                       import upload
    from DIRAC.DataManagementSystem.Client.DataManager           import DataManager
    from DIRAC import gConfig, exit as dexit

    datMan = DataManager()
    LOG.notice("Removing process list from file catalog" + path_to_process_list)
    res = datMan.removeFile(path_to_process_list)
    if not res['OK']:
      LOG.error("Could not remove process list from file catalog, do it by hand")
      dexit(2)
    LOG.notice("Done removing process list from file catalog")

    res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
    if not res['OK']:
      LOG.error("something went wrong in the copy")
      dexit(2)

    LOG.notice("Putting process list to local processlist directory")
    localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
    if localprocesslistpath['Value']:

      try:
        localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
        shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
      except OSError, err:
        LOG.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))

      try:
        subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
      except OSError, err:
        LOG.error("Commit failed! Error: %s" % str(err))

def getFile(lfn, se=""):
    dm = DataManager()

    download_ok = 0
    get_active_replicas_ok = False
    lfn_on_se = False
    error_msg = ""
    if se:
        for i in range(0, 5):
            result = dm.getActiveReplicas(lfn)
            if result["OK"] and result["Value"]["Successful"]:
                get_active_replicas_ok = True
                lfnReplicas = result["Value"]["Successful"]
                if se in lfnReplicas[lfn]:
                    lfn_on_se = True
                    break
            time.sleep(3)
            print "- Get replicas for %s failed, try again" % lfn

        if not get_active_replicas_ok:
            return S_ERROR("Get replicas error: %s" % lfn)

    if lfn_on_se:
        se = StorageElement(se)
        # try 5 times
        for j in range(0, 5):
            result = se.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- %s getStorageFile(%s) failed, try again" % (lfn, se)
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 1
            else:
                error_msg = "Downloading %s from SE %s error!" % (lfn, se)
        else:
            error_msg = result["Message"]
    else:
        if se:
            print 'File %s not found on SE "%s" after %s tries, trying other SE' % (lfn, se, i + 1)
        # try 5 times
        for j in range(0, 5):
            result = dm.getFile(lfn)
            if result["OK"] and result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                break
            time.sleep(random.randint(180, 600))
            print "- getFile(%s) failed, try again" % lfn
        if result["OK"]:
            if result["Value"]["Successful"] and result["Value"]["Successful"].has_key(lfn):
                download_ok = 2
            else:
                error_msg = "Downloading %s from random SE error!" % lfn
        else:
            error_msg = result["Message"]

    if download_ok:
        return S_OK({lfn: {"DownloadOK": download_ok, "Retry": j + 1}})

    return S_ERROR(error_msg)

 def _replicate(self, lfn, destinationSE, sourceSE="", localCache=""):
     dm = DataManager()
     result = dm.replicateAndRegister(lfn, destinationSE, sourceSE, '', localCache)
     if not result['OK']:
         print 'ERROR %s' % (result['Message'])
         return result
     else:
         return S_OK(result['Value']['Successful'][lfn])

def doesFileExist(lfn):
    from DIRAC.DataManagementSystem.Client.DataManager import DataManager
    dm = DataManager()
    result = dm.getActiveReplicas(lfn)
    if result[('Value')][('Successful')]:
        return True
    else:
        return False

 def _remove_file(self, lfn):
     dm = DataManager()
     res = dm.removeFile([lfn])
     if not res['OK']:
         gLogger.error("Failed to remove data", res['Message'])
         return res
     if lfn in res['Value']['Successful']:
         return S_OK(res['Value']['Successful'])
     return S_ERROR(res['Value']['Failed'])

    def _get_file(self, lfn):
        dm = DataManager()
        result = dm.getFile(lfn, "")
        if not result['OK']:
            return S_ERROR(result['Message'])

        if result['Value']['Failed']:
            return S_ERROR(result['Value'])
        return result

  def __call__( self ):
    """ call me maybe """
    # # counter for failed files
    failedFiles = 0
    # # catalog(s) to use
    catalogs = self.operation.Catalog
    if catalogs:
      catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
    dm = DataManager( catalogs = catalogs )
    # # get waiting files
    waitingFiles = self.getWaitingFilesList()
    # # loop over files
    for opFile in waitingFiles:

      gMonitor.addMark( "RegisterAtt", 1 )

      # # get LFN
      lfn = opFile.LFN
      # # and others
      fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
      # # call DataManager
      registerFile = dm.registerFile( fileTuple )
      # # check results
      if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:

        gMonitor.addMark( "RegisterFail", 1 )
#        self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )

        reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
        errorStr = "failed to register LFN"
        opFile.Error = "%s: %s" % ( errorStr, reason )
        if 'GUID already registered' in reason:
          opFile.Status = 'Failed'
          self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
        elif 'File already registered with no replicas' in reason:
          self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
          dm.removeFile( lfn )
        else:
          self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
        failedFiles += 1

      else:

        gMonitor.addMark( "RegisterOK", 1 )
#        self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )

        self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
        opFile.Status = "Done"

    # # final check
    if failedFiles:
      self.log.warn( "all files processed, %s files failed to register" % failedFiles )
      self.operation.Error = "some files failed to register"
      return S_ERROR( self.operation.Error )

    return S_OK()

  def setUp( self ):
    super( IntegrationTest, self ).setUp()

    dm = DataManager()
    res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
                          '/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
                        force = True )
    if not res['OK']:
      print "Could not remove files", res['Message']
      exit( 1 )

def doesFileExist(lfn):
    from DIRAC.DataManagementSystem.Client.DataManager import DataManager
    dm = DataManager()
    result = dm.getActiveReplicas(lfn)
    if not result['OK']:
        print "ERROR",result['Message']
        return False
    if lfn in result['Value']['Successful']:
        return True
    else:
        return False

    def _add_file(self, lfn, localfile, SE, guid=None):
        dm = DataManager()
        self._create_test_file()
        if not os.path.exists(self.options['test_file']):
            gLogger.error("File %s must exist locally" % localfile)
        if not os.path.isfile(self.options['test_file']):
            gLogger.error("%s is not a file" % localfile)

        res = dm.putAndRegister(lfn, localfile, SE, guid)
        if not res['OK']:
            gLogger.error('Error: failed to upload %s to %s' % (lfn, SE))
            return S_ERROR(res['Message'])
        return S_OK(res['Value']['Successful'][lfn])

def removeOutputData(baseDir, logLevel="INFO"):
    gLogger.setLevel(logLevel)
    res = getProxyInfo(False, False)
    if not res["OK"]:
        gLogger.error("Failed to get client proxy information.", res["Message"])
        return {"OK": False, "Message": "Failed to get client proxy information: %s" % str(res["Message"]), "RC": 71}

    # ######################################################################################################## #
    rm = DataManager()
    try:
        result = rm.cleanLogicalDirectory(baseDir)
    except KeyError, ke:
        return {"OK": False, "Message": "Caught key error, full stacktrace below\n%s" % str(ke), "RC": 137}

    def setUp(self):
        super(FailingUserJobTestCase, self).setUp()

        dm = DataManager()
        res = dm.removeFile(
            [
                "/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar",
                "/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim",
            ],
            force=True,
        )
        if not res["OK"]:
            print "Could not remove files", res["Message"]
            exit(1)

def getFilesToStage( lfnList ):
  """ Utility that returns out of a list of LFNs those files that are offline,
      and those for which at least one copy is online
  """
  onlineLFNs = set()
  offlineLFNsDict = {}

  if not lfnList:
    return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
  
  dm = DataManager()

  lfnListReplicas = dm.getActiveReplicas( lfnList )
  if not lfnListReplicas['OK']:
    return lfnListReplicas

  seObjectsDict = dict()
  seToLFNs = dict()
  
  if lfnListReplicas['Value']['Failed']:
    return S_ERROR( "Failures in getting replicas" )
  for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
    for se, _ in ld.iteritems():
      seObjectsDict.setdefault( se, StorageElement( se ) )
      seToLFNs.setdefault( se, list() ).append( lfn )

  for se, lfnsInSEList in seToLFNs.iteritems():
    fileMetadata = seObjectsDict[se].getFileMetadata( lfnsInSEList )
    if not fileMetadata['OK']:
      return fileMetadata

    if fileMetadata['Value']['Failed']:
      return S_ERROR( "Failures in getting file metadata" )
    # is there at least one online?
    for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
      if mDict['Cached']:
        onlineLFNs.add( lfn )

  offlineLFNs = set( lfnList ).difference( onlineLFNs )
  

  for offlineLFN in offlineLFNs:
    ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
    random.shuffle( ses )
    se = ses[0]
    offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
  
  return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )

 def __init__(self):
   super(WhizardAnalysis, self).__init__()
   self.enable = True
   self.STEP_NUMBER = ''
   self.debug = True
   self.SteeringFile = ''
   self.OutputFile = ''
   self.NumberOfEvents = 1
   self.Lumi = 0
   self.applicationName = 'whizard'
   self.evttype = ""
   self.RandomSeed = 0
   self.getProcessInFile = False
   self.datMan = DataManager()
   self.processlist = None
   self.parameters = {}
   self.susymodel = 0
   self.Model = ''
   self.genmodel = GeneratorModels()
   self.eventstring = ['! ', 'Fatal error:', 'PYSTOP', 'No matrix element available',
                       'Floating point exception', 'Event generation finished.', " n_events","luminosity", 
                       "  sum            "]
   self.excludeAllButEventString = False
   self.steeringparameters = ''
   self.options = None
   self.optionsdict = {}
   self.OptionsDictStr = ''
   self.GenLevelCutDictStr = ''
   self.genlevelcuts = {}
   self.willCut = False
   self.useGridFiles = False

 def __init__(self):
   super(OverlayInput, self).__init__()
   self.enable = True
   self.STEP_NUMBER = ''
   self.applicationName = 'OverlayInput'
   self.curdir = os.getcwd()
   self.applicationLog = ''
   self.printoutflag = ''
   self.prodid = 0
   self.detector = '' ##needed for backward compatibility
   self.detectormodel = ""
   self.energytouse = ''
   self.energy = 0
   self.nbofeventsperfile = 100
   self.lfns = []
   self.nbfilestoget = 0
   self.BkgEvtType = 'gghad'
   self.metaEventType = self.BkgEvtType
   self.BXOverlay = 0
   self.ggtohadint = 3.2
   self.nbsigeventsperfile = 0
   self.nbinputsigfile = 1
   self.NbSigEvtsPerJob = 0
   self.datMan = DataManager()
   self.fcc = FileCatalogClient()
   self.site = DIRAC.siteName()
   self.useEnergyForFileLookup = True
   self.machine = 'clic_cdr'
   self.pathToOverlayFiles = ''
   self.processorName = ''