Python FileCatalogClient.findFilesByMetadata方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
  def _getProdInfoFromIDs(self):
    """get the processName, energy and eventsPerJob from the MetaData catalog

    :raises: AttributeError if some of the information cannot be found
    :returns: None
    """
    if not self.prodIDs:
      raise AttributeError("No prodIDs defined")

    self.eventsPerJobs = []
    self.processes = []
    self.energies = []
    from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
    from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    trc = TransformationClient()
    fc = FileCatalogClient()
    for prodID in self.prodIDs:
      gLogger.notice("Getting information for %s" % prodID)
      tRes = trc.getTransformation(str(prodID))
      if not tRes['OK']:
        raise AttributeError("No prodInfo found for %s" % prodID)
      self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask']))
      lfnRes = fc.findFilesByMetadata({'ProdID': prodID})
      if not lfnRes['OK'] or not lfnRes['Value']:
        raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value'))))
      path = os.path.dirname(lfnRes['Value'][0])
      fileRes = fc.getDirectoryUserMetadata(path)
      self.processes.append(fileRes['Value']['EvtType'])
      self.energies.append(fileRes['Value']['Energy'])
      gLogger.notice("Found (Evts,Type,Energy): %s %s %s " %
                     (self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
def getFiles():
  """ Get the lfns: This is not the point of this example, so keep it out of the main
  """
  fc = FileCatalogClient()
  
  meta = {}
  meta['ProdID'] = 1543
  meta["Datatype"] = "DST"
  
  result = fc.findFilesByMetadata(meta, "/ilc/prod/clic")
  if not result["OK"]:
    gLogger.error(result["Message"])
    dexit(1)
  return result['Value']

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
def _findInFC():
  """Find something in the FileCatalog"""
  from DIRAC import exit as dexit
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  args = Script.getPositionalArgs()
  if len(args)<2:
    Script.showHelp('ERROR: Not enough arguments')
    gLogger.error("Run %s --help" % SCRIPTNAME )
    dexit(1)
    
  path = args[0]
  if path == '.':
    path = '/'

  ## Check that the first argument is not a MetaQuery
  if any( op in path for op in OPLIST ):
    gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path)
    gLogger.error("Run %s --help" % SCRIPTNAME )
    dexit(1)

  gLogger.verbose("Path:", path)
  metaQuery = args[1:]
  metaDataDict = _createQueryDict(metaQuery)
  gLogger.verbose("Query:",str(metaDataDict))
  if not metaDataDict:
    gLogger.info("No query")
    dexit(1)
  
  fc = FileCatalogClient()
  res = fc.findFilesByMetadata(metaDataDict, path)
  if not res['OK']:
    gLogger.error(res['Message'])
    dexit(1)
  if not res['Value']:
    gLogger.notice("No files found")

  listToPrint = None

  if clip.printOnlyDirectories:
    listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] )
  else:
    listToPrint = res['Value']

  for entry in listToPrint:
    print entry

  dexit(0)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]

#.........这里部分代码省略.........
    directories = res['Value']
    # Clean the jobs in the WMS and any failover requests found
    res = self.cleanTransformationTasks(transID)
    if not res['OK']:
      return res
    # Clean the log files for the jobs
    for directory in directories:
      if re.search('/LOG/', directory):
        res = self.cleanTransformationLogFiles(directory)
        if not res['OK']:
          return res
      res = self.cleanContent(directory)
      if not res['OK']:
        return res

    # Clean ALL the possible remnants found
    res = self.cleanMetadataCatalogFiles(transID)
    if not res['OK']:
      return res
    # Clean the transformation DB of the files and job information
    res = self.transClient.cleanTransformation(transID)
    if not res['OK']:
      return res
    self.log.info("Successfully cleaned transformation %d" % transID)
    res = self.transClient.setTransformationParameter(transID, 'Status', 'Cleaned')
    if not res['OK']:
      self.log.error("Failed to update status of transformation %s to Cleaned" % (transID), res['Message'])
      return res
    self.log.info("Updated status of transformation %s to Cleaned" % (transID))
    return S_OK()

  def cleanMetadataCatalogFiles(self, transID):
    """ wipe out files from catalog """
    res = self.metadataClient.findFilesByMetadata({self.transfidmeta: transID})
    if not res['OK']:
      return res
    fileToRemove = res['Value']
    if not fileToRemove:
      self.log.info('No files found for transID %s' % transID)
      return S_OK()

    # Executing with shifter proxy
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'false')
    res = DataManager().removeFile(fileToRemove, force=True)
    gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', 'true')

    if not res['OK']:
      return res
    for lfn, reason in res['Value']['Failed'].items():
      self.log.error("Failed to remove file found in metadata catalog", "%s %s" % (lfn, reason))
    if res['Value']['Failed']:
      return S_ERROR("Failed to remove all files found in the metadata catalog")
    self.log.info("Successfully removed all files found in the BK")
    return S_OK()

  #############################################################################
  #
  # These are the methods for removing the jobs from the WMS and transformation DB
  #

  def cleanTransformationTasks(self, transID):
    """ clean tasks from WMS, or from the RMS if it is a DataManipulation transformation
    """
    self.log.verbose("Cleaning Transformation tasks of transformation %d" % transID)
    res = self.__getTransformationExternalIDs(transID)
    if not res['OK']:

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
   continue
 prodtype = res['Value']['Type']
 proddetail = res['Value']['Description']
 if prodtype == 'MCReconstruction' or prodtype == 'MCReconstruction_Overlay' :
   meta['Datatype']='DST'
 elif prodtype == 'MCGeneration':
   meta['Datatype']='gen'
 elif prodtype == 'MCSimulation':
   meta['Datatype']='SIM'
 elif prodtype in ['Split','Merge']:
   gLogger.warn("Invalid query for %s productions" % prodtype)
   continue
 else:
   gLogger.error("Unknown production type %s"% prodtype)
   continue
 res = fc.findFilesByMetadata(meta)  
 if not res['OK']:
   gLogger.error(res['Message'])
   continue
 lfns = res['Value']
 nb_files = len(lfns)
 path = ""
 if not len(lfns):
   gLogger.warn("No files found for prod %s" % prodID)
   continue
 path = os.path.dirname(lfns[0])
 res = fc.getDirectoryMetadata(path)
 if not res['OK']:
   gLogger.warn('No meta data found for %s' % path)
   continue
 dirmeta = {}

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]

#.........这里部分代码省略.........
    res = self.cleanTransformationTasks( transID )
    if not res['OK']:
      return res
    # Clean the log files for the jobs
    for directory in directories:
      if re.search( '/LOG/', directory ):
        res = self.cleanTransformationLogFiles( directory )
        if not res['OK']:
          return res
      res = self.cleanCatalogContents( directory )
      if not res['OK']:
        return res
      res = self.cleanStorageContents( directory )
      if not res['OK']:
        return res
    # Clean ALL the possible remnants found in the BK
    res = self.cleanMetadataCatalogFiles( transID, directories )
    if not res['OK']:
      return res
    # Clean the transformation DB of the files and job information
    res = self.transClient.cleanTransformation( transID )
    if not res['OK']:
      return res
    gLogger.info( "Successfully cleaned transformation %d" % transID )
    # Change the status of the transformation to deleted
    res = self.transClient.setTransformationParameter( transID, 'Status', 'Deleted' )
    if not res['OK']:
      gLogger.error( "Failed to update status of transformation %s to Deleted" % ( transID ), res['Message'] )
      return res
    gLogger.info( "Updated status of transformation %s to Deleted" % ( transID ) )
    return S_OK()

  def cleanMetadataCatalogFiles( self, transID, directories ):
    res = self.metadataClient.findFilesByMetadata( {self.transfidmeta:transID} )
    if not res['OK']:
      return res
    fileToRemove = res['Value']
    if not len(fileToRemove):
      gLogger.info('No files found for transID %s'%transID)
      return S_OK()
    res = self.replicaManager.removeFile( fileToRemove )
    if not res['OK']:
      return res
    for lfn, reason in res['Value']['Failed'].items():
      gLogger.error( "Failed to remove file found in metadata catalog", "%s %s" % ( lfn, reason ) )
    if res['Value']['Failed']:
      return S_ERROR( "Failed to remove all files found in the metadata catalog" )
    gLogger.info( "Successfully removed all files found in the BK" )
    return S_OK()

  #############################################################################
  #
  # These are the methods for removing the jobs from the WMS and transformation DB
  #

  def cleanTransformationTasks( self, transID ):
    res = self.__getTransformationExternalIDs( transID )
    if not res['OK']:
      return res
    externalIDs = res['Value']
    if externalIDs:
      res = self.transClient.getTransformationParameters( transID, ['Type'] )
      if not res['OK']:
        gLogger.error( "Failed to determine transformation type" )
        return res
      transType = res['Value']

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
    eventType = eventSelection['EventType']
    detectorModel = eventSelection['DetectorModel']
    reconstructionVariant = eventSelection['ReconstructionVariant']
    analysisTag = eventSelection['AnalysisTag']

    for energy in eventSelection['Energies']:
        localPath = '/r06/lc/sg568/' + jobDescription + '/MarlinJobs/Detector_Model_' + str(detectorModel) + '/Reconstruction_Variant_' + str(reconstructionVariant) + '/' + eventType + '/' + str(energy) + 'GeV/AnalysisTag' + str(analysisTag)
        if not os.path.exists(localPath):
            os.makedirs(localPath)

        gridPath = '/ilc/user/s/sgreen/' + jobDescription + '/MarlinJobs/Detector_Model_' + str(detectorModel) + '/Reconstruction_Variant_' + str(reconstructionVariant) + '/' + eventType + '/' + str(energy) + 'GeV/AnalysisTag' + str(analysisTag)

        meta = {}
        meta['Owner'] = 'sgreen'

        res = fc.findFilesByMetadata(meta, gridPath)
        if not res['OK']:
            print res['Message']

        lfns = res['Value']
        fileFormat = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenN_(\d+)_(\d+)_(\d+)_Tag' + str(analysisTag) + '(.*?).root'

        for lfn in lfns:
            matchObj = re.match(fileFormat, localFile, re.M|re.I)
            if not matchObj:
                continue

            fineLocalPath = os.path.join(localPath, matchObj.group(4))
            localFile = os.path.basename(lfn)
            localFilePath = os.path.join(fineLocalPath, localFile)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
    # Configure the logging.
    lg.basicConfig(filename='log_perform_cluster_query.log', filemode='w', level=level)

    print("*")
    print("* Input JSON          : '%s'" % (datapath))
    print("* Output path         : '%s'" % (outputpath))
    print("* DFC base directory  : '%s'" % (dfc_base))

    ## The DFC client.
    fc = FileCatalogClient()

    ## The frame query JSON file - FIXME: check it exists...
    qf = open(datapath, "r")
    #
    qd = json.load(qf)
    qf.close()

    meta_dict = {\
        "size" : { ">=" : int(qd[0]["size_min"]) },
        }

    ## The query result.
    result = fc.findFilesByMetadata(meta_dict, path=dfc_base)

    print("*")
    print "* Metadata query:", meta_dict
    print("*")

    #print result
    print("* Number of clusters found      : %d" % len(result["Value"]))

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
      jdict['files'].append(item['file'])
      startfromevt = int(cur_events)
      cur_events = 0
      mylist.remove(item)
      joblist.append(jdict)
      continue
    if cur_events > nb_evts_in_file:
      if not 'files' in jdict:
        jdict['files'] = []
      jdict['files'].append(item['file'])
      mylist.remove(item)
      continue
    #joblist.append(jdict)
    #print "final jdict", jdict
  return S_OK(joblist)

if __name__=="__main__":
  from DIRAC.Core.Base import Script
  Script.parseCommandLine()
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()

  res = fc.findFilesByMetadata({"ProdID":1978})
  if not res['OK']:
    print res['Message']
    exit(1)
  lfns = res['Value']
  lfns.sort()
  res = SplitByFilesAndEvents(lfns,70)
  print res['Value'][1]

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
    if not result['OK']:
        print "Error looking up the file catalog for metadata"
        sys.exit(2)
    metaData = result['Value']
    print '  Found production ID %s. Associated meta data:'%( prodID )
    for key, value in metaData.iteritems():
        if not len(value) == 0:
            print '    %s: %s'%( key, value[0] )
    if metaData['Datatype'][0] == 'SIM':
        slicVer = ''        # data has been simulated, can skip SLIC step
    if metaData['EvtType'] and not process:
        process = metaData['EvtType'][0]
    if slicVer and metaData['NumberOfEvents']:
        if nEvts < 1:
            nEvts = int(metaData['NumberOfEvents'][0])
    result = fileCatalog.findFilesByMetadata( meta, lfnProdPath )
    if not result['OK']:
        print "Error looking up the file catalog for metadata"
        sys.exit(2)
    lfnlist = map(lambda x: "LFN:"+x, result['Value'])    # need to add "LFN:" to each entry
    print '  Found %s files associated with meta data'%( len(lfnlist) )
    print ''
    repositoryFile += 'prod'+prodID+'.cfg'
elif inputFileList:     # read a file containing an lfnlist
    repositoryFile += inputFileList.split('/')[-1].replace('.py','.cfg')
    f = open( inputFileList, 'r')
    exec(f.read())
    if not lfnlist:
        print "Error no lfnlist in %s"%(inputFileList)
        sys.exit(2)
elif lfnFile:

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
    ## The frame query JSON file - FIXME: check it exists...
    qf = open(datapath, "r")
    #
    qd = json.load(qf)
    qf.close()

    meta_dict = {\
        "start_time" : { ">=" : int(qd[0]["start_time"]) },
        "end_time"   : { "<=" : int(qd[0]["end_time"  ]) }
#        #"lat" : { ">" : 60.0 }\
#        #"n_pixel" : { ">" : 700 }\
#        #"n_kluster" : { ">" : 40}\
        }

    ## The query result.
    result = fc.findFilesByMetadata(meta_dict, path=dfc_query_path)

    print("*")
    print "* Metadata query:", meta_dict
    print("*")

    ## The frames retrieved from the metadata query.
    retrieved_frames = ['LFN:%s' % fn for fn in sorted(result["Value"])]
    #
    if len(retrieved_frames) == 0:
        raise IOError("* ERROR: no frames found. Quitting.")

    # Get the expected cluster file names from metadata interface.

    ## Kluster file names.
    kluster_file_names = []

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]
def _getProductionSummary():
  clip = _Params()
  clip.registerSwitch()
  Script.parseCommandLine()
  from ILCDIRAC.Core.Utilities.HTML                             import Table
  from ILCDIRAC.Core.Utilities.ProcessList                      import ProcessList
  from DIRAC.TransformationSystem.Client.TransformationClient   import TransformationClient
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC import gConfig, gLogger
  prod = clip.prod
  full_detail = clip.full_det
  fc = FileCatalogClient()
  
  processlist = gConfig.getValue('/LocalSite/ProcessListPath')
  prl = ProcessList(processlist)
  processesdict = prl.getProcessesDict()
  
  trc = TransformationClient()
  prodids = []
  if not prod:
    conddict = {}
    conddict['Status'] = clip.statuses
    if clip.ptypes:
      conddict['Type'] = clip.ptypes
    res = trc.getTransformations( conddict )
    if res['OK']:
      for transfs in res['Value']:
        prodids.append(transfs['TransformationID'])
  else:
    prodids.extend(prod)

  metadata = []
  
  gLogger.info("Will run on prods %s" % str(prodids))
  
  for prodID in prodids:
    if prodID<clip.minprod:
      continue
    meta = {}
    meta['ProdID']=prodID
    res = trc.getTransformation(str(prodID))
    if not res['OK']:
      gLogger.error("Error getting transformation %s" % prodID )
      continue
    prodtype = res['Value']['Type']
    proddetail = res['Value']['Description']
    if prodtype == 'MCReconstruction' or prodtype == 'MCReconstruction_Overlay' :
      meta['Datatype']='DST'
    elif prodtype == 'MCGeneration':
      meta['Datatype']='gen'
    elif prodtype == 'MCSimulation':
      meta['Datatype']='SIM'
    elif prodtype in ['Split','Merge']:
      gLogger.warn("Invalid query for %s productions" % prodtype)
      continue
    else:
      gLogger.error("Unknown production type %s"% prodtype)
      continue
    res = fc.findFilesByMetadata(meta)  
    if not res['OK']:
      gLogger.error(res['Message'])
      continue
    lfns = res['Value']
    nb_files = len(lfns)
    path = ""
    if not len(lfns):
      gLogger.warn("No files found for prod %s" % prodID)
      continue
    path = os.path.dirname(lfns[0])
    res = fc.getDirectoryUserMetadata(path)
    if not res['OK']:
      gLogger.warn('No meta data found for %s' % path)
      continue
    dirmeta = {}
    dirmeta['proddetail'] = proddetail
    dirmeta['prodtype'] = prodtype
    dirmeta['nb_files']=nb_files
    dirmeta.update(res['Value'])
    lumi  = 0.
    nbevts = 0
    addinfo = None
    files = 0
    xsec = 0.0
    if not full_detail:
      lfn  = lfns[0]
      info = _getFileInfo(lfn)
      nbevts = info[1]*len(lfns)
      lumi = info[0]*len(lfns)
      addinfo = info[2]
      if 'xsection' in addinfo:
        if 'sum' in addinfo['xsection']:
          if 'xsection' in addinfo['xsection']['sum']:
            xsec += addinfo['xsection']['sum']['xsection']
            files += 1
    else:
      for lfn in lfns:
        info = _getFileInfo(lfn)
        lumi += info[0]
        nbevts += info[1]
        addinfo = info[2]
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]

#.........这里部分代码省略.........
    res = self.cleanTransformationTasks( transID )
    if not res['OK']:
      return res
    # Clean the log files for the jobs
    for directory in directories:
      if re.search( '/LOG/', directory ):
        res = self.cleanTransformationLogFiles( directory )
        if not res['OK']:
          return res
      res = self.cleanCatalogContents( directory )
      if not res['OK']:
        return res
      res = self.cleanStorageContents( directory )
      if not res['OK']:
        return res
    # Clean ALL the possible remnants found in the BK
    res = self.cleanMetadataCatalogFiles( transID )
    if not res['OK']:
      return res
    # Clean the transformation DB of the files and job information
    res = self.transClient.cleanTransformation( transID )
    if not res['OK']:
      return res
    self.log.info( "Successfully cleaned transformation %d" % transID )
    res = self.transClient.setTransformationParameter( transID, 'Status', 'Cleaned' )
    if not res['OK']:
      self.log.error( "Failed to update status of transformation %s to Cleaned" % ( transID ), res['Message'] )
      return res
    self.log.info( "Updated status of transformation %s to Cleaned" % ( transID ) )
    return S_OK()

  def cleanMetadataCatalogFiles( self, transID ):
    """ wipe out files from catalog """
    res = self.metadataClient.findFilesByMetadata( { self.transfidmeta : transID } )
    if not res['OK']:
      return res
    fileToRemove = res['Value']
    if not fileToRemove:
      self.log.info( 'No files found for transID %s' % transID )
      return S_OK()

    # Executing with shifter proxy
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
    res = DataManager().removeFile( fileToRemove, force = True )
    gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )

    if not res['OK']:
      return res
    for lfn, reason in res['Value']['Failed'].items():
      self.log.error( "Failed to remove file found in metadata catalog", "%s %s" % ( lfn, reason ) )
    if res['Value']['Failed']:
      return S_ERROR( "Failed to remove all files found in the metadata catalog" )
    self.log.info( "Successfully removed all files found in the BK" )
    return S_OK()

  #############################################################################
  #
  # These are the methods for removing the jobs from the WMS and transformation DB
  #

  def cleanTransformationTasks( self, transID ):
    """ clean tasks from WMS, or from the RMS if it is a DataManipulation transformation
    """
    res = self.__getTransformationExternalIDs( transID )
    if not res['OK']:
      return res

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]

#.........这里部分代码省略.........
    #res = self.fc.getCompatibleMetadata(meta)
    #if not res['OK']:
    #  return res
    #compatmeta = res['Value']
    #if not self.prodid:
    #  if compatmeta.has_key('ProdID'):
    #    #take the latest prodID as
    #    list = compatmeta['ProdID']
    #    list.sort()
    #    self.prodid=list[-1]
    #  else:
    #    return S_ERROR("Could not determine ProdID from compatible metadata")
    #meta['ProdID']=self.prodid
    #refetch the compat metadata to get nb of events

    #res = self.fc.getCompatibleMetadata(meta)
    #if not res['OK']:
    #  return res
    #compatmeta = res['Value']
    #if compatmeta.has_key('NumberOfEvents'):
    #  if type(compatmeta['NumberOfEvents'])==type([]):
    #    self.nbofeventsperfile = compatmeta['NumberOfEvents'][0]
    #  elif type(compatmeta['NumberOfEvents']) in types.StringTypes:
    #    self.nbofeventsperfile = compatmeta['NumberOfEvents']
    #else:
    #  return S_ERROR("Number of events could not be determined, cannot proceed.")
    
    ##Below might still be needed
    #if self.site == "LCG.CERN.ch":
    #  return self.__getFilesFromCastor(meta)
#    elif   self.site == "LCG.IN2P3-CC.fr": ##but not this
#      return self.__getFilesFromLyon(meta) ## nor this
    #else:
    return self.fc.findFilesByMetadata(meta)

  def __getFilesFromLyon(self, meta):
    """ List the files present at Lyon, not used.
    """
    ProdID = meta['ProdID']
    prod = str(ProdID).zfill(8)
    energy = meta['Energy']
    bkg = meta["EvtType"]
    detector = meta["DetectorType"]
    path ="/ilc/prod/clic/%s/%s/%s/SIM/%s/" % (energy, bkg, detector, prod)
    comm = ["nsls", "%s" % path]
    res = subprocess.Popen(comm, stdout = subprocess.PIPE).communicate()
    dirlist = res[0].rstrip().split("\n")
    mylist = []
    for mydir in dirlist:
      if mydir.count("dirac_directory"):
        continue
      curdir = path + mydir
      comm2 = ["nsls", curdir]
      res = subprocess.Popen(comm2, stdout = subprocess.PIPE).communicate()
      for f in res[0].rstrip().split("\n"):
        if f.count("dirac_directory"):
          continue
        mylist.append(path + mydir + "/" + f)
    if not mylist:
      return S_ERROR("File list is empty")
    return S_OK(mylist)

  def __getFilesFromCastor(self, meta):
    """ Get the available files (list) from the CERN castor storage
    """ 
    ProdID = meta['ProdID']

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import findFilesByMetadata [as 别名]

#.........这里部分代码省略.........
                                                 100)

      self.metaEventType = self.ops.getValue( "/Overlay/%s/%s/%s/%s/EvtType" % ( self.machine,
                                                                                 self.energytouse,
                                                                                 self.detector,
                                                                                 self.BkgEvtType),
                                              self.BkgEvtType)

    else:
      res = self.ops.getValue("/Overlay/%s/%s/%s/%s/ProdID" % (self.machine, 
                                                               self.energytouse, 
                                                               self.detectormodel, 
                                                               self.BkgEvtType), 
                              0)
      self.nbofeventsperfile = self.ops.getValue("/Overlay/%s/%s/%s/%s/NbEvts" % (self.machine, 
                                                                                  self.energytouse,
                                                                                  self.detectormodel,
                                                                                  self.BkgEvtType), 
                                                 100)
      self.metaEventType = self.ops.getValue( "/Overlay/%s/%s/%s/%s/EvtType" % ( self.machine,
                                                                                 self.energytouse,
                                                                                 self.detectormodel,
                                                                                 self.BkgEvtType),
                                              self.BkgEvtType)

    LOG.info("Number of Events Per BackgroundFile: %d " % self.nbofeventsperfile)

    meta['EvtType'] = self.metaEventType
    meta['ProdID'] = res
    if self.prodid:
      meta['ProdID'] = self.prodid
    LOG.info("Using %s as metadata" % (meta))

    return self.fcc.findFilesByMetadata(meta)


  def __getFilesFromPath(self):
    """ Get the list of files from the FileCatalog via the user specified path.
    """
    meta = {}
    return self.fcc.findFilesByMetadata(meta, self.pathToOverlayFiles)

  def __getFilesFromLyon(self, meta):
    """ List the files present at Lyon, not used.
    """
    prodID = meta['ProdID']
    prod = str(prodID).zfill(8)
    energy = meta['Energy']
    bkg = meta["EvtType"]
    detector = meta["DetectorType"]
    path ="/ilc/prod/clic/%s/%s/%s/SIM/%s/" % (energy, bkg, detector, prod)
    comm = ["nsls", "%s" % path]
    res = subprocess.Popen(comm, stdout = subprocess.PIPE).communicate()
    dirlist = res[0].rstrip().split("\n")
    mylist = []
    for mydir in dirlist:
      if mydir.count("dirac_directory"):
        continue
      curdir = path + mydir
      comm2 = ["nsls", curdir]
      res = subprocess.Popen(comm2, stdout = subprocess.PIPE).communicate()
      for oFile in res[0].rstrip().split("\n"):
        if oFile.count("dirac_directory"):
          continue
        mylist.append(path + mydir + "/" + oFile)
    if not mylist: