Python FileCatalogClient.getDirectoryUserMetadata方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryUserMetadata [as 别名]
  def _getProdInfoFromIDs(self):
    """get the processName, energy and eventsPerJob from the MetaData catalog

    :raises: AttributeError if some of the information cannot be found
    :returns: None
    """
    if not self.prodIDs:
      raise AttributeError("No prodIDs defined")

    self.eventsPerJobs = []
    self.processes = []
    self.energies = []
    from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
    from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    trc = TransformationClient()
    fc = FileCatalogClient()
    for prodID in self.prodIDs:
      gLogger.notice("Getting information for %s" % prodID)
      tRes = trc.getTransformation(str(prodID))
      if not tRes['OK']:
        raise AttributeError("No prodInfo found for %s" % prodID)
      self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask']))
      lfnRes = fc.findFilesByMetadata({'ProdID': prodID})
      if not lfnRes['OK'] or not lfnRes['Value']:
        raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value'))))
      path = os.path.dirname(lfnRes['Value'][0])
      fileRes = fc.getDirectoryUserMetadata(path)
      self.processes.append(fileRes['Value']['EvtType'])
      self.energies.append(fileRes['Value']['Energy'])
      gLogger.notice("Found (Evts,Type,Energy): %s %s %s " %
                     (self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryUserMetadata [as 别名]
def getNumberOfEvents(inputfile):
  """ Find from the FileCatalog the number of events in a file
  """

  files = inputfile
  flist = {}
  for myfile in files:
    if not myfile:
      continue
    bpath = os.path.dirname(myfile)
    if not bpath in flist.keys():
      flist[bpath] = [myfile]
    else:
      flist[bpath].append(myfile)
      
  fc = FileCatalogClient()
  nbevts = {} 
  luminosity = 0
  numberofevents = 0
  evttype = ''
  others = {}
  completeFailure = True

  for path, files in flist.items():
    found_nbevts = False
    found_lumi = False

    if len(files) == 1:
      res = fc.getFileUserMetadata(files[0])
      if not res['OK']:
        gLogger.warn("Failed to get Metadata from file: %s, because: %s" % (files[0], res['Message']))
      else:
        tags = res['Value']
        if "NumberOfEvents" in tags and not found_nbevts:
          numberofevents += int(tags["NumberOfEvents"])
          found_nbevts = True
          completeFailure = False
        if "Luminosity" in tags and not found_lumi:
          luminosity += float(tags["Luminosity"])
          found_lumi = True
        others.update(tags)
        if found_nbevts:
          continue

    res = fc.getDirectoryUserMetadata(path)
    if res['OK']:   
      tags = res['Value']
      if tags.has_key("NumberOfEvents") and not found_nbevts:
        numberofevents += len(files)*int(tags["NumberOfEvents"])
        found_nbevts = True
        completeFailure = False
      if tags.has_key("Luminosity") and not found_lumi:
        luminosity += len(files) * float(tags["Luminosity"])
        found_lumi = True
      if tags.has_key("EvtType"):
        evttype = tags["EvtType"]
      others.update(tags)    
      if found_nbevts: 
        continue
    else:
      gLogger.warn("Failed to get Metadata from path: %s, because: %s" % (path, res['Message']))

    for myfile in files:
      res = fc.getFileUserMetadata(myfile)
      if not res['OK']:
        gLogger.warn("Failed to get Metadata from file: %s, because: %s" % (myfile, res['Message']))
        continue
      tags = res['Value']
      if tags.has_key("NumberOfEvents"):
        numberofevents += int(tags["NumberOfEvents"])
        completeFailure = False
      if tags.has_key("Luminosity") and not found_lumi:
        luminosity += float(tags["Luminosity"])
      others.update(tags)  
        
  nbevts['nbevts'] = numberofevents
  nbevts['lumi'] = luminosity
  nbevts['EvtType'] = evttype
  if 'NumberOfEvents' in others:
    del others['NumberOfEvents']
  if 'Luminosity' in others:
    del others['Luminosity']
  nbevts['AdditionalMeta'] = others

  if completeFailure:
    gLogger.warn("Did not obtain NumberOfEvents from FileCatalog")
    return S_ERROR("Failed to get Number of Events")

  return S_OK(nbevts)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryUserMetadata [as 别名]

#.........这里部分代码省略.........
  #############################################################################
  def createWorkflow(self):
    """ Create XML for local testing.
    """
    name = '%s.xml' % self.name
    if os.path.exists(name):
      shutil.move(name,'%s.backup' % name)
    self.workflow.toXMLFile(name)
    
  #############################################################################
  def setOutputSE(self, outputse):
    """ Define where the output file(s) will go. 
    """
    self.outputStorage = outputse
    return S_OK()
  
  #############################################################################
  def setInputDataQuery(self, metadata):
    """ Define the input data query needed
    """

    retMetaKey = self._checkMetaKeys( metadata.keys() )
    if not retMetaKey['OK']:
      return retMetaKey

    if "ProdID" not in metadata:
      return self._reportError("Input metadata dictionary must contain at least a key 'ProdID' as reference")
    retDirs = self._checkFindDirectories( metadata )
    if not retDirs['OK']:
      return retDirs
    dirs = retDirs['Value'].values()
    for mdir in dirs[:self.maxFCFoldersToCheck]:
      LOG.notice("Directory: %s" % mdir)
      res = self.fc.getDirectoryUserMetadata(mdir)
      if not res['OK']:
        return self._reportError("Error looking up the catalog for directory metadata")
      compatmeta = res['Value']
      compatmeta.update(metadata)

    if 'EvtType' in compatmeta:
      self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring )
    else:
      return self._reportError("EvtType is not in the metadata, it has to be!")

    if 'NumberOfEvents' in compatmeta:
      self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None )

    self.basename = self.evttype
    LOG.notice("MetaData: %s" % compatmeta)
    LOG.notice("MetaData: %s" % metadata)
    if "Energy" in compatmeta:
      self.energycat = JobHelpers.getValue( compatmeta["Energy"], str, (int, long, basestring) )
        
    if self.energycat.count("tev"):
      self.energy = Decimal("1000.") * Decimal(self.energycat.split("tev")[0])
    elif self.energycat.count("gev"):
      self.energy = Decimal("1.") * Decimal(self.energycat.split("gev")[0])
    else:
      self.energy = Decimal("1.") * Decimal(self.energycat)
    gendata = False
    if 'Datatype' in compatmeta:
      self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring )
      if self.datatype == 'gen':
        gendata = True
    if "DetectorType" in compatmeta and not gendata:
      self.detector = JobHelpers.getValue( compatmeta["DetectorType"], str, basestring )

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryUserMetadata [as 别名]
def _getInfo():
  """gets info about transformation"""
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  if not clip.prodid and not clip.filename:
    Script.showHelp()
    dexit(1)

  from DIRAC import gLogger
  import os

  from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
  tc = TransformationClient()

  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  fmeta = {}
  trans = None
  info = []

  if clip.prodid:
    res = tc.getTransformation(clip.prodid)
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    trans = res['Value']
    res = tc.getTransformationInputDataQuery( clip.prodid )
    if res['OK']:
      trans['InputDataQuery'] = res['Value']
    res = tc.getAdditionalParameters ( clip.prodid )
    if res['OK']:
      trans['AddParams'] = res['Value']
    #do something with transf
    res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
    if res1['OK'] and len(res1['Value'].values()):
      gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
      for dirs in res1['Value'].values():
        res = fc.getDirectoryUserMetadata(dirs)
        if res['OK']:
          fmeta.update(res['Value'])
        else:
          gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
          continue
        res = fc.listDirectory(dirs)
        if not res['OK']:
          continue
        content = res['Value']['Successful'][dirs]
        if content["Files"]:
          for f_ex in content["Files"].keys():
            res = fc.getFileUserMetadata(f_ex)
            if res['OK']:
              fmeta.update(res['Value'])
              break

    #here we have trans and fmeta
    info.append("")
    info.append("Production %s has the following parameters:" % trans['TransformationID'])
    info.extend(_createTransfoInfo(trans))

    if fmeta:
      info.append('The files created by this production have the following metadata:')
      info.extend(_createFileInfo(fmeta))
      info.append("It's possible that some meta data was not brought back,")
      info.append("in particular file level metadata, so check some individual files")

  if clip.filename:
    pid = ""
    if clip.filename.count("/"):
      fpath = os.path.dirname(clip.filename)
      res = fc.getDirectoryUserMetadata(fpath)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(0)
      fmeta.update(res['Value'])
      res = fc.getFileUserMetadata(clip.filename)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(1)
      fmeta.update(res['Value'])
      if 'ProdID' in fmeta:
        pid = str(fmeta['ProdID'])
      res = fc.getFileAncestors([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
          if ancestorsDict.keys():
            fmeta["Ancestors"] = ancestorsDict.keys()
      res = fc.getFileDescendents([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,descendDict in res['Value']['Successful'].items():
          if descendDict.keys():
            fmeta['Descendants'] = descendDict.keys()
    else:
      ext = clip.filename.split(".")[-1]
      fitems = []
      for i in clip.filename.split('.')[:-1]:
        fitems.extend(i.split('_'))
      pid = ''
      if ext == 'stdhep':
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryUserMetadata [as 别名]
def _getProductionSummary():
  clip = _Params()
  clip.registerSwitch()
  Script.parseCommandLine()
  from ILCDIRAC.Core.Utilities.HTML                             import Table
  from ILCDIRAC.Core.Utilities.ProcessList                      import ProcessList
  from DIRAC.TransformationSystem.Client.TransformationClient   import TransformationClient
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC import gConfig, gLogger
  prod = clip.prod
  full_detail = clip.full_det
  fc = FileCatalogClient()
  
  processlist = gConfig.getValue('/LocalSite/ProcessListPath')
  prl = ProcessList(processlist)
  processesdict = prl.getProcessesDict()
  
  trc = TransformationClient()
  prodids = []
  if not prod:
    conddict = {}
    conddict['Status'] = clip.statuses
    if clip.ptypes:
      conddict['Type'] = clip.ptypes
    res = trc.getTransformations( conddict )
    if res['OK']:
      for transfs in res['Value']:
        prodids.append(transfs['TransformationID'])
  else:
    prodids.extend(prod)

  metadata = []
  
  gLogger.info("Will run on prods %s" % str(prodids))
  
  for prodID in prodids:
    if prodID<clip.minprod:
      continue
    meta = {}
    meta['ProdID']=prodID
    res = trc.getTransformation(str(prodID))
    if not res['OK']:
      gLogger.error("Error getting transformation %s" % prodID )
      continue
    prodtype = res['Value']['Type']
    proddetail = res['Value']['Description']
    if prodtype == 'MCReconstruction' or prodtype == 'MCReconstruction_Overlay' :
      meta['Datatype']='DST'
    elif prodtype == 'MCGeneration':
      meta['Datatype']='gen'
    elif prodtype == 'MCSimulation':
      meta['Datatype']='SIM'
    elif prodtype in ['Split','Merge']:
      gLogger.warn("Invalid query for %s productions" % prodtype)
      continue
    else:
      gLogger.error("Unknown production type %s"% prodtype)
      continue
    res = fc.findFilesByMetadata(meta)  
    if not res['OK']:
      gLogger.error(res['Message'])
      continue
    lfns = res['Value']
    nb_files = len(lfns)
    path = ""
    if not len(lfns):
      gLogger.warn("No files found for prod %s" % prodID)
      continue
    path = os.path.dirname(lfns[0])
    res = fc.getDirectoryUserMetadata(path)
    if not res['OK']:
      gLogger.warn('No meta data found for %s' % path)
      continue
    dirmeta = {}
    dirmeta['proddetail'] = proddetail
    dirmeta['prodtype'] = prodtype
    dirmeta['nb_files']=nb_files
    dirmeta.update(res['Value'])
    lumi  = 0.
    nbevts = 0
    addinfo = None
    files = 0
    xsec = 0.0
    if not full_detail:
      lfn  = lfns[0]
      info = _getFileInfo(lfn)
      nbevts = info[1]*len(lfns)
      lumi = info[0]*len(lfns)
      addinfo = info[2]
      if 'xsection' in addinfo:
        if 'sum' in addinfo['xsection']:
          if 'xsection' in addinfo['xsection']['sum']:
            xsec += addinfo['xsection']['sum']['xsection']
            files += 1
    else:
      for lfn in lfns:
        info = _getFileInfo(lfn)
        lumi += info[0]
        nbevts += info[1]
        addinfo = info[2]
#.........这里部分代码省略.........