Python FileCatalogClient.getFileUserMetadata方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
def getFileInfo(lfn):
  """ Retrieve the file info
  """
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC.Core.Utilities import DEncode
  from DIRAC import gLogger
  fc = FileCatalogClient()
  lumi = 0
  nbevts = 0
  res  = fc.getFileUserMetadata(lfn)
  if not res['OK']:
    gLogger.error("Failed to get metadata of %s" % lfn)
    return (0,0,{})
  if res['Value'].has_key('Luminosity'):   
    lumi += float(res['Value']['Luminosity'])
  addinfo = {}
  if 'AdditionalInfo' in res['Value']:
    addinfo = res['Value']['AdditionalInfo']
    if addinfo.count("{"):
      addinfo = eval(addinfo)
    else:
      addinfo = DEncode.decode(addinfo)[0]
  if "NumberOfEvents" in res['Value'].keys():
    nbevts += int(res['Value']['NumberOfEvents'])
  return (float(lumi),int(nbevts),addinfo)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
def getNumberOfEvents(inputfile):
  """ Find from the FileCatalog the number of events in a file
  """

  files = inputfile
  flist = {}
  for myfile in files:
    if not myfile:
      continue
    bpath = os.path.dirname(myfile)
    if not bpath in flist.keys():
      flist[bpath] = [myfile]
    else:
      flist[bpath].append(myfile)
      
  fc = FileCatalogClient()
  nbevts = {} 
  luminosity = 0
  numberofevents = 0
  evttype = ''
  others = {}
  completeFailure = True

  for path, files in flist.items():
    found_nbevts = False
    found_lumi = False

    if len(files) == 1:
      res = fc.getFileUserMetadata(files[0])
      if not res['OK']:
        gLogger.warn("Failed to get Metadata from file: %s, because: %s" % (files[0], res['Message']))
      else:
        tags = res['Value']
        if "NumberOfEvents" in tags and not found_nbevts:
          numberofevents += int(tags["NumberOfEvents"])
          found_nbevts = True
          completeFailure = False
        if "Luminosity" in tags and not found_lumi:
          luminosity += float(tags["Luminosity"])
          found_lumi = True
        others.update(tags)
        if found_nbevts:
          continue

    res = fc.getDirectoryUserMetadata(path)
    if res['OK']:   
      tags = res['Value']
      if tags.has_key("NumberOfEvents") and not found_nbevts:
        numberofevents += len(files)*int(tags["NumberOfEvents"])
        found_nbevts = True
        completeFailure = False
      if tags.has_key("Luminosity") and not found_lumi:
        luminosity += len(files) * float(tags["Luminosity"])
        found_lumi = True
      if tags.has_key("EvtType"):
        evttype = tags["EvtType"]
      others.update(tags)    
      if found_nbevts: 
        continue
    else:
      gLogger.warn("Failed to get Metadata from path: %s, because: %s" % (path, res['Message']))

    for myfile in files:
      res = fc.getFileUserMetadata(myfile)
      if not res['OK']:
        gLogger.warn("Failed to get Metadata from file: %s, because: %s" % (myfile, res['Message']))
        continue
      tags = res['Value']
      if tags.has_key("NumberOfEvents"):
        numberofevents += int(tags["NumberOfEvents"])
        completeFailure = False
      if tags.has_key("Luminosity") and not found_lumi:
        luminosity += float(tags["Luminosity"])
      others.update(tags)  
        
  nbevts['nbevts'] = numberofevents
  nbevts['lumi'] = luminosity
  nbevts['EvtType'] = evttype
  if 'NumberOfEvents' in others:
    del others['NumberOfEvents']
  if 'Luminosity' in others:
    del others['Luminosity']
  nbevts['AdditionalMeta'] = others

  if completeFailure:
    gLogger.warn("Did not obtain NumberOfEvents from FileCatalog")
    return S_ERROR("Failed to get Number of Events")

  return S_OK(nbevts)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
import os.path as path
import sys
from pprint import pprint


fc = FileCatalogClient()

# gridPath = "/ilc/prod/ilc/mc-dbd.generated/500-TDR_ws/4f/temp001/"
gridPath = "/ilc/prod/ilc/mc-dbd.generated/500-TDR_ws/4f/temp004/"

# filepref="E500-TDR_ws.P4f_sznu_l.Gwhizard-1_95.eR.pL.I250056.001"
filepref="E500-TDR_ws.P4f_sze_l.Gwhizard-1_95.eR.pR.I250035.003"
# srcpath="/ilc/prod/ilc/mc-dbd/generated/500-TDR_ws/4f/"+filepref+".stdhep"
srcpath="/ilc/prod/ilc/mc-dbd/generated/500-TDR_ws/4f/E500-TDR_ws.P4f_sze_l.Gwhizard-1_95.eR.pR.I250035.003.stdhep"
print "Original stdhep : "+srcpath
res=fc.getFileUserMetadata(srcpath)
if not res["OK"] : 
  print "Failed to get meta data of "+srcpath
  exit()

srcMeta = res["Value"]
pprint(srcMeta)

nseq_from=int(0)
nseq_to=int( 135)
nevents=int(1000)
last_fileseq=int( 135)
last_file_nevents=int( 76)

for i in range(nseq_from, nseq_to+1) : 
  filename=filepref+"_%3.3d.stdhep"%i

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
   if res['OK']:
     if len(res['Value'].values()):
       gLogger.verbose("Found some directory matching the metadata")
       for dirs in res['Value'].values():
         res = fc.getDirectoryMetadata(dirs)
         if res['OK']:
           fmeta.update(res['Value'])
         else:
           gLogger.warn("Failed to get dir metadata")
         res = fc.listDirectory(dirs)
         if not res['OK']:
           continue
         content = res['Value']['Successful'][dirs]
         if content["Files"]:
           for f_ex in content["Files"].keys():
             res = fc.getFileUserMetadata(f_ex)
             if res['OK']:
               fmeta.update(res['Value'])
               break
         
   #here we have trans and fmeta
   info.append("")
   info.append("Production %s has the following parameters:" % trans['TransformationID'])
   info.extend(createTransfoInfo(trans))
   if fmeta:
     info.append('The files created by this production have the following metadata:')
     info.extend(createFileInfo(fmeta))
     info.append("It's possible that some meta data was not brought back,")
     info.append("in particular file level metadata, so check some individual files")  
 
 if clip.file:

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
def getNumberOfevents(inputfile):
    """ Find from the FileCatalog the number of events in a file
  """

    files = inputfile
    flist = {}
    for myfile in files:
        if not myfile:
            continue
        bpath = os.path.dirname(myfile)
        if not bpath in flist.keys():
            flist[bpath] = [myfile]
        else:
            flist[bpath].append(myfile)

    fc = FileCatalogClient()
    nbevts = {}
    luminosity = 0
    numberofevents = 0
    evttype = ""
    others = {}

    for path, files in flist.items():
        found_nbevts = False
        found_lumi = False

        if len(files) == 1:
            res = fc.getFileUserMetadata(files[0])
            if not res["OK"]:
                gLogger.verbose("Failed to get meta data")
                continue
            tags = res["Value"]
            if tags.has_key("NumberOfEvents") and not found_nbevts:
                numberofevents += int(tags["NumberOfEvents"])
                found_nbevts = True
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += float(tags["Luminosity"])
                found_lumi = True
            others.update(tags)
            if found_nbevts:
                continue

        res = fc.getDirectoryMetadata(path)
        if res["OK"]:
            tags = res["Value"]
            if tags.has_key("NumberOfEvents") and not found_nbevts:
                numberofevents += len(files) * int(tags["NumberOfEvents"])
                found_nbevts = True
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += len(files) * float(tags["Luminosity"])
                found_lumi = True
            if tags.has_key("EvtType"):
                evttype = tags["EvtType"]
            others.update(tags)
            if found_nbevts:
                continue

        for myfile in files:
            res = fc.getFileUserMetadata(myfile)
            if not res["OK"]:
                continue
            tags = res["Value"]
            if tags.has_key("NumberOfEvents"):
                numberofevents += int(tags["NumberOfEvents"])
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += float(tags["Luminosity"])
            others.update(tags)

    nbevts["nbevts"] = numberofevents
    nbevts["lumi"] = luminosity
    nbevts["EvtType"] = evttype
    if "NumberOfEvents" in others:
        del others["NumberOfEvents"]
    if "Luminosity" in others:
        del others["Luminosity"]
    nbevts["AdditionalMeta"] = others
    return nbevts

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]

#.........这里部分代码省略.........

    #####################################################################
    # File Options
    def registerFile(self, lfn, dfcAttrDict):
        """Register a new file in the DFC.
        
        """
        # TODO:need more tests,if directory of file doesn't exist,
        # addFile will create it without setting any metadata(lin lei)
        # need to check whether directory of file exists in dfc?(lin lei)
        # pass
        fc = self.client
        result = fc.addFile({lfn: dfcAttrDict})
        if result["OK"]:
            if result["Value"]["Successful"]:
                if result["Value"]["Successful"].has_key(lfn):
                    return S_OK()
            elif result["Value"]["Failed"]:
                if result["Value"]["Failed"].has_key(lfn):
                    print "Failed to add this file:", result["Value"]["Failed"][lfn]
                    return S_ERROR()
        else:
            print "Failed to add this file :", result["Message"]
            return S_ERROR()
        # need to register file (inc. creating appropriate directory
        # if it doesn't already exist; and register metadata for that
        # file / directory
        # Q: how / where to pass the metadata?

    def getFileMetaVal(self, lfn):
        """get the File Meta Value of given file

      """
        result = self.client.getFileUserMetadata(lfn)
        if result["OK"]:
            return result["Value"]
        else:
            print "Failed to get meta Value of this file"
            return {}

    def reCalcCount(self, fileList, plus=True):
        """calculate the value of metadata 'count',when a file contain in a dataset
      count+1,when del a dataset,then all file in this dataset count -1
      default plus=True,means count+1,if count-1,set plus=False 
      return the value of count, count = -1 means error.
      NOTE:this function should only be called when create or delete a dataset.
      """
        countDict = {}
        if type(fileList) != type([]):
            fileList = [fileList]
        for file in fileList:
            result = self.getFileMetaVal(file)
            if len(result) != 0:
                count = result["count"]
                if plus:
                    count += 1
                else:
                    if count > 0:
                        count -= 1
                cDict = {"count": count}
                self.registerFileMetadata(file, cDict)
                countDict[file] = count
            else:
                print "Failed reCalculate value of count of file %s" % file
                countDict[file] = -1

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
def _getInfo():
  """gets info about transformation"""
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  if not clip.prodid and not clip.filename:
    Script.showHelp()
    dexit(1)

  from DIRAC import gLogger
  import os

  from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
  tc = TransformationClient()

  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  fmeta = {}
  trans = None
  info = []

  if clip.prodid:
    res = tc.getTransformation(clip.prodid)
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    trans = res['Value']
    res = tc.getTransformationInputDataQuery( clip.prodid )
    if res['OK']:
      trans['InputDataQuery'] = res['Value']
    res = tc.getAdditionalParameters ( clip.prodid )
    if res['OK']:
      trans['AddParams'] = res['Value']
    #do something with transf
    res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
    if res1['OK'] and len(res1['Value'].values()):
      gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
      for dirs in res1['Value'].values():
        res = fc.getDirectoryUserMetadata(dirs)
        if res['OK']:
          fmeta.update(res['Value'])
        else:
          gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
          continue
        res = fc.listDirectory(dirs)
        if not res['OK']:
          continue
        content = res['Value']['Successful'][dirs]
        if content["Files"]:
          for f_ex in content["Files"].keys():
            res = fc.getFileUserMetadata(f_ex)
            if res['OK']:
              fmeta.update(res['Value'])
              break

    #here we have trans and fmeta
    info.append("")
    info.append("Production %s has the following parameters:" % trans['TransformationID'])
    info.extend(_createTransfoInfo(trans))

    if fmeta:
      info.append('The files created by this production have the following metadata:')
      info.extend(_createFileInfo(fmeta))
      info.append("It's possible that some meta data was not brought back,")
      info.append("in particular file level metadata, so check some individual files")

  if clip.filename:
    pid = ""
    if clip.filename.count("/"):
      fpath = os.path.dirname(clip.filename)
      res = fc.getDirectoryUserMetadata(fpath)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(0)
      fmeta.update(res['Value'])
      res = fc.getFileUserMetadata(clip.filename)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(1)
      fmeta.update(res['Value'])
      if 'ProdID' in fmeta:
        pid = str(fmeta['ProdID'])
      res = fc.getFileAncestors([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
          if ancestorsDict.keys():
            fmeta["Ancestors"] = ancestorsDict.keys()
      res = fc.getFileDescendents([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,descendDict in res['Value']['Successful'].items():
          if descendDict.keys():
            fmeta['Descendants'] = descendDict.keys()
    else:
      ext = clip.filename.split(".")[-1]
      fitems = []
      for i in clip.filename.split('.')[:-1]:
        fitems.extend(i.split('_'))
      pid = ''
      if ext == 'stdhep':
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getFileUserMetadata [as 别名]
    ## The frames retrieved from the metadata query.
    retrieved_frames = ['LFN:%s' % fn for fn in sorted(result["Value"])]
    #
    if len(retrieved_frames) == 0:
        raise IOError("* ERROR: no frames found. Quitting.")

    # Get the expected cluster file names from metadata interface.

    ## Kluster file names.
    kluster_file_names = []

    for fn in retrieved_frames:
        #print("* Frame: '%s'." % (fn))
        # Ah - LFN: needs to be removed from the start...
        filemetadata = fc.getFileUserMetadata(fn[4:])
        #print filemetadata
        frameid = str(filemetadata['Value']['frameid'])
        n_kluster = int(filemetadata['Value']['n_kluster'])
        #print("*--> Frame ID           : '%s'" % (frameid))
        #print("*--> Number of clusters = %d" % (n_kluster))
        #print("*")
        for i in range(n_kluster):
            kn = "%s_k%05d.png" % (frameid, i)
            kluster_file_names.append(kn)
    print("*")

    #lg.info(" * Clusters to be downloaded:")
    #for kn in kluster_file_names:
    #    lg.info(" *--> '%s'" % (kn))