Python FileCatalogClient.getDirectoryMetadata方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]
def compare(attributes,input):
    error_list = {}
    inDFC = {}
    
    #print "input keys:",input.keys()
    keys = sorted(input.keys())
    #print "keys after being sorted:",keys
    
    expNum = _get_exp_num(attributes["expNum"])
    eventType = _get_event_type(attributes["eventType"])
    #dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    if attributes["streamId"]=="stream0":
        dir = dir+ "/data"+"/"+eventType + "/"+expNum+"/"+attributes["LFN"]
    else:
        dir = dir+"/mc"+"/"+eventType+"/"+expNum+"/"+attributes["streamId"]+"/"+attributes["LFN"]

    client=FileCatalogClient()
    
    result = client.getFileMetadata(dir)

    file_exist = len(result['Value']['Successful'])
    if file_exist == 0:
        print "this file does't exist in DFC",attributes['LFN']
    else:    
        result = client.getDirectoryMetadata(dir)

        if result["OK"]:
            inDFC["resonance"] = result["Value"]["resonance"]
            inDFC["streamId"] = result["Value"]["streamId"]
            inDFC["eventType"] = result["Value"]["eventType"]
            inDFC["bossVer"] = result["Value"]["bossVer"]
            inDFC["expNum"] = result["Value"]["expNum"]
        
        for key in keys:
            if input[key] != inDFC[key]:
                error_list[key] = inDFC[key]


        if error_list is not None:
            return error_list

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]
   continue
 else:
   gLogger.error("Unknown production type %s"% prodtype)
   continue
 res = fc.findFilesByMetadata(meta)  
 if not res['OK']:
   gLogger.error(res['Message'])
   continue
 lfns = res['Value']
 nb_files = len(lfns)
 path = ""
 if not len(lfns):
   gLogger.warn("No files found for prod %s" % prodID)
   continue
 path = os.path.dirname(lfns[0])
 res = fc.getDirectoryMetadata(path)
 if not res['OK']:
   gLogger.warn('No meta data found for %s' % path)
   continue
 dirmeta = {}
 dirmeta['proddetail'] = proddetail
 dirmeta['prodtype'] = prodtype
 dirmeta['nb_files']=nb_files
 dirmeta.update(res['Value'])
 lumi  = 0.
 nbevts = 0
 addinfo = None
 files = 0
 xsec = 0.0
 if not full_detail:
   lfn  = lfns[0]

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]
def getExpRes(runids):
    entries = []
    expRes = {}
    expNumList = []
    resList = []
    
    #print"runids",runids
    #from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    client = FileCatalogClient()
    #get all entries under catalog "/BES3/ExpSearch"
    dir = '/BES3/ExpSearch'
    result = client.listDirectory(dir)
    #print "result",result
    #print result
    if result['OK']:
        for i,v in enumerate(result['Value']['Successful'][dir]['SubDirs']):
            result = client.getDirectoryMetadata(v)['Value']
            #print "result",result
            runfrm = string.atoi(result['runFrm'])
            runto = string.atoi(result['runTo'])
            #print "runfrm",runfrm
            #print "runto",runto
    
            for runid in runids:
                #print "runid",runid
                #check all runid whether between runfrm and runto of each entry 
                #under catalog "/BES3/ExpSearch"
                if runfrm<=runid<=runto:
                    #print "true"
                    #if this runid between runfrm and runto,and expNum isn't in expNumList
                    #add this expNum to expNumList
                    if result['expNum'] not in expNumList:
                        expNumList.append(result['expNum'])

                    #resonance of this id isn't in resonance List,add it to resList
                    if result['resonance'] not in resList:
                        resList.append(result['resonance'])
                    
        #only including one resonance
        if len(resList) == 1:
            expRes["resonance"] = resList[0]
        else:
            #has several resonances,may be has something wrong to this file
            print "several resonances found:",resList
            return False

        #only including one expNum
        if len(expNumList) == 1:
            expRes["expNum"] = expNumList[0]
        else:
            #if including several expNums,combine these expNum into mexpN1pN2p...
            sorted(expNumList)
            str = "m" + expNumList[0]
            for expNum in expNumList[1:]:
                str = str + "p+" + getNum(expNum)
           
            expRes["expNum"] = str

    else:
        print "ExpSearch directory is empty, please run createBesDirDFC first"
        return False
  
    return expRes

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]
   gLogger.error(res['Message'])
   dexit(1)
 trans = res['Value']
 res = tc.getTransformationInputDataQuery( clip.prodid )
 if res['OK']:
   trans['InputDataQuery'] = res['Value']
 res = tc.getAdditionalParameters ( clip.prodid )
 if res['OK']:
   trans['AddParams'] = res['Value']
 #do something with transf
 res = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
 if res['OK']:
   if len(res['Value'].values()):
     gLogger.verbose("Found some directory matching the metadata")
     for dirs in res['Value'].values():
       res = fc.getDirectoryMetadata(dirs)
       if res['OK']:
         fmeta.update(res['Value'])
       else:
         gLogger.warn("Failed to get dir metadata")
       res = fc.listDirectory(dirs)
       if not res['OK']:
         continue
       content = res['Value']['Successful'][dirs]
       if content["Files"]:
         for f_ex in content["Files"].keys():
           res = fc.getFileUserMetadata(f_ex)
           if res['OK']:
             fmeta.update(res['Value'])
             break
       

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]
def getNumberOfevents(inputfile):
    """ Find from the FileCatalog the number of events in a file
  """

    files = inputfile
    flist = {}
    for myfile in files:
        if not myfile:
            continue
        bpath = os.path.dirname(myfile)
        if not bpath in flist.keys():
            flist[bpath] = [myfile]
        else:
            flist[bpath].append(myfile)

    fc = FileCatalogClient()
    nbevts = {}
    luminosity = 0
    numberofevents = 0
    evttype = ""
    others = {}

    for path, files in flist.items():
        found_nbevts = False
        found_lumi = False

        if len(files) == 1:
            res = fc.getFileUserMetadata(files[0])
            if not res["OK"]:
                gLogger.verbose("Failed to get meta data")
                continue
            tags = res["Value"]
            if tags.has_key("NumberOfEvents") and not found_nbevts:
                numberofevents += int(tags["NumberOfEvents"])
                found_nbevts = True
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += float(tags["Luminosity"])
                found_lumi = True
            others.update(tags)
            if found_nbevts:
                continue

        res = fc.getDirectoryMetadata(path)
        if res["OK"]:
            tags = res["Value"]
            if tags.has_key("NumberOfEvents") and not found_nbevts:
                numberofevents += len(files) * int(tags["NumberOfEvents"])
                found_nbevts = True
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += len(files) * float(tags["Luminosity"])
                found_lumi = True
            if tags.has_key("EvtType"):
                evttype = tags["EvtType"]
            others.update(tags)
            if found_nbevts:
                continue

        for myfile in files:
            res = fc.getFileUserMetadata(myfile)
            if not res["OK"]:
                continue
            tags = res["Value"]
            if tags.has_key("NumberOfEvents"):
                numberofevents += int(tags["NumberOfEvents"])
            if tags.has_key("Luminosity") and not found_lumi:
                luminosity += float(tags["Luminosity"])
            others.update(tags)

    nbevts["nbevts"] = numberofevents
    nbevts["lumi"] = luminosity
    nbevts["EvtType"] = evttype
    if "NumberOfEvents" in others:
        del others["NumberOfEvents"]
    if "Luminosity" in others:
        del others["Luminosity"]
    nbevts["AdditionalMeta"] = others
    return nbevts

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getDirectoryMetadata [as 别名]

#.........这里部分代码省略.........
    # dir options
    def removeDir(self, dir):
        """remove the dir include files and subdirs
        """
        result = self.client.listDirectory(dir)
        if result["OK"]:
            if not result["Value"]["Successful"][dir]["Files"] and not result["Value"]["Successful"][dir]["SubDirs"]:
                # print 'no file and subDirs in this dir'
                self.client.removeDirectory(dir)
                return S_OK()
            else:
                if result["Value"]["Successful"][dir]["Files"]:
                    for file in result["Value"]["Successful"][dir]["Files"]:
                        self.client.removeFile(file)
                else:
                    for subdir in result["Value"]["Successful"][dir]["SubDirs"]:
                        self.removeDir(subdir)
                    self.removeDir(dir)

    def listDir(self, dir):
        """list the files under the given DFC dir"""
        fileList = []
        result = self.client.listDirectory(dir)
        if result["OK"]:
            if result["Value"]["Successful"][dir]["Files"]:
                fileList = result["Value"]["Successful"][dir]["Files"].keys()
                fileList.sort()
        else:
            print "no files under this dir"
        return fileList

    def getDirMetaVal(self, dir):
        """list the registed metadata value of the given dir"""
        result = self.client.getDirectoryMetadata(dir)
        if result["OK"]:
            return result["Value"]
        else:
            print "Failed to get meta Value of the directory"
            return {}

    #################################################################################
    # meta fields operations
    #
    def addNewFields(self, fieldName, fieldType, metaType="-d"):
        """add new fields,if metaType is '-f',add file field,
        fileType is datatpye in MySQL notation
      """
        result = self.client.addMetadataField(fieldName, fieldType, metaType)
        if not result["OK"]:
            return S_ERROR(result)
        else:
            return S_OK()

    def deleteMetaField(self, fieldName):
        """delete a exist metafield"""
        result = self.client.deleteMetadataField(fieldName)
        if not result["OK"]:
            return S_ERROR(result)
        else:
            return S_OK()

    def getAllFields(self):
        """get all meta fields,include file metafield and dir metafield.
        """
        result = self.client.getMetadataFields()
        if not result["OK"]: