Python FileCatalogClient.listDirectory方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
def eventTypeCheck(eventType):
    entries = []
    
    client = FileCatalogClient()
    dir = '/BES3/EventTypeList'
    result = client.listDirectory(dir)
    if result['OK']:
        for i,v in enumerate(result['Value']['Successful'][dir]['SubDirs']):
            if eventType == v.split('/')[2]:   
                return True

    return False 

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
f = open('time.txt', 'w')
f2 = open('clock.txt', 'w')
f.write("QueryStart\tQueryEnd\tQueryTime\textra(port %s)\n"%port)
f2.write("QueryStart\tQueryEnd\tQueryClock\textra(port %s)\n"%port)

start = time.time()

done = False

while not done:
  # Between 0 and 3 because in generate we have 4 subdirs per dir. Adapt :-)
  rndTab = [random.randint( 0, 3 ) for i in xrange( readDepth ) ]
  dirPath = '/' + '/'.join(map(str,rndTab))
  before = time.time()
  beforeC = time.clock()
  res = fc.listDirectory(dirPath)
  afterC = time.clock()
  after = time.time()
  queryTime = after - before
  queryTimeC = afterC - beforeC
  if not res['OK']:
    extra = res['Message']
  else:
    out = res['Value']['Successful'][dirPath]
    extra = "%s %s %s"%(dirPath, len(out['Files']), len(out['SubDirs']))

  f.write("%s\t%s\t%s\t%s\n"%(before, after, queryTime, extra))
  f.flush()
  os.fsync(f)
  f2.write("%s\t%s\t%s\t%s\n"%(beforeC, afterC, queryTimeC, extra))
  f2.flush()

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
def getExpRes(runids):
    entries = []
    expRes = {}
    expNumList = []
    resList = []
    
    #print"runids",runids
    #from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    client = FileCatalogClient()
    #get all entries under catalog "/BES3/ExpSearch"
    dir = '/BES3/ExpSearch'
    result = client.listDirectory(dir)
    #print "result",result
    #print result
    if result['OK']:
        for i,v in enumerate(result['Value']['Successful'][dir]['SubDirs']):
            result = client.getDirectoryMetadata(v)['Value']
            #print "result",result
            runfrm = string.atoi(result['runFrm'])
            runto = string.atoi(result['runTo'])
            #print "runfrm",runfrm
            #print "runto",runto
    
            for runid in runids:
                #print "runid",runid
                #check all runid whether between runfrm and runto of each entry 
                #under catalog "/BES3/ExpSearch"
                if runfrm<=runid<=runto:
                    #print "true"
                    #if this runid between runfrm and runto,and expNum isn't in expNumList
                    #add this expNum to expNumList
                    if result['expNum'] not in expNumList:
                        expNumList.append(result['expNum'])

                    #resonance of this id isn't in resonance List,add it to resList
                    if result['resonance'] not in resList:
                        resList.append(result['resonance'])
                    
        #only including one resonance
        if len(resList) == 1:
            expRes["resonance"] = resList[0]
        else:
            #has several resonances,may be has something wrong to this file
            print "several resonances found:",resList
            return False

        #only including one expNum
        if len(expNumList) == 1:
            expRes["expNum"] = expNumList[0]
        else:
            #if including several expNums,combine these expNum into mexpN1pN2p...
            sorted(expNumList)
            str = "m" + expNumList[0]
            for expNum in expNumList[1:]:
                str = str + "p+" + getNum(expNum)
           
            expRes["expNum"] = str

    else:
        print "ExpSearch directory is empty, please run createBesDirDFC first"
        return False
  
    return expRes

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
   trans['InputDataQuery'] = res['Value']
 res = tc.getAdditionalParameters ( clip.prodid )
 if res['OK']:
   trans['AddParams'] = res['Value']
 #do something with transf
 res = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
 if res['OK']:
   if len(res['Value'].values()):
     gLogger.verbose("Found some directory matching the metadata")
     for dirs in res['Value'].values():
       res = fc.getDirectoryMetadata(dirs)
       if res['OK']:
         fmeta.update(res['Value'])
       else:
         gLogger.warn("Failed to get dir metadata")
       res = fc.listDirectory(dirs)
       if not res['OK']:
         continue
       content = res['Value']['Successful'][dirs]
       if content["Files"]:
         for f_ex in content["Files"].keys():
           res = fc.getFileUserMetadata(f_ex)
           if res['OK']:
             fmeta.update(res['Value'])
             break
       
 #here we have trans and fmeta
 info.append("")
 info.append("Production %s has the following parameters:" % trans['TransformationID'])
 info.extend(createTransfoInfo(trans))
 if fmeta:

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]

#.........这里部分代码省略.........
        """Internal function to set metadata values on a given lfn. 
          Returns True for success, False for failure.
        """
        metadataDict = {}
        metadataDict["runL"] = attributes["runL"]
        metadataDict["runH"] = attributes["runH"]
        metadataDict["status"] = attributes["status"]
        metadataDict["eventNumber"] = attributes["eventNumber"]
        metadataDict["count"] = attributes["count"]
        result = self.client.setMetadata(lfn, metadataDict)
        if not result["OK"]:
            return S_ERROR()
        else:
            return S_OK()

    def __registerDirMetadata(self, dir, metaDict):
        """Internal function to set metadata to a directory
           Returns True for success, False for failure.
        """
        fc = self.client
        result = fc.setMetadata(dir, metaDict)
        if result["OK"]:
            return S_OK()
        else:
            message = "Error for setting metadata %s to %s: %s" % (metaDict, dir, result["Message"])
            return S_ERROR("Message")

    def __dirExists(self, dir, parentDir):
        """ Internal function to check whether 'dir' is the subdirectory of 'parentDir'
            Returns 1 for Yes, 0 for NO
        """
        fc = self.client
        dir_exists = 0
        result = fc.listDirectory(parentDir)
        if result["OK"]:
            for i, v in enumerate(result["Value"]["Successful"][parentDir]["SubDirs"]):
                if v == dir:
                    dir_exists = 1
                    break
        else:
            print "Failed to list subdirectories of %s:%s" % (parentDir, result["Message"])

        return dir_exists

    def __registerSubDirs(self, dirs_dict, dirs_meta):
        """Internal function to create directories in dirs_dict
           Returns True for sucess, False for failure
        """
        creation_ok = True

        for dir in dirs_dict:
            if (dir != "dir_file") & (dir != "dir_data_mc"):
                if self.__registerDir(dirs_meta[dir][0])["OK"]:
                    result = self.__registerDirMetadata(dirs_meta[dir][0], {dir.split("_")[1]: dirs_meta[dir][1]})
                    if not result["OK"]:
                        creation_ok = False
                        break
                else:
                    print "Failed to create %s" % dir
                    creation_ok = False
                    break
            else:
                if not self.__registerDir(dirs_meta[dir])["OK"]:
                    print "Failed to create %s" % dir
                    creation_ok = False
                    break

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
def _getInfo():
  """gets info about transformation"""
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  if not clip.prodid and not clip.filename:
    Script.showHelp()
    dexit(1)

  from DIRAC import gLogger
  import os

  from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
  tc = TransformationClient()

  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  fmeta = {}
  trans = None
  info = []

  if clip.prodid:
    res = tc.getTransformation(clip.prodid)
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    trans = res['Value']
    res = tc.getTransformationInputDataQuery( clip.prodid )
    if res['OK']:
      trans['InputDataQuery'] = res['Value']
    res = tc.getAdditionalParameters ( clip.prodid )
    if res['OK']:
      trans['AddParams'] = res['Value']
    #do something with transf
    res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
    if res1['OK'] and len(res1['Value'].values()):
      gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
      for dirs in res1['Value'].values():
        res = fc.getDirectoryUserMetadata(dirs)
        if res['OK']:
          fmeta.update(res['Value'])
        else:
          gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
          continue
        res = fc.listDirectory(dirs)
        if not res['OK']:
          continue
        content = res['Value']['Successful'][dirs]
        if content["Files"]:
          for f_ex in content["Files"].keys():
            res = fc.getFileUserMetadata(f_ex)
            if res['OK']:
              fmeta.update(res['Value'])
              break

    #here we have trans and fmeta
    info.append("")
    info.append("Production %s has the following parameters:" % trans['TransformationID'])
    info.extend(_createTransfoInfo(trans))

    if fmeta:
      info.append('The files created by this production have the following metadata:')
      info.extend(_createFileInfo(fmeta))
      info.append("It's possible that some meta data was not brought back,")
      info.append("in particular file level metadata, so check some individual files")

  if clip.filename:
    pid = ""
    if clip.filename.count("/"):
      fpath = os.path.dirname(clip.filename)
      res = fc.getDirectoryUserMetadata(fpath)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(0)
      fmeta.update(res['Value'])
      res = fc.getFileUserMetadata(clip.filename)
      if not res['OK']:
        gLogger.error(res['Message'])
        dexit(1)
      fmeta.update(res['Value'])
      if 'ProdID' in fmeta:
        pid = str(fmeta['ProdID'])
      res = fc.getFileAncestors([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
          if ancestorsDict.keys():
            fmeta["Ancestors"] = ancestorsDict.keys()
      res = fc.getFileDescendents([clip.filename], 1)
      if res["OK"]:
        for dummy_lfn,descendDict in res['Value']['Successful'].items():
          if descendDict.keys():
            fmeta['Descendants'] = descendDict.keys()
    else:
      ext = clip.filename.split(".")[-1]
      fitems = []
      for i in clip.filename.split('.')[:-1]:
        fitems.extend(i.split('_'))
      pid = ''
      if ext == 'stdhep':
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import listDirectory [as 别名]
class DfcOperation:
    '''Class for DFC operation'''

    _username = ''
    _groupname = ''
    _rootdir = ''

    def __init__(self):
        _fcType = 'DataManagement/FileCatalog'
        self.client = FileCatalogClient(_fcType)

    def getUserName(self):
        '''get username'''
        if not DfcOperation._username:
            DfcOperation._username = getProxyInfo()['Value'].get('username', 'unknown')
        return DfcOperation._username

    def getGroupName(self):
        '''get groupname'''
        if not DfcOperation._groupname:
            DfcOperation._groupname = getProxyInfo()['Value'].get('group', 'unknown')
        return DfcOperation._groupname

    def setRootDir(self, rootDir):
        DfcOperation._rootdir = rootDir

    def getRootDir(self):
        if not DfcOperation._rootdir:
            if self.getGroupName() == 'production':
                DfcOperation._rootdir = self.getOfficialRootDir()
            DfcOperation._rootdir = self.getUserRootDir()
        return DfcOperation._rootdir

    def getOfficialRootDir(self):
        return '/bes'

    def getUserRootDir(self):
        '''get user's initial root directory'''
        username = self.getUserName()
        initial = username[:1]

        vo = getVO()
        if not vo:
            vo = 'bes'

        ops = Operations(vo = vo)
        user_prefix = ops.getValue('LFNUserPrefix', 'user')

        basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username

        return basePath

    def isDirExists(self,dir):
        '''check whether dir on DFC exists'''
        result = self.client.listDirectory(dir)
        if result['OK'] and result['Value']['Successful']:
            return True
        return False

    def validateDir(self, dir):
        '''if the dir on DFC does not exist, create it'''
        if not self.isDirExists(dir):
            logger.debug('Creating dir: %s', dir)
            return self.client.createDirectory(dir)

    def _getDatasetPrefix(self):
        if self.getGroupName() == 'production':
            return 'Prod'
        return 'User_' + self.getUserName()

    def createDataset(self, metadata):
        metaDic = {}
        metaDic['resonance'] = metadata.get('resonance', 'unknown')
        metaDic['bossVer']   = metadata.get('bossVer',   'xxx')
        metaDic['eventType'] = metadata.get('eventType', 'unknown')
        metaDic['round']     = metadata.get('round',     'roundxx')
        metaDic['streamId']  = metadata.get('streamId',  'streamxxx')
        metaDic['dataType']  = metadata.get('dataType',  'unknown')
        metaDic['Path']      = self.getRootDir() + ('/Log' if metadata.get('dataType', 'unknown').lower() == 'log' else '/File')
        runFrom = metadata.get('runFrom', 0)
        runTo = metadata.get('runTo', 0)
        metaDic['runL']      = {'>=': runFrom}
        metaDic['runH']      = {'<=': runTo}

        datasetName = '%s_%s_%s_%s_%s_%s_%s_%s_%s' % (self._getDatasetPrefix(),
                                                      metaDic['resonance'], metaDic['bossVer'], metaDic['eventType'], metaDic['round'],
                                                      runFrom, runTo, metaDic['streamId'], metaDic['dataType'])
        result = self.client.addDataset({'/dataset/'+datasetName: metaDic})
        if not result['OK']:
            logger.warning("Can not create dataset: %s", result['Message'])

        return datasetName

    def getDirName(self, metadata):
        return self._getDirNames(metadata)[6]

    def _getDirNames(self, metadata):
        rootDir = self.getRootDir()
        dir_start = rootDir + ('/Log' if metadata.get('dataType', 'unknown').lower() == 'log' else '/File')
        dir_resonance = dir_start + '/' + metadata.get('resonance', 'unknown')
#.........这里部分代码省略.........