Python FileCatalogClient.addFile方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFile [as 别名]
def registerInputData(filepath, size, prefix = '/cepc/lustre-ro'):
    infoDict = {}
    infoDict['PFN'] = ''
    infoDict['Size'] = size
    infoDict['SE'] = 'IHEP-STORM'
    infoDict['GUID'] = commands.getoutput('uuidgen')
    infoDict['Checksum'] = ''
    fileDict = {}
    lfn =  prefix + filepath
    fileDict[lfn] = infoDict
    fcc = FileCatalogClient('DataManagement/FileCatalog')
    rm = ReplicaManager()
    result = {}
    result['lfn'] = lfn
    result['is_registered'] = False
    
    #查询
    for repeatTimes in range(10):
        is_registered = fcc.isFile(lfn)
        if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)):
            break
#         else:
#             continue
    if not is_registered['OK']:#查询失败
        result['is_registered'] = 'querry error. unkown'
        print 'Failed to query %s in DFC. Error message is %s' %(lfn, is_registered['Message'])
        
    if is_registered['Value']['Successful'][lfn]:#已注册
        result['is_registered'] = True
        for repeatTimes in range(10):
            is_removed = rm.removeCatalogFile(lfn)#删除
            if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']):
                result['is_removed'] = True
                break
#             else:
#                 continue
        if not is_removed['OK']:#删除失败
            result['is_removed'] = 'remove error'
            print 'Failed to remove %s from DFC.' %lfn
    #add       
    for repeatTimes in range(10):
        is_added = fcc.addFile(fileDict)#add/register
        if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
            result['OK'] = True
            return result
#         else:
#             continue
    if not is_added['OK']:#add unsuccessfully
        result['OK'] = False
        result['Message'] = is_added['Message']
    elif is_added['Value']['Failed']:
        result['OK'] = False
        result['Message'] = 'Failed to add file' + lfn
    return result

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFile [as 别名]
def insert(attributes):

    dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    #get real eventType and expNum in catalog in amga
    eventType = _get_event_type(attributes["eventType"])
    expNum = _get_exp_num(attributes["expNum"])

    if attributes["streamId"]=="stream0":
        dir=dir+"/data"
    else:
        dir=dir+"/mc"

    client = FileCatalogClient()
    
    # make sure insertion directory exists in DFC
    insertDir = createCatalog(client,dir,eventType,expNum,attributes)

    #do we need to explicitly add LFN attr in DFC?
    # we do need to get the PFN, size, se for addFile (below)
    #TODO pfn = ?
    # need to add PFN attribute to the list, probably in control.py
    pfn = attributes["PFN"]
    lfn=insertDir+"/"+attributes["LFN"]
    attributes["LFN"] = lfn
    #TODO se = ?  temporarily use a made-up SE name
    se = 'BES-TEST' 

    # register files in DFC
    infoDict = {}
    #infoDict['PFN'] = attributes['PFN']
    infoDict['PFN'] = pfn
    infoDict['Size'] = attributes['fileSize']
    infoDict['SE'] = se
   # infoDict['guid'] = attributes['guid']
    infoDict['Checksum'] = ''
    result = client.addFile({lfn:infoDict})
    print result

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFile [as 别名]

#.........这里部分代码省略.........
      if not result['OK']:
        print "Error adding directories:",result['Message']

      dirCount += nDir
      print "Total directories added", dirCount

    e_dirs = time.time() - s

    # Add files

    s = time.time()

    fileDict = resultList['Value']['Successful'][initPath]['Files']
    lfns = {}
    for lfn,info in fileDict.items():


      lfns[lfn] = {}
      lfns[lfn]['Size'] = info['MetaData']['Size']
      lfns[lfn]['Checksum'] = info['MetaData']['Checksum']
      if 'GUID' in info['MetaData']:
        lfns[lfn]['GUID'] = info['MetaData']['GUID']
      else:
        lfns[lfn]['GUID'] = makeGuid()
      lfns[lfn]['Mode'] = info['MetaData']['Mode']
      lfns[lfn]['PFN'] = ''
      owner = getUserNameAndGroup( info['MetaData'] )
      if owner:
        lfns[lfn]['Owner'] = owner

      if info['Replicas']:
        seList = info['Replicas'].keys()
        lfns[lfn]['SE'] = seList

    p_files = time.time() - s
    s = time.time()
    
    nFile = len(lfns)
    nRep = 0
    if nFile:

      for lfn in lfns:
        if 'SE' in lfns[lfn]:
          nRep += len(lfns[lfn]['SE'])

      print "Adding %d files in %s" % (nFile,initPath)
      #print lfns
      
      done = False
      count = 0
      error = False
      while not done:
        count += 1
        result = dfc.addFile(lfns)
        #print result
        if not result['OK']:
          print "Error adding files %d:" % count,result['Message']
          if count > 10:
            print "Completely failed path", initPath
            break
          error = True
          time.sleep(2)
        elif error:
          print "Successfully added files on retry %d" % count
          done = True
        else:
          done = True
        

      fileCount += nFile
      print "Total files added", fileCount


    e_files = time.time() - s

    dfc_time = time.time() - start - lfc_time
    total_time = time.time() - globalStart

    format = "== %s: time lfc/dfc %.2f/%.2f, files %d/%d, dirs %d/%d, time: %.2f/%.2f/%.2f/%.2f %.2f \n"
    outputFile = open('lfc_dfc.out','a')
    outputFile.write( format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time) )
    outputFile.close()

#    print format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time)

    # Go into directories
    if recursive:
      for path in paths:
        result = processDir(path,True,host=host,fcInit=fc,dfcInit=dfc)
        if result['OK']:
          nFile += result['Value'].get('NumberOfFiles',0)
          nDir += result['Value'].get('NumberOfDirectories',0)
          nRep += result['Value'].get('NumberOfReplicas',0)

    resultDict = {}
    resultDict['NumberOfFiles'] = nFile
    resultDict['NumberOfDirectories'] = nDir
    resultDict['NumberOfReplicas'] = nRep
    resultDict['Path'] = initPath
    return S_OK(resultDict)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFile [as 别名]
  dirPath = '/' + '/'.join(map(str, rndTab))
  nbOfFiles = random.randint( 1, maxInsert )
  lfnDict = {}
  for f in xrange(nbOfFiles):
    filename = "%s.txt"%(f)
    lfn = "%s/%s"%(dirPath, filename)
    size = random.randint(1,1000)
    se = random.choice(storageElements)
    guid = "%s%s"%(''.join(map(str,rndTab)),filename)
    checksum = guid
    lfnDict[ lfn ] = { 'PFN' : lfn, 'SE' : se, 'Size' :  size, 'GUID' : guid, 'Checksum' : checksum}


  beforeI = time.time()
  beforeCI = time.clock()
  res = fc.addFile(lfnDict)
  afterCI = time.clock()
  afterI = time.time()
  queryInsertTime = afterI - beforeI
  queryInsertTimeC = afterCI - beforeCI
  extra = "insert "
  if not res['OK']:
    extra += res['Message']
  else:
    extra += "%s %s %s"%(len(lfnDict), len(res['Value'].get('Successful', [])), len(res['Value'].get('Failed', [])))
    pass

  fl.write("%s\t%s\t%s\t%s\n"%(beforeI, afterI, queryInsertTime, extra))
  fl.flush()
  os.fsync(fl)
  fl2.write("%s\t%s\t%s\t%s\n"%(beforeCI, afterCI, queryInsertTimeC, extra))