Python FileCatalogClient.createDirectory方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
def addUserToFC(clip):
  """
  Add the user to the filecatalog
  Try to figure out in which VOs the user must be, and create the catalog entries accordingly
  """
  from DIRAC.ConfigurationSystem.Client.Helpers  import Registry
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
  fc = FileCatalogClient()
  res = fc.addUser(clip.uname)
  if not res['OK']:
    gLogger.error("Failed to add user to FC, but it does not really matter:", res['Message'])
    gLogger.error("Add by hand (in the FC-CLI: user add %s)" %(clip.uname) )

  bpath = ''
  for grp in clip.groups:
    bpath = '/'
    voName = Registry.getVOForGroup(grp)
    if not voName:
      gLogger.error("NO VO for group", grp )
      continue
    bpath += voName+"/"
    lfnprefix = Operations( vo = voName ).getValue("LFNUserPrefix")
    if lfnprefix:
      bpath += lfnprefix+"/"
    bpath += clip.uname[0]+"/"+clip.uname

    res = fc.createDirectory(bpath)
    if not res['OK']:
      gLogger.error(res['Message'])
      continue

    res = fc.changePathGroup({ bpath: grp }, False)
    if not res['OK']:
      gLogger.error(res['Message'])

    res = fc.changePathOwner({ bpath: clip.uname }, False)
    if not res['OK']:
      gLogger.error(res['Message'])

    res = fc.setMetadata(bpath, {"Owner":clip.uname})
    if not res['OK']:
      gLogger.error(res['Message'])

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
def processDir(initPath,recursive=False,host=None,fcInit=None,dfcInit=None):
  """ Process one directory,  possibly recursively 
  """

  global dirCount, fileCount, globalStart, dnCache, roleCache, outputFile

  fc = fcInit
  if not fc:
    fc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
    #fc = FileCatalogClient()
  dfc = dfcInit
  if not dfc:
    #dfc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
    dfc = FileCatalogClient()
  start = time.time()
  initPath = initPath.rstrip("/")
  resultList = fc.listDirectory(initPath,True)
  #print resultList
  #return S_OK()

  lfc_time = (time.time() - start)

  s = time.time()
  print resultList
  if resultList['OK']:
  # Add directories

    if resultList['Value']['Failed']:
      return S_ERROR("Path %s failed: %s" % (initPath,resultList['Value']['Failed'][initPath]))

    dirDict = resultList['Value']['Successful'][initPath]['SubDirs']
    paths = {}
    for path,info in dirDict.items():
      print info
      paths[path] = {}
      paths[path]['Mode'] = info['Mode']
      owner = getUserNameAndGroup( info )
      #owner = getDNandRole( info )
      if owner:
        paths[path]['Owner'] = owner
    #return S_OK()
    p_dirs = time.time() - s
    s = time.time()
    nDir = len(paths)
    if nDir:
      print "Adding %d directories in %s" % (nDir,initPath)
      result = dfc.createDirectory(paths)
      if not result['OK']:
        print "Error adding directories:",result['Message']

      dirCount += nDir
      print "Total directories added", dirCount

    e_dirs = time.time() - s

    # Add files

    s = time.time()

    fileDict = resultList['Value']['Successful'][initPath]['Files']
    lfns = {}
    for lfn,info in fileDict.items():


      lfns[lfn] = {}
      lfns[lfn]['Size'] = info['MetaData']['Size']
      lfns[lfn]['Checksum'] = info['MetaData']['Checksum']
      if 'GUID' in info['MetaData']:
        lfns[lfn]['GUID'] = info['MetaData']['GUID']
      else:
        lfns[lfn]['GUID'] = makeGuid()
      lfns[lfn]['Mode'] = info['MetaData']['Mode']
      lfns[lfn]['PFN'] = ''
      owner = getUserNameAndGroup( info['MetaData'] )
      if owner:
        lfns[lfn]['Owner'] = owner

      if info['Replicas']:
        seList = info['Replicas'].keys()
        lfns[lfn]['SE'] = seList

    p_files = time.time() - s
    s = time.time()
    
    nFile = len(lfns)
    nRep = 0
    if nFile:

      for lfn in lfns:
        if 'SE' in lfns[lfn]:
          nRep += len(lfns[lfn]['SE'])

      print "Adding %d files in %s" % (nFile,initPath)
      #print lfns
      
      done = False
      count = 0
      error = False
      while not done:
        count += 1
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]

#.........这里部分代码省略.........

    # as this is the very last call all applications are registered, so all software packages are known
    #add them the the metadata registration
    for finalpath in self.finalpaths:
      if finalpath not in self.finalMetaDictNonSearch:
        self.finalMetaDictNonSearch[finalpath] = {}
      if "SWPackages" in self.prodparameters:
        self.finalMetaDictNonSearch[finalpath]["SWPackages"] = self.prodparameters["SWPackages"]
        
      if self.metadict_external:
        self.finalMetaDictNonSearch[finalpath].update(self.metadict_external)  
    
    info.append('- Registered metadata: ')
    for path, metadata in sorted( self.finalMetaDict.iteritems() ):
      info.append('    %s = %s' % (path, metadata))
    info.append('- Registered non searchable metadata: ')
    for path, metadata in sorted( self.finalMetaDictNonSearch.iteritems() ):
      info.append('    %s = %s' % (path, metadata))

    infoString = '\n'.join(info)
    self.prodparameters['DetailedInfo'] = infoString
    
    for name, val in self.prodparameters.iteritems():
      result = self._setProdParameter(currtrans, name, val)
      if not result['OK']:
        LOG.error(result['Message'])

    res = self._registerMetadata()
    if not res['OK']:
      LOG.error('Could not register the following directories:', res['Message'])
      return res
    return S_OK()

  def _createDirectory(self, path, failed, mode=0o775):
    """Create the directory at path if it does not exist.

    :param str path: path to check
    :param list failed: list of failed paths
    :param int mode: mode to set for directory
    """
    exists = returnSingleResult(self.fc.isDirectory(path))
    if exists['OK'] and exists['Value']:
      LOG.verbose('Directory already exists:', path)
      return S_OK()
    result = returnSingleResult(self.fc.createDirectory(path))
    if not result['OK']:
      LOG.error('Failed to create directory:', '%s: %s' % (path, result['Message']))
      failed[path].append(result['Message'])
      return S_ERROR()
    LOG.verbose('Successfully created directory:', path)
    res = self.fc.changePathMode({path: mode}, False)
    if not res['OK']:
      LOG.error(res['Message'])
      failed[path].append(res['Message'])
      return S_ERROR()
    LOG.verbose('Successfully changed mode:', path)
    return S_OK()

  def _checkMetadata(self, path, metaCopy):
    """Get existing metadata, if it is the same do not set it again, otherwise return error."""
    existingMetadata = self.fc.getDirectoryUserMetadata(path.rstrip('/'))
    if not existingMetadata['OK']:
      return S_OK()
    failure = False
    for key, value in existingMetadata['Value'].iteritems():
      if key in metaCopy and metaCopy[key] != value:

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
  def addCatalogEntry(self,catalogdict,prodid = None):
    if not self.created:
      return S_ERROR("You need to have created the production before calling this")
    
    allowedkeys = ['process_id',
#'process_names',
'process_type',
#'CM_energy_in_GeV',
'program_name_version',
'pythia_version',
'stdhep_version',
#'OS_version_build=2.6.18-238.5.1.el5;x86_64;GNU/Linux',
#'OS_version_run=2.6.18-194.32.1.el5;x86_64;GNU/Linux'
#'libc_version=glibc-2.5-49.el5_5.7.x86_64',
#'fortran_version',
'hadronisation_tune',
'tau_decays',
#'beam_particle1',
#'beam_particle2',
'polarization1',
'polarization2',
'luminosity',
'cross_section_in_fb',
'cross_section_error_in_fb',
#'lumi_file',
'file_type',
#'total_number_of_events',
#'number_of_files',
#'file_names',
#'number_of_events_in_files',
'fileurl',
#'logurl',
'comment']
    
    inputdict = {}
    for key in catalogdict.keys():
      if not key in allowedkeys:
        return S_ERROR("No allowed to use this key '%s', please check"%key)
      else:
        inputdict[key]=catalogdict[key]
      
    whizardparams = self.prodparameters['whizardparams']
    processes = whizardparams['process_input']['process_id']
    energy = whizardparams['process_input']['sqrts']
    beam_particle1 = whizardparams['beam_input_1']['particle_name']
    beam_particle2 = whizardparams['beam_input_2']['particle_name']
    currtrans = 0
    if self.currtrans:
      currtrans = self.currtrans.getTransformationID()['Value']
    if prodid:
      currtrans = prodid
    
    self.basepath += "/"+str(currtrans).zfill(8) 
    
    fc = FileCatalogClient()
    
    result = fc.createDirectory(self.basepath)
    if result['OK']:
      if result['Value']['Successful']:
        if result['Value']['Successful'].has_key(self.basepath):
          print "Successfully created directory:", self.basepath
      elif result['Value']['Failed']:
        if result['Value']['Failed'].has_key(self.basepath):  
          print 'Failed to create directory:',result['Value']['Failed'][self.basepath]
    else:
      print 'Failed to create directory:',result['Message']
    
    metadict = {}
    
    metadict['process_names']=processes
    metadict['CM_energy_in_GeV']=energy
    metadict['beam_particle1']=beam_particle1
    metadict['beam_particle2']=beam_particle2
    metadict.update(inputdict)
    
    res = fc.setMetadata(self.basepath,metadict)
    if not res['OK']:
      self.log.error("Could not preset metadata")        
    return S_OK()  

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
def _uploadGenFiles():
  """uploads the generator files"""
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  
  from DIRAC import gLogger, exit as dexit

  if not clip.dir:
    gLogger.error('You need to set the path')
    Script.showHelp()
    dexit(1)
  if not clip.storageElement:
    gLogger.error('You need a storage element')
    Script.showHelp()
    dexit(1)
  
  for key in MANDATORY_KEYS:
    if key not in clip.fmeta:
      gLogger.error("Not all mandatory meta data defined, please check and add key: ", key)
      Script.showHelp()
      dexit(1)
    
  #resolve the inout files
  flist = []
  if os.path.isdir(clip.dir):
    flistd = os.listdir(clip.dir)
    for filename in flistd:
      if filename.count(".stdhep"):
        flist.append( os.path.join(clip.dir, filename) )
  elif os.path.isfile(clip.dir):
    flist.append(clip.dir)
  else:
    gLogger.error("%s is not a file nor a directory" % clip.dir)
    dexit(1)  
  
  gLogger.notice("Will eventually upload %s file(s)" % len(flist))
    
  from DIRAC.Core.Utilities.PromptUser import promptUser
    
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
  basepath = Operations().getValue('Production/ILC_ILD/BasePath','')
  if not basepath:
    gLogger.error('Failed to contact CS, please try again')
    dexit(1)
  
  basepath = "/".join(basepath.split("/")[:-2])+"/" #need to get rid of the ild/ part at the end
    
  finalpath = os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass, str(clip.fmeta['GenProcessID']))
  gLogger.notice("Will upload the file(s) under %s" % finalpath)
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)
  
  dirmeta = []
  dirmeta.append({'path':os.path.join(basepath, 'generated'), 'meta':{'Datatype':'gen'}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams), 'meta':{'Energy':clip.energy, 'MachineParams':clip.machineParams}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass), 'meta':{'EvtClass':clip.evtclass }})
  dirmeta.append({'path':finalpath, 'meta': {'EvtType':clip.evttype ,'Luminosity':clip.lumi, 'ProcessID': clip.fmeta['GenProcessID']} })
  
  final_fname_base = 'E'+clip.energy+"-"+clip.machineParams+".P"+clip.fmeta['GenProcessName']+".G"+clip.fmeta['ProgramNameVersion'] + "."+clip.particle1+clip.pol1+"."+clip.particle2+clip.pol2+".I"+str(clip.fmeta['GenProcessID'])
  gLogger.notice("Final file name(s) will be %s where XX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XX.ext") )
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)    

  
  from DIRAC.DataManagementSystem.Client.DataManager import DataManager
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  
  for pathdict in dirmeta:
    res = fc.createDirectory(pathdict['path'])
    if not res['OK']:
      gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] )
      dexit(0)

    res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    if not res['OK']:
      gLogger.error( "Failed to set meta data %s to %s\n" %(pathdict['meta'], pathdict['path']), res['Message'] )

  datMan = DataManager()
  for filename in flist:
    fnum = filename.split(".")[-2]
    fext = filename.split(".")[-1]
    final_fname = final_fname_base + '.' + fnum + "." + fext
    gLogger.notice("Uploading %s to" % filename, finalpath+"/"+final_fname)
    if not clip.force:
      res = promptUser('Continue?', ['y','n'], 'n')
      if not res['OK']:
        gLogger.error(res['Message'])
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
class DfcOperation:
    '''Class for DFC operation'''

    _username = ''
    _groupname = ''
    _rootdir = ''

    def __init__(self):
        _fcType = 'DataManagement/FileCatalog'
        self.client = FileCatalogClient(_fcType)

    def getUserName(self):
        '''get username'''
        if not DfcOperation._username:
            DfcOperation._username = getProxyInfo()['Value'].get('username', 'unknown')
        return DfcOperation._username

    def getGroupName(self):
        '''get groupname'''
        if not DfcOperation._groupname:
            DfcOperation._groupname = getProxyInfo()['Value'].get('group', 'unknown')
        return DfcOperation._groupname

    def setRootDir(self, rootDir):
        DfcOperation._rootdir = rootDir

    def getRootDir(self):
        if not DfcOperation._rootdir:
            if self.getGroupName() == 'production':
                DfcOperation._rootdir = self.getOfficialRootDir()
            DfcOperation._rootdir = self.getUserRootDir()
        return DfcOperation._rootdir

    def getOfficialRootDir(self):
        return '/bes'

    def getUserRootDir(self):
        '''get user's initial root directory'''
        username = self.getUserName()
        initial = username[:1]

        vo = getVO()
        if not vo:
            vo = 'bes'

        ops = Operations(vo = vo)
        user_prefix = ops.getValue('LFNUserPrefix', 'user')

        basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username

        return basePath

    def isDirExists(self,dir):
        '''check whether dir on DFC exists'''
        result = self.client.listDirectory(dir)
        if result['OK'] and result['Value']['Successful']:
            return True
        return False

    def validateDir(self, dir):
        '''if the dir on DFC does not exist, create it'''
        if not self.isDirExists(dir):
            logger.debug('Creating dir: %s', dir)
            return self.client.createDirectory(dir)

    def _getDatasetPrefix(self):
        if self.getGroupName() == 'production':
            return 'Prod'
        return 'User_' + self.getUserName()

    def createDataset(self, metadata):
        metaDic = {}
        metaDic['resonance'] = metadata.get('resonance', 'unknown')
        metaDic['bossVer']   = metadata.get('bossVer',   'xxx')
        metaDic['eventType'] = metadata.get('eventType', 'unknown')
        metaDic['round']     = metadata.get('round',     'roundxx')
        metaDic['streamId']  = metadata.get('streamId',  'streamxxx')
        metaDic['dataType']  = metadata.get('dataType',  'unknown')
        metaDic['Path']      = self.getRootDir() + ('/Log' if metadata.get('dataType', 'unknown').lower() == 'log' else '/File')
        runFrom = metadata.get('runFrom', 0)
        runTo = metadata.get('runTo', 0)
        metaDic['runL']      = {'>=': runFrom}
        metaDic['runH']      = {'<=': runTo}

        datasetName = '%s_%s_%s_%s_%s_%s_%s_%s_%s' % (self._getDatasetPrefix(),
                                                      metaDic['resonance'], metaDic['bossVer'], metaDic['eventType'], metaDic['round'],
                                                      runFrom, runTo, metaDic['streamId'], metaDic['dataType'])
        result = self.client.addDataset({'/dataset/'+datasetName: metaDic})
        if not result['OK']:
            logger.warning("Can not create dataset: %s", result['Message'])

        return datasetName

    def getDirName(self, metadata):
        return self._getDirNames(metadata)[6]

    def _getDirNames(self, metadata):
        rootDir = self.getRootDir()
        dir_start = rootDir + ('/Log' if metadata.get('dataType', 'unknown').lower() == 'log' else '/File')
        dir_resonance = dir_start + '/' + metadata.get('resonance', 'unknown')
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import createDirectory [as 别名]
  gLogger.notice("Final file name(s) will be %s where XXX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XXX.ext") )
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)    

  
  from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  
  for pathdict in dirmeta:
    res = fc.createDirectory(pathdict['path'])
    if not res['OK']:
      gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] )
      dexit(0)

    res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    if not res['OK']:
      gLogger.error("Failed to set meta data %s to %s" %(pathdict['meta'], pathdict['path']))

  rm = ReplicaManager()
  for f in flist:
    fnum = f.split(".")[-2]
    fext = f.split(".")[-1]
    final_fname = final_fname_base + '.' + fnum + "." + fext
    gLogger.notice("Uploading %s to" % f, finalpath+"/"+final_fname)
    if not clip.force: