Python FileCatalogClient.setMetadata方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
class DBDGenRegisterOutputData(ModuleBase):
  """ Normally, was supposed to be used to produce the DBD gen level files. Dropped in the end.
  """
  def __init__(self):
    super(DBDGenRegisterOutputData).__init__()
    self.version = "DBDGenRegisterOutputData v1"
    self.log = gLogger.getSubLogger( "DBDGenRegisterOutputData" )
    self.commandTimeOut = 10 * 60
    self.enable = True
    self.fc = FileCatalogClient()
    self.nbofevents = 0
    self.prodOutputLFNs = []
    
  def applicationSpecificInputs(self):
    if self.workflow_commons.has_key('ProductionOutputData'):
      self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
      
    if self.workflow_commons.has_key('NbOfEvts'):
      self.nbofevents = self.workflow_commons[ 'NbOfEvts']          
    return S_OK("Parameters resolved")
      
  def execute(self):
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error(result['Message'])
      return result
    
    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if not len(self.prodOutputLFNs):
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s" % '\n'.join(self.prodOutputLFNs))
    
    for files in self.prodOutputLFNs:
      metadict = {}
      metadict['NumberOfEvents'] = self.nbofevents
      path = files
      
      res = self.fc.setMetadata(files, metadict)
      if not res['OK']:
        self.log.error("Could not register %s for %s"%(metadict, path))
        return res
    
    return S_OK()

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
def runOne ():
    fc = FileCatalogClient()
    for mokkaJobNumber in range(39,45):
        for eventType in ['Z_uds','Photon','Kaon0L','Muon']:
            energies = []
            if eventType == 'Z_uds':
                energies = [91,200,360,500,750,1000,2000,3000]
            elif eventType == 'Photon':
                energies = [10]
            elif eventType == 'Muon':
                energies = [10]
            elif eventType == 'Kaon0L':
                energies = [1,2,3,4,5,6,7,8,9,10,15,20,25,30,35,40,45,50]
            for energy in energies:
                path = '/ilc/user/s/sgreen/0804/HCalStudiessg568/GJN' + str(mokkaJobNumber) + '/' + eventType + '/' + str(energy) + 'GeV'
                pathdict = {'path':path, 'meta':{'MokkaJobNumber':mokkaJobNumber, 'Type':eventType, 'Energy':energy}}
                res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    return

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
def addUserToFC(clip):
  """
  Add the user to the filecatalog
  Try to figure out in which VOs the user must be, and create the catalog entries accordingly
  """
  from DIRAC.ConfigurationSystem.Client.Helpers  import Registry
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
  fc = FileCatalogClient()
  res = fc.addUser(clip.uname)
  if not res['OK']:
    gLogger.error("Failed to add user to FC, but it does not really matter:", res['Message'])
    gLogger.error("Add by hand (in the FC-CLI: user add %s)" %(clip.uname) )

  bpath = ''
  for grp in clip.groups:
    bpath = '/'
    voName = Registry.getVOForGroup(grp)
    if not voName:
      gLogger.error("NO VO for group", grp )
      continue
    bpath += voName+"/"
    lfnprefix = Operations( vo = voName ).getValue("LFNUserPrefix")
    if lfnprefix:
      bpath += lfnprefix+"/"
    bpath += clip.uname[0]+"/"+clip.uname

    res = fc.createDirectory(bpath)
    if not res['OK']:
      gLogger.error(res['Message'])
      continue

    res = fc.changePathGroup({ bpath: grp }, False)
    if not res['OK']:
      gLogger.error(res['Message'])

    res = fc.changePathOwner({ bpath: clip.uname }, False)
    if not res['OK']:
      gLogger.error(res['Message'])

    res = fc.setMetadata(bpath, {"Owner":clip.uname})
    if not res['OK']:
      gLogger.error(res['Message'])

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
import os

from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient

energies = [10,20,50,100,200,500]
particles = ['Photon','Kaon0L']

fc = FileCatalogClient()

for particle in particles:
    for energy in energies:
        path = '/ilc/user/s/sgreen/HEPEvtFiles/' + str(particle) + '/' + str(energy) + 'GeV'
        pathdict = {'path':path, 'meta':{'JobDescription':'HEPEvt','EvtType':str(particle),'Energy':energy}}
        res = fc.setMetadata(pathdict['path'], pathdict['meta'])

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
class RegisterOutputData( ModuleBase ):
  """ At the end of a production Job, we need to register meta data info for the files. 
  """
  def __init__(self):
    super(RegisterOutputData, self).__init__()
    self.version = "RegisterOutputData v1"
    self.log = gLogger.getSubLogger( "RegisterOutputData" )
    self.commandTimeOut = 10*60
    self.enable = True
    self.prodOutputLFNs = []
    self.nbofevents = 0
    self.luminosity = 0
    self.sel_eff = 0
    self.cut_eff = 0
    self.add_info = ''
    self.filecatalog = FileCatalogClient()
    self.filemeta = {}

  def applicationSpecificInputs(self):
    if self.step_commons.has_key('Enable'):
      self.enable = self.step_commons['Enable']
      if not type(self.enable) == type(True):
        self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
        self.enable = False
        
    if self.workflow_commons.has_key('ProductionOutputData'):
      self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
    else:
      self.prodOutputLFNs = []
      
    self.nbofevents = self.NumberOfEvents
    if self.workflow_commons.has_key('Luminosity'):
      self.luminosity = self.workflow_commons['Luminosity']
    
    ##Additional info: cross section only for the time being, comes from WHIZARD
    if self.workflow_commons.has_key('Info'):
      if 'stdhepcut' in self.workflow_commons['Info']:
        self.sel_eff = self.workflow_commons['Info']['stdhepcut']['Reduction']
        self.cut_eff = self.workflow_commons['Info']['stdhepcut']['CutEfficiency']
        del self.workflow_commons['Info']['stdhepcut']
      self.add_info = DEncode.encode(self.workflow_commons['Info'])
    
    return S_OK('Parameters resolved')
  
  def execute(self):
    """ Run the module
    """
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error("failed to resolve input parameters:", result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))

    for files in self.prodOutputLFNs:
      metafiles = {}

      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if self.workflow_commons.has_key('file_number_of_event_relation'):
          if self.workflow_commons['file_number_of_event_relation'].has_key(os.path.basename(files)):
            nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
        metafiles.update(nbevts)  
      if self.luminosity:
        metafiles.update({'Luminosity': self.luminosity})
      
      if self.sel_eff:
        metafiles.update({'Reduction':self.sel_eff})
      if self.cut_eff:
        metafiles.update({'CutEfficiency': self.cut_eff})  
      if self.add_info:
        metafiles.update({'AdditionalInfo':self.add_info})
        
      elif 'AdditionalInfo' in self.inputdataMeta:
        metafiles.update({'AdditionalInfo':self.inputdataMeta['AdditionalInfo']})
      
      if 'CrossSection' in self.inputdataMeta:
        metafiles.update({'CrossSection':self.inputdataMeta['CrossSection']})
      
      if self.WorkflowStartFrom:
        metafiles.update({"FirstEventFromInput":self.WorkflowStartFrom})
      
      if self.enable:
        res = self.filecatalog.setMetadata(files, metafiles)
        if not res['OK']:
          self.log.error(res['Message'])
          self.log.error('Could not register metadata for %s' % files)
          return res
        
      self.log.info("Registered %s with tags %s" % (files, metafiles))
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]

#.........这里部分代码省略.........
      self.prodOutputLFNs = []
      
    if 'SoftwarePackages' in self.workflow_commons:
      self.swpackages = self.workflow_commons['SoftwarePackages'].split(";")

    self.nbofevents = self.NumberOfEvents #comes from ModuleBase
    if 'ILDConfigPackage' in self.workflow_commons:
      self.ildconfig = self.workflow_commons['ILDConfigPackage']
    return S_OK('Parameters resolved')
  
  def execute(self):
    LOG.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      LOG.error("Failed to resolve input parameters:", result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      LOG.info('No production data found, so no metadata registration to be done')
      return S_OK("No files' metadata to be registered")
    
    LOG.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))

    #TODO: What meta data should be stored at file level?

    for files in self.prodOutputLFNs:
      meta = {}  

      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if 'file_number_of_event_relation' in self.workflow_commons:
          if os.path.basename(files) in self.workflow_commons['file_number_of_event_relation']:
            nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
        meta.update(nbevts) 
        
      if 'CrossSection' in self.inputdataMeta:
        xsec = {'CrossSection':self.inputdataMeta['CrossSection']}
        meta.update(xsec)
        
      if 'CrossSectionError' in self.inputdataMeta:
        xsec = {'CrossSectionError':self.inputdataMeta['CrossSectionError']}
        meta.update(xsec)
        
      if 'GenProcessID' in self.inputdataMeta:
        fmeta = {'GenProcessID':self.inputdataMeta['GenProcessID']}
        meta.update(fmeta)
        
      if 'GenProcessType' in self.inputdataMeta:
        fmeta = {'GenProcessType':self.inputdataMeta['GenProcessType']}
        meta.update(fmeta)
        
      if 'GenProcessName' in self.inputdataMeta:
        fmeta = {'GenProcessName':self.inputdataMeta['GenProcessName']}
        meta.update(fmeta)
        
      if 'Luminosity' in self.inputdataMeta:
        fmeta = {'Luminosity':self.inputdataMeta['Luminosity']}
        meta.update(fmeta)
        
      if 'BeamParticle1' in self.inputdataMeta:
        fmeta = {'BeamParticle1':self.inputdataMeta['BeamParticle1'],
                 'BeamParticle2':self.inputdataMeta['BeamParticle2']}
        meta.update(fmeta)
        
      if 'PolarizationB1' in self.inputdataMeta:
        fmeta = {'PolarizationB1':self.inputdataMeta['PolarizationB1'],
                 'PolarizationB2':self.inputdataMeta['PolarizationB2']}
        meta.update(fmeta)
        
      if self.ildconfig:
        fmeta = {'ILDConfig' : self.ildconfig}
        meta.update(fmeta)
      
      if self.WorkflowStartFrom:
        meta.update({"FirstEventFromInput":self.WorkflowStartFrom})

        
      if self.enable:
        res = self.filecatalog.setMetadata(files, meta)
        if not res['OK']:
          LOG.error('Could not register metadata:', res['Message'])
          return res
      LOG.info("Registered %s with tags %s" % (files, meta))
      
      ###Now, set the ancestors
      if self.InputData:
        inputdata = self.InputData
        if self.enable:
          res = self.filecatalog.addFileAncestors({files : {'Ancestors' : inputdata}})
          if not res['OK']:
            LOG.error('Registration of Ancestors failed for:', str(files))
            LOG.error('because of', res['Message'])
            return res

    return S_OK('Output data metadata registered in catalog')

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
  exit()

srcMeta = res["Value"]
pprint(srcMeta)

nseq_from=int(0)
nseq_to=int( 135)
nevents=int(1000)
last_fileseq=int( 135)
last_file_nevents=int( 76)

for i in range(nseq_from, nseq_to+1) : 
  filename=filepref+"_%3.3d.stdhep"%i
  print filename
  thisMeta=srcMeta
  nevents=nevents
  if i == last_fileseq : 
    nevents = last_file_nevents
  thisMeta["NumberOfEvents"]=nevents
  thisMeta["SerialNumber"]=3 
  thisMeta["SubSerialNumber"]=i
#  meta = fc.getFileUserMetadata(gridPath+filename)
  lfn=gridPath+filename
  res = fc.setMetadata(lfn, thisMeta)
  if not res['OK']:
    gLogger.error("Failed to set meta data %s to %s\n" % (lfn, thisMeta), res['Message'] )
  res = fc.getFileUserMetadata(lfn)
  print "Meta added to "+lfn
  pprint(res)
 

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
    print("*")
    print("* Input JSON          : '%s'" % (datapath))
    print("* Output path         : '%s'" % (outputpath))
    print("* DFC output dir.     : '%s'" % (gridoutdir))
    print("*")

    ## The frame properties JSON file - FIXME: check it exists...
    ff = open(datapath, "r")
    #
    fd = json.load(ff)
    ff.close()

    ## The File Catalog client object.
    fc = FileCatalogClient()

    # Loop over the frames and upload them to the DFC.
    for f in fd:

        ## The file name.
        fn = f["frameid"] + ".txt"

        print("* Found           : '%s'" % (fn))

        ## The full LFN for the datafile.
        lfn = gridoutdir + "/" + fn

        metadataresult = fc.setMetadata(lfn, f)
        print("*--> '%s'" % (lfn))
        print metadataresult

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]

#.........这里部分代码省略.........
    if failure:
      return S_ERROR('Error when setting new metadata, already existing metadata disagrees!')
    return S_OK()

  def _registerMetadata(self):
    """Set metadata for given folders.

    Register path and metadata before the production actually runs. This allows for the definition
    of the full chain in 1 go.
    """
    prevent_registration = self.ops.getValue('Production/PreventMetadataRegistration', False)

    if self.dryrun or prevent_registration:
      LOG.notice('Would have created and registered the following\n',
                 '\n '.join([' * %s: %s' % (fPath, val) for fPath, val in self.finalMetaDict.iteritems()]))
      LOG.notice('Would have set this as non searchable metadata', str(self.finalMetaDictNonSearch))
      return S_OK()

    failed = defaultdict(list)
    for path, meta in sorted(self.finalMetaDict.items()):
      res = self._createDirectory(path, failed)
      if not res['OK']:
        continue
      LOG.verbose('Checking to set metadata:', meta)
      metaCopy = dict(meta)
      res = self._checkMetadata(path, metaCopy)
      if not res['OK']:
        return res
      if not metaCopy:
        LOG.verbose('No new metadata to set')
        continue

      LOG.verbose('Setting metadata information: ', '%s: %s' % (path, metaCopy))
      result = self.fc.setMetadata(path.rstrip('/'), metaCopy)
      if not result['OK']:
        LOG.error('Could not preset metadata', str(metaCopy))
        LOG.error('Could not preset metadata', result['Message'])
        failed[path].append(result['Message'])

    for path, meta in sorted(self.finalMetaDictNonSearch.items()):
      res = self._createDirectory(path, failed)
      if not res['OK']:
        continue
      LOG.verbose('Setting non searchable metadata information: ', '%s: %s' % (path, meta))
      result = self.fc.setMetadata(path.rstrip('/'), meta)
      if not result['OK']:
        LOG.error('Could not preset non searchable metadata', str(meta))
        LOG.error('Could not preset non searchable metadata', result['Message'])
        failed[path].append(result['Message'])

    if failed:
      return S_ERROR('Failed to register some metadata: %s' % dict(failed))
    return S_OK()

  def getMetadata(self):
    """ Return the corresponding metadata of the last step
    """
    metadict = {}
    for meta in self.finalMetaDict.values():
      metadict.update(meta)
    if 'NumberOfEvents' in metadict:
      del metadict['NumberOfEvents'] #As this is not supposed to be a searchable thing
    return metadict
  
  def _setProdParameter(self, prodID, pname, pvalue):
    """ Set a production parameter.

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
  def addCatalogEntry(self,catalogdict,prodid = None):
    if not self.created:
      return S_ERROR("You need to have created the production before calling this")
    
    allowedkeys = ['process_id',
#'process_names',
'process_type',
#'CM_energy_in_GeV',
'program_name_version',
'pythia_version',
'stdhep_version',
#'OS_version_build=2.6.18-238.5.1.el5;x86_64;GNU/Linux',
#'OS_version_run=2.6.18-194.32.1.el5;x86_64;GNU/Linux'
#'libc_version=glibc-2.5-49.el5_5.7.x86_64',
#'fortran_version',
'hadronisation_tune',
'tau_decays',
#'beam_particle1',
#'beam_particle2',
'polarization1',
'polarization2',
'luminosity',
'cross_section_in_fb',
'cross_section_error_in_fb',
#'lumi_file',
'file_type',
#'total_number_of_events',
#'number_of_files',
#'file_names',
#'number_of_events_in_files',
'fileurl',
#'logurl',
'comment']
    
    inputdict = {}
    for key in catalogdict.keys():
      if not key in allowedkeys:
        return S_ERROR("No allowed to use this key '%s', please check"%key)
      else:
        inputdict[key]=catalogdict[key]
      
    whizardparams = self.prodparameters['whizardparams']
    processes = whizardparams['process_input']['process_id']
    energy = whizardparams['process_input']['sqrts']
    beam_particle1 = whizardparams['beam_input_1']['particle_name']
    beam_particle2 = whizardparams['beam_input_2']['particle_name']
    currtrans = 0
    if self.currtrans:
      currtrans = self.currtrans.getTransformationID()['Value']
    if prodid:
      currtrans = prodid
    
    self.basepath += "/"+str(currtrans).zfill(8) 
    
    fc = FileCatalogClient()
    
    result = fc.createDirectory(self.basepath)
    if result['OK']:
      if result['Value']['Successful']:
        if result['Value']['Successful'].has_key(self.basepath):
          print "Successfully created directory:", self.basepath
      elif result['Value']['Failed']:
        if result['Value']['Failed'].has_key(self.basepath):  
          print 'Failed to create directory:',result['Value']['Failed'][self.basepath]
    else:
      print 'Failed to create directory:',result['Message']
    
    metadict = {}
    
    metadict['process_names']=processes
    metadict['CM_energy_in_GeV']=energy
    metadict['beam_particle1']=beam_particle1
    metadict['beam_particle2']=beam_particle2
    metadict.update(inputdict)
    
    res = fc.setMetadata(self.basepath,metadict)
    if not res['OK']:
      self.log.error("Could not preset metadata")        
    return S_OK()  

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
class SIDRegisterOutputData(ModuleBase):
  """ Register output data in the FC for the SID productions 
  """
  def __init__(self):
    super(SIDRegisterOutputData, self).__init__()
    self.version = "SIDRegisterOutputData v1"
    self.log = gLogger.getSubLogger( "SIDRegisterOutputData" )
    self.commandTimeOut = 10 * 60
    self.enable = True
    self.prodOutputLFNs = []
    self.swpackages = []
    self.nbofevents = 0
    self.luminosity = 0
    self.filecatalog = FileCatalogClient()

  def applicationSpecificInputs(self):
    if self.step_commons.has_key('Enable'):
      self.enable = self.step_commons['Enable']
      if not type(self.enable) == type(True):
        self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
        self.enable = False
        
    if self.workflow_commons.has_key('ProductionOutputData'):
      self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
    else:
      self.prodOutputLFNs = []
      
    if self.workflow_commons.has_key('SoftwarePackages'):
      self.swpackages = self.workflow_commons['SoftwarePackages'].split(";")

    self.nbofevents = self.NumberOfEvents #comes from ModuleBase
    if self.workflow_commons.has_key('Luminosity'):
      self.luminosity = self.workflow_commons['Luminosity']
    return S_OK('Parameters resolved')
  
  def execute(self):
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error(result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s"% string.join(self.prodOutputLFNs, 
                                                                                                "\n"))

    for files in self.prodOutputLFNs:
#      elements = files.split("/")
#      metaprodid = {}
#      metaforfiles = {}
      meta = {}  
#      metaen={}
#      metaen['Energy']=elements[5]
#      meta.update(metaen)
#      energy = string.join(elements[0:6],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(energy,metaen)
#        if not res['OK']:
#          self.log.error('Could not register metadata Energy, with value %s for %s'%(elements[4],energy))
#          return res      
#      metaevt={}
#      metaevt['EvtType']=elements[6]
#      meta.update(metaevt)
#      evttype = string.join(elements[0:7],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(evttype,metaevt)
#        if not res['OK']:
#          self.log.error('Could not register metadata EvtType, with value %s for %s'%(elements[5],evttype))
#          return res
#      prodid = ''
#      
#      metadat={}
#      metadat['Datatype']=elements[7]
#      datatype = string.join(elements[0:8],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(datatype,metadat)
#        if not res['OK']:
#          self.log.error('Could not register metadata Datatype, with value %s for %s'%(elements[7],datatype))
#          return res 
#      metaprodid['ProdID'] = elements[8]
#      prodid = string.join(elements[0:9],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(prodid,metaprodid)
#        if not res['OK']:
#          self.log.error('Could not register metadata ProdID, with value %s for %s'%(elements[8],prodid))
#          return res
#        
      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if self.enable:
          res = self.filecatalog.setMetadata(files, nbevts)
          if not res['OK']:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]

#.........这里部分代码省略.........
        )
        print result

    def __registerDir(self, dir):
        """Internal function to register a new directory in DFC .
           Returns True for success, False for failure.
        """
        fc = self.client
        result = fc.createDirectory(dir)
        if result["OK"]:
            if result["Value"]["Successful"]:
                if result["Value"]["Successful"].has_key(dir):
                    return S_OK()
                elif result["Value"]["Failed"]:
                    if result["Value"]["Failed"].has_key(dir):
                        print "Failed to create directory %s:%s" % (dir, result["Value"]["Failed"][dir])
                        return S_ERROR(result)
            else:
                return S_ERROR(result)
        else:
            print "Failed to create directory %s:%s" % (dir, result["Message"])
            return S_ERROR(result)

    def __registerFileMetadata(self, lfn, attributes):
        """Internal function to set metadata values on a given lfn. 
          Returns True for success, False for failure.
        """
        metadataDict = {}
        metadataDict["runL"] = attributes["runL"]
        metadataDict["runH"] = attributes["runH"]
        metadataDict["status"] = attributes["status"]
        metadataDict["eventNumber"] = attributes["eventNumber"]
        metadataDict["count"] = attributes["count"]
        result = self.client.setMetadata(lfn, metadataDict)
        if not result["OK"]:
            return S_ERROR()
        else:
            return S_OK()

    def __registerDirMetadata(self, dir, metaDict):
        """Internal function to set metadata to a directory
           Returns True for success, False for failure.
        """
        fc = self.client
        result = fc.setMetadata(dir, metaDict)
        if result["OK"]:
            return S_OK()
        else:
            message = "Error for setting metadata %s to %s: %s" % (metaDict, dir, result["Message"])
            return S_ERROR("Message")

    def __dirExists(self, dir, parentDir):
        """ Internal function to check whether 'dir' is the subdirectory of 'parentDir'
            Returns 1 for Yes, 0 for NO
        """
        fc = self.client
        dir_exists = 0
        result = fc.listDirectory(parentDir)
        if result["OK"]:
            for i, v in enumerate(result["Value"]["Successful"][parentDir]["SubDirs"]):
                if v == dir:
                    dir_exists = 1
                    break
        else:
            print "Failed to list subdirectories of %s:%s" % (parentDir, result["Message"])

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
fc = FileCatalogClient()
for detNumber in detNumbers:
    for recoStage in recoStages:
        for energy in energies:
            path = (
                "/ilc/user/s/sgreen/"
                + jobDescription
                + "/AnalysePerformance/Detector_Model_"
                + str(detNumber)
                + "/Reco_Stage_"
                + str(recoStage)
                + "/"
                + evtType
                + "/"
                + str(energy)
                + "GeV"
            )
            pathdict = {
                "path": path,
                "meta": {
                    "Energy": energy,
                    "EvtType": evtType,
                    "JobDescription": jobDescription,
                    "MokkaJobNumber": detNumber,
                    "ReconstructionVariant": recoStage,
                    "Type": fileType,
                },
            }
            res = fc.setMetadata(pathdict["path"], pathdict["meta"])

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]
def _uploadGenFiles():
  """uploads the generator files"""
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  
  from DIRAC import gLogger, exit as dexit

  if not clip.dir:
    gLogger.error('You need to set the path')
    Script.showHelp()
    dexit(1)
  if not clip.storageElement:
    gLogger.error('You need a storage element')
    Script.showHelp()
    dexit(1)
  
  for key in MANDATORY_KEYS:
    if key not in clip.fmeta:
      gLogger.error("Not all mandatory meta data defined, please check and add key: ", key)
      Script.showHelp()
      dexit(1)
    
  #resolve the inout files
  flist = []
  if os.path.isdir(clip.dir):
    flistd = os.listdir(clip.dir)
    for filename in flistd:
      if filename.count(".stdhep"):
        flist.append( os.path.join(clip.dir, filename) )
  elif os.path.isfile(clip.dir):
    flist.append(clip.dir)
  else:
    gLogger.error("%s is not a file nor a directory" % clip.dir)
    dexit(1)  
  
  gLogger.notice("Will eventually upload %s file(s)" % len(flist))
    
  from DIRAC.Core.Utilities.PromptUser import promptUser
    
  from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
  basepath = Operations().getValue('Production/ILC_ILD/BasePath','')
  if not basepath:
    gLogger.error('Failed to contact CS, please try again')
    dexit(1)
  
  basepath = "/".join(basepath.split("/")[:-2])+"/" #need to get rid of the ild/ part at the end
    
  finalpath = os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass, str(clip.fmeta['GenProcessID']))
  gLogger.notice("Will upload the file(s) under %s" % finalpath)
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)
  
  dirmeta = []
  dirmeta.append({'path':os.path.join(basepath, 'generated'), 'meta':{'Datatype':'gen'}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams), 'meta':{'Energy':clip.energy, 'MachineParams':clip.machineParams}})
  dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass), 'meta':{'EvtClass':clip.evtclass }})
  dirmeta.append({'path':finalpath, 'meta': {'EvtType':clip.evttype ,'Luminosity':clip.lumi, 'ProcessID': clip.fmeta['GenProcessID']} })
  
  final_fname_base = 'E'+clip.energy+"-"+clip.machineParams+".P"+clip.fmeta['GenProcessName']+".G"+clip.fmeta['ProgramNameVersion'] + "."+clip.particle1+clip.pol1+"."+clip.particle2+clip.pol2+".I"+str(clip.fmeta['GenProcessID'])
  gLogger.notice("Final file name(s) will be %s where XX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XX.ext") )
  if not clip.force:
    res = promptUser('Continue?', ['y','n'], 'n')
    if not res['OK']:
      gLogger.error(res['Message'])
      dexit(1)
    if not res['Value'].lower()=='y':
      dexit(0)    

  
  from DIRAC.DataManagementSystem.Client.DataManager import DataManager
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  fc = FileCatalogClient()
  
  for pathdict in dirmeta:
    res = fc.createDirectory(pathdict['path'])
    if not res['OK']:
      gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] )
      dexit(0)

    res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    if not res['OK']:
      gLogger.error( "Failed to set meta data %s to %s\n" %(pathdict['meta'], pathdict['path']), res['Message'] )

  datMan = DataManager()
  for filename in flist:
    fnum = filename.split(".")[-2]
    fext = filename.split(".")[-1]
    final_fname = final_fname_base + '.' + fnum + "." + fext
    gLogger.notice("Uploading %s to" % filename, finalpath+"/"+final_fname)
    if not clip.force:
      res = promptUser('Continue?', ['y','n'], 'n')
      if not res['OK']:
        gLogger.error(res['Message'])
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import setMetadata [as 别名]

#.........这里部分代码省略.........
        runTo = metadata.get('runTo', 0)
        metaDic['runL']      = {'>=': runFrom}
        metaDic['runH']      = {'<=': runTo}

        datasetName = '%s_%s_%s_%s_%s_%s_%s_%s_%s' % (self._getDatasetPrefix(),
                                                      metaDic['resonance'], metaDic['bossVer'], metaDic['eventType'], metaDic['round'],
                                                      runFrom, runTo, metaDic['streamId'], metaDic['dataType'])
        result = self.client.addDataset({'/dataset/'+datasetName: metaDic})
        if not result['OK']:
            logger.warning("Can not create dataset: %s", result['Message'])

        return datasetName

    def getDirName(self, metadata):
        return self._getDirNames(metadata)[6]

    def _getDirNames(self, metadata):
        rootDir = self.getRootDir()
        dir_start = rootDir + ('/Log' if metadata.get('dataType', 'unknown').lower() == 'log' else '/File')
        dir_resonance = dir_start + '/' + metadata.get('resonance', 'unknown')
        dir_bossVer = dir_resonance + '/' + metadata.get('bossVer', 'xxx')
        dir_data_mc = dir_bossVer + ('/data' if metadata.get('streamId', 'streamxxx').lower() == 'stream0' else '/mc')
        dir_eventType = dir_data_mc + '/' +metadata.get('eventType', 'unknown')
        dir_round = dir_eventType + '/' + metadata.get('round', 'roundxx')
        dir_streamId = dir_round + '/' + metadata.get('streamId', 'streamxxx')
        return (dir_start, dir_resonance, dir_bossVer, dir_data_mc, dir_eventType, dir_round, dir_streamId)

    def createDir(self, metadata):
#        (dir_start, dir_resonance, dir_bossVer, dir_data_mc, dir_eventType, dir_round, dir_streamId) = self._getDirNames(metadata)
        dirNames = self._getDirNames(metadata)

        dirs = {}
        dirs[dirNames[0]] = None
        dirs[dirNames[1]] = {'resonance':  metadata.get('resonance', 'unknown')}
        dirs[dirNames[2]] = {'bossVer':    metadata.get('bossVer',   'xxx')}
        dirs[dirNames[3]] = None
        dirs[dirNames[4]] = {'eventType':  metadata.get('eventType', 'unknown')}
        dirs[dirNames[5]] = {'round':      metadata.get('round',     'roundxx')}
        dirs[dirNames[6]] = {'streamId':   metadata.get('streamId',  'streamxxx'),
                             'dataType':   metadata.get('dataType',  'unknown'),}
        if metadata.has_key('jobOptions'):
            dirs[dirNames[6]]['jobOptions'] = self.truncate(metadata['jobOptions'], 4000)
        if metadata.has_key('decayCard'):
            dirs[dirNames[6]]['decayCard'] = self.truncate(metadata['decayCard'], 4000)

        for dirName in dirNames:
            if not self.isDirExists(dirName):
                result = self.client.createDirectory(dirName)
                if not result['OK']:
                    logger.error(result['Message'])
                    continue

                if dirs[dirName]:
                    result = self.client.setMetadata(dirName, dirs[dirName])
                    if not result['OK']:
                        logger.error(result['Message'])

        return dirNames[6]

    def getUnusedStream(self, metadata):
        dirNames = self._getDirNames(metadata)
        dir_round = dirNames[5]
        result = self.client.listDirectory(dir_round)
        streamDirs = []
        if result['OK'] and result['Value']['Successful']:
            for fn in result['Value']['Successful'][dir_round]['SubDirs'].keys():
                streamDirs.append(os.path.basename(fn))

        streamIds = []
        for streamDir in streamDirs:
            if streamDir.startswith('stream'):
                try:
                    streamId = int(streamDir[6:])
                except ValueError:
                    pass
                else:
                    streamIds.append(streamId)

        maxId = max(streamIds) if streamIds else 0
        newId = 1
        if maxId != 999:
            newId = maxId + 1
        else:
            for id in range(1, 10000):
                if not id in streamIds:
                    newId = id
                    break
        return 'stream%03d' % newId


    def truncate(self, srcstr, limit):
        '''Truncate too long strings'''
        last_line = '\n# File is too long. There are still lines not kept here...\n'

        if len(srcstr) <= limit:
            return srcstr

        trunc_pos = limit - len(last_line)
        truncated = srcstr[:trunc_pos] + last_line
        return truncated