Python FileCatalogClient.addFileAncestors方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFileAncestors [as 别名]

#.........这里部分代码省略.........
      self.prodOutputLFNs = []
      
    if 'SoftwarePackages' in self.workflow_commons:
      self.swpackages = self.workflow_commons['SoftwarePackages'].split(";")

    self.nbofevents = self.NumberOfEvents #comes from ModuleBase
    if 'ILDConfigPackage' in self.workflow_commons:
      self.ildconfig = self.workflow_commons['ILDConfigPackage']
    return S_OK('Parameters resolved')
  
  def execute(self):
    LOG.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      LOG.error("Failed to resolve input parameters:", result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      LOG.info('No production data found, so no metadata registration to be done')
      return S_OK("No files' metadata to be registered")
    
    LOG.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))

    #TODO: What meta data should be stored at file level?

    for files in self.prodOutputLFNs:
      meta = {}  

      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if 'file_number_of_event_relation' in self.workflow_commons:
          if os.path.basename(files) in self.workflow_commons['file_number_of_event_relation']:
            nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
        meta.update(nbevts) 
        
      if 'CrossSection' in self.inputdataMeta:
        xsec = {'CrossSection':self.inputdataMeta['CrossSection']}
        meta.update(xsec)
        
      if 'CrossSectionError' in self.inputdataMeta:
        xsec = {'CrossSectionError':self.inputdataMeta['CrossSectionError']}
        meta.update(xsec)
        
      if 'GenProcessID' in self.inputdataMeta:
        fmeta = {'GenProcessID':self.inputdataMeta['GenProcessID']}
        meta.update(fmeta)
        
      if 'GenProcessType' in self.inputdataMeta:
        fmeta = {'GenProcessType':self.inputdataMeta['GenProcessType']}
        meta.update(fmeta)
        
      if 'GenProcessName' in self.inputdataMeta:
        fmeta = {'GenProcessName':self.inputdataMeta['GenProcessName']}
        meta.update(fmeta)
        
      if 'Luminosity' in self.inputdataMeta:
        fmeta = {'Luminosity':self.inputdataMeta['Luminosity']}
        meta.update(fmeta)
        
      if 'BeamParticle1' in self.inputdataMeta:
        fmeta = {'BeamParticle1':self.inputdataMeta['BeamParticle1'],
                 'BeamParticle2':self.inputdataMeta['BeamParticle2']}
        meta.update(fmeta)
        
      if 'PolarizationB1' in self.inputdataMeta:
        fmeta = {'PolarizationB1':self.inputdataMeta['PolarizationB1'],
                 'PolarizationB2':self.inputdataMeta['PolarizationB2']}
        meta.update(fmeta)
        
      if self.ildconfig:
        fmeta = {'ILDConfig' : self.ildconfig}
        meta.update(fmeta)
      
      if self.WorkflowStartFrom:
        meta.update({"FirstEventFromInput":self.WorkflowStartFrom})

        
      if self.enable:
        res = self.filecatalog.setMetadata(files, meta)
        if not res['OK']:
          LOG.error('Could not register metadata:', res['Message'])
          return res
      LOG.info("Registered %s with tags %s" % (files, meta))
      
      ###Now, set the ancestors
      if self.InputData:
        inputdata = self.InputData
        if self.enable:
          res = self.filecatalog.addFileAncestors({files : {'Ancestors' : inputdata}})
          if not res['OK']:
            LOG.error('Registration of Ancestors failed for:', str(files))
            LOG.error('because of', res['Message'])
            return res

    return S_OK('Output data metadata registered in catalog')

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFileAncestors [as 别名]

#.........这里部分代码省略.........
    self.luminosity = 0
    self.sel_eff = 0
    self.cut_eff = 0
    self.add_info = ''
    self.filecatalog = FileCatalogClient()
    self.filemeta = {}

  def applicationSpecificInputs(self):
    if self.step_commons.has_key('Enable'):
      self.enable = self.step_commons['Enable']
      if not type(self.enable) == type(True):
        self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
        self.enable = False
        
    if self.workflow_commons.has_key('ProductionOutputData'):
      self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
    else:
      self.prodOutputLFNs = []
      
    self.nbofevents = self.NumberOfEvents
    if self.workflow_commons.has_key('Luminosity'):
      self.luminosity = self.workflow_commons['Luminosity']
    
    ##Additional info: cross section only for the time being, comes from WHIZARD
    if self.workflow_commons.has_key('Info'):
      if 'stdhepcut' in self.workflow_commons['Info']:
        self.sel_eff = self.workflow_commons['Info']['stdhepcut']['Reduction']
        self.cut_eff = self.workflow_commons['Info']['stdhepcut']['CutEfficiency']
        del self.workflow_commons['Info']['stdhepcut']
      self.add_info = DEncode.encode(self.workflow_commons['Info'])
    
    return S_OK('Parameters resolved')
  
  def execute(self):
    """ Run the module
    """
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error("failed to resolve input parameters:", result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s" % "\n".join(self.prodOutputLFNs))

    for files in self.prodOutputLFNs:
      metafiles = {}

      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if self.workflow_commons.has_key('file_number_of_event_relation'):
          if self.workflow_commons['file_number_of_event_relation'].has_key(os.path.basename(files)):
            nbevts['NumberOfEvents'] = self.workflow_commons['file_number_of_event_relation'][os.path.basename(files)]
        metafiles.update(nbevts)  
      if self.luminosity:
        metafiles.update({'Luminosity': self.luminosity})
      
      if self.sel_eff:
        metafiles.update({'Reduction':self.sel_eff})
      if self.cut_eff:
        metafiles.update({'CutEfficiency': self.cut_eff})  
      if self.add_info:
        metafiles.update({'AdditionalInfo':self.add_info})
        
      elif 'AdditionalInfo' in self.inputdataMeta:
        metafiles.update({'AdditionalInfo':self.inputdataMeta['AdditionalInfo']})
      
      if 'CrossSection' in self.inputdataMeta:
        metafiles.update({'CrossSection':self.inputdataMeta['CrossSection']})
      
      if self.WorkflowStartFrom:
        metafiles.update({"FirstEventFromInput":self.WorkflowStartFrom})
      
      if self.enable:
        res = self.filecatalog.setMetadata(files, metafiles)
        if not res['OK']:
          self.log.error(res['Message'])
          self.log.error('Could not register metadata for %s' % files)
          return res
        
      self.log.info("Registered %s with tags %s" % (files, metafiles))
      
      ###Now, set the ancestors
      if self.InputData:
        if self.enable:
          res = self.filecatalog.addFileAncestors({ files : {'Ancestors' : self.InputData } })
          if not res['OK']:
            self.log.error('Registration of Ancestors for %s failed' % files)
            self.log.error('Because of ', res['Message'])
            return res

    return S_OK('Output data metadata registered in catalog')

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import addFileAncestors [as 别名]

#.........这里部分代码省略.........
  
  def execute(self):
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error(result['Message'])
      return result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if len(self.prodOutputLFNs) == 0:
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s"% string.join(self.prodOutputLFNs, 
                                                                                                "\n"))

    for files in self.prodOutputLFNs:
#      elements = files.split("/")
#      metaprodid = {}
#      metaforfiles = {}
      meta = {}  
#      metaen={}
#      metaen['Energy']=elements[5]
#      meta.update(metaen)
#      energy = string.join(elements[0:6],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(energy,metaen)
#        if not res['OK']:
#          self.log.error('Could not register metadata Energy, with value %s for %s'%(elements[4],energy))
#          return res      
#      metaevt={}
#      metaevt['EvtType']=elements[6]
#      meta.update(metaevt)
#      evttype = string.join(elements[0:7],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(evttype,metaevt)
#        if not res['OK']:
#          self.log.error('Could not register metadata EvtType, with value %s for %s'%(elements[5],evttype))
#          return res
#      prodid = ''
#      
#      metadat={}
#      metadat['Datatype']=elements[7]
#      datatype = string.join(elements[0:8],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(datatype,metadat)
#        if not res['OK']:
#          self.log.error('Could not register metadata Datatype, with value %s for %s'%(elements[7],datatype))
#          return res 
#      metaprodid['ProdID'] = elements[8]
#      prodid = string.join(elements[0:9],"/")
#      if self.enable:
#        res = self.filecatalog.setMetadata(prodid,metaprodid)
#        if not res['OK']:
#          self.log.error('Could not register metadata ProdID, with value %s for %s'%(elements[8],prodid))
#          return res
#        
      if self.nbofevents:
        nbevts = {}
        nbevts['NumberOfEvents'] = self.nbofevents
        if self.enable:
          res = self.filecatalog.setMetadata(files, nbevts)
          if not res['OK']:
            self.log.error('Could not register metadata NumberOfEvents, with value %s for %s' % (self.nbofevents, 
                                                                                                 files))
            return res
        meta.update(nbevts)
      if self.luminosity:
        lumi = {}
        lumi['Luminosity'] = self.luminosity
        if self.enable:
          res = self.filecatalog.setMetadata(files, lumi)
          if not res['OK']:
            self.log.error('Could not register metadata Luminosity, with value %s for %s'%(self.luminosity, files))
            return res
        meta.update(lumi)
#      meta.update(metaprodid)
      
      
      
      self.log.info("Registered %s with tags %s"%(files, meta))
      
      ###Now, set the ancestors
      if self.InputData:
        inputdata = self.InputData
        if self.enable:
          res = self.filecatalog.addFileAncestors({files : {'Ancestors' : inputdata}})
          if not res['OK']:
            self.log.error('Registration of Ancestors for %s failed' % files)
            return res
      # FIXME: in next DIRAC release, remove loop and replace key,value below by meta  
      #res = self.filecatalog.setMetadata(os.path.dirname(files),meta)
      #if not res['OK']:
      #  self.log.error('Could not register metadata %s for %s'%(meta, files))
      #  return res
    
    return S_OK('Output data metadata registered in catalog')