Python FileCatalogClient.getMetadataFields方法代码示例

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getMetadataFields [as 别名]

#.........这里部分代码省略.........
          self.finalMetaDict[path] = {"DetectorType" : application.detectortype}
          path += '/'
        elif self.detector:
          path += self.detector
          self.finalMetaDict[path] = {"DetectorType" : self.detector}
          path += '/'
      if not application.datatype and self.datatype:
        application.datatype = self.datatype
      path += application.datatype
      self.finalMetaDict[path] = {'Datatype' : application.datatype}
      LOG.info("Will store the files under", "%s" % path)
      self.finalpaths.append(path)
      extension = 'stdhep'
      if application.datatype in ['SIM', 'REC']:
        extension = 'slcio'
      fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension
      application.setOutputFile(fname, path)  
      
    self.basepath = path

    res = self._updateProdParameters(application)
    if not res['OK']:
      return res
    
    self.checked = True
      
    return S_OK()

  def _updateProdParameters(self, application):
    """ Update the prod parameters stored in the production parameters visible from the web
    """
    try:
      self.prodparameters.update(application.prodparameters)
    except Exception as x:
      return S_ERROR("Exception: %r" % x )

    if hasattr( application, 'extraCLIArguments' ) and application.extraCLIArguments:
      self.prodparameters['extraCLIArguments'] = repr(application.extraCLIArguments)

    return S_OK()

  def _jobSpecificModules(self, application, step):
    return application._prodjobmodules(step)

  def getEnergyPath(self):
    """returns the energy path 250gev or 3tev or 1.4tev etc."""
    energy = Decimal(str(self.energy))
    tD = Decimal('1000.0')
    unit = 'gev' if energy < tD else 'tev'
    energy = energy if energy < tD else energy/tD
    energyPath = ("%1.2f" % energy).rstrip('0').rstrip('.')
    energyPath = energyPath+unit+'/'

    LOG.info("Energy path is: ", energyPath)
    return energyPath


  def _checkMetaKeys( self, metakeys, extendFileMeta=False ):
    """ check if metadata keys are allowed to be metadata

    :param list metakeys: metadata keys for production metadata
    :param bool extendFileMeta: also use FileMetaFields for checking meta keys
    :returns: S_OK, S_ERROR
    """

    res = self.fc.getMetadataFields()
    if not res['OK']:
      LOG.error("Could not contact File Catalog")
      return S_ERROR("Could not contact File Catalog")
    metaFCkeys = res['Value']['DirectoryMetaFields'].keys()
    if extendFileMeta:
      metaFCkeys.extend( res['Value']['FileMetaFields'].keys() )

    for key in metakeys:
      for meta in metaFCkeys:
        if meta != key and meta.lower() == key.lower():
          return self._reportError("Key syntax error %r, should be %r" % (key, meta), name = self.__class__.__name__)
      if key not in metaFCkeys:
        return self._reportError("Key %r not found in metadata keys, allowed are %r" % (key, metaFCkeys))

    return S_OK()

  def _checkFindDirectories( self, metadata ):
    """ find directories by metadata and check that there are directories found

    :param dict metadata: metadata dictionary
    :returns: S_OK, S_ERROR
    """

    res = self.fc.findDirectoriesByMetadata(metadata)
    if not res['OK']:
      return self._reportError("Error looking up the catalog for available directories")
    elif len(res['Value']) < 1:
      return self._reportError('Could not find any directories corresponding to the query issued')
    return res

  def setReconstructionBasePaths( self, recPath, dstPath ):
    """ set the output Base paths for the reconstruction REC and DST files """
    self._recBasePaths['REC'] = recPath
    self._recBasePaths['DST'] = dstPath

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getMetadataFields [as 别名]

#.........这里部分代码省略.........
    def getDirMetaVal(self, dir):
        """list the registed metadata value of the given dir"""
        result = self.client.getDirectoryMetadata(dir)
        if result["OK"]:
            return result["Value"]
        else:
            print "Failed to get meta Value of the directory"
            return {}

    #################################################################################
    # meta fields operations
    #
    def addNewFields(self, fieldName, fieldType, metaType="-d"):
        """add new fields,if metaType is '-f',add file field,
        fileType is datatpye in MySQL notation
      """
        result = self.client.addMetadataField(fieldName, fieldType, metaType)
        if not result["OK"]:
            return S_ERROR(result)
        else:
            return S_OK()

    def deleteMetaField(self, fieldName):
        """delete a exist metafield"""
        result = self.client.deleteMetadataField(fieldName)
        if not result["OK"]:
            return S_ERROR(result)
        else:
            return S_OK()

    def getAllFields(self):
        """get all meta fields,include file metafield and dir metafield.
        """
        result = self.client.getMetadataFields()
        if not result["OK"]:
            return S_ERROR(result["Message"])
        else:
            return result["Value"]

    def registerFileMetadata(self, lfn, metaDict):

        """Add file level metadata to an entry
           True for success, False for failure
           (maybe used to registerNewMetadata
           Example:
           >>>lfn = '/bes/File/psipp/6.6.1/data/all/exp1/run_0011414_All_file001_SFO-1'
           >>>entryDict = {'runL':1000,'runH':898898}
           >>>badger.registerFileMetadata(lfn,entryDict)
           True
        """
        fc = self.client
        result = fc.setMetadata(lfn, metaDict)
        if result["OK"]:
            return S_OK()
        else:
            print "Error:%s" % (result["Message"])
            return S_ERROR(result["Message"])

    #####################################################################
    # File Options
    def registerFile(self, lfn, dfcAttrDict):
        """Register a new file in the DFC.
        
        """
        # TODO:need more tests,if directory of file doesn't exist,
        # addFile will create it without setting any metadata(lin lei)

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getMetadataFields [as 别名]
def createQueryDict(argss):
  """
  Create a proper dictionary, stolen from FC CLI
  """  
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  
  fc = FileCatalogClient()
  result = fc.getMetadataFields()

  if not result['OK']:
    gLogger.error("Failed checking for metadata fields")
    return None
  if not result['Value']:
    gLogger.error('No meta data fields available')
    return None
  typeDict = result['Value']['FileMetaFields']
  typeDict.update(result['Value']['DirectoryMetaFields'])
  metaDict = {}
  contMode = False
  for arg in argss:
    if not contMode:
      operation = ''
      for op in ['>=','<=','>','<','!=','=']:
        if arg.find(op) != -1:
          operation = op
          break
      if not operation:
        gLogger.error("Error: operation is not found in the query")
        return None
        
      name,value = arg.split(operation)
      if not name in typeDict:
        gLogger.error("Error: metadata field %s not defined" % name)
        return None
      mtype = typeDict[name]
    else:
      value += ' ' + arg
      value = value.replace(contMode,'')
      contMode = False  
    
    if value[0] == '"' or value[0] == "'":
      if value[-1] != '"' and value != "'":
        contMode = value[0]
        continue 
    
    if value.find(',') != -1:
      valueList = [ x.replace("'","").replace('"','') for x in value.split(',') ]
      mvalue = valueList
      if mtype[0:3].lower() == 'int':
        mvalue = [ int(x) for x in valueList if not x in ['Missing','Any'] ]
        mvalue += [ x for x in valueList if x in ['Missing','Any'] ]
      if mtype[0:5].lower() == 'float':
        mvalue = [ float(x) for x in valueList if not x in ['Missing','Any'] ]
        mvalue += [ x for x in valueList if x in ['Missing','Any'] ]
      if operation == "=":
        operation = 'in'
      if operation == "!=":
        operation = 'nin'    
      mvalue = {operation:mvalue}  
    else:            
      mvalue = value.replace("'","").replace('"','')
      if not value in ['Missing','Any']:
        if mtype[0:3].lower() == 'int':
          mvalue = int(value)
        if mtype[0:5].lower() == 'float':
          mvalue = float(value)               
      if operation != '=':     
        mvalue = {operation:mvalue}      
                              
    if name in metaDict:
      if type(metaDict[name]) == DictType:
        if type(mvalue) == DictType:
          op,value = mvalue.items()[0]
          if op in metaDict[name]:
            if type(metaDict[name][op]) == ListType:
              if type(value) == ListType:
                metaDict[name][op] = uniqueElements(metaDict[name][op] + value)
              else:
                metaDict[name][op] = uniqueElements(metaDict[name][op].append(value))     
            else:
              if type(value) == ListType:
                metaDict[name][op] = uniqueElements([metaDict[name][op]] + value)
              else:
                metaDict[name][op] = uniqueElements([metaDict[name][op],value])       
          else:
            metaDict[name].update(mvalue)
        else:
          if type(mvalue) == ListType:
            metaDict[name].update({'in':mvalue})
          else:  
            metaDict[name].update({'=':mvalue})
      elif type(metaDict[name]) == ListType:   
        if type(mvalue) == DictType:
          metaDict[name] = {'in':metaDict[name]}
          metaDict[name].update(mvalue)
        elif type(mvalue) == ListType:
          metaDict[name] = uniqueElements(metaDict[name] + mvalue)
        else:
          metaDict[name] = uniqueElements(metaDict[name].append(mvalue))      
      else:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient [as 别名]
# 或者: from DIRAC.Resources.Catalog.FileCatalogClient.FileCatalogClient import getMetadataFields [as 别名]
  def setInputDataQuery(self, metadata):
    """ Define the input data query needed, also get from the data the meta info requested to build the path
    """
    metakeys = metadata.keys()
    client = FileCatalogClient()
    res = client.getMetadataFields()
    if not res['OK']:
      print "Could not contact File Catalog"
      return S_ERROR()
    metaFCkeys = res['Value']['DirectoryMetaFields'].keys()
    for key in metakeys:
      for meta in metaFCkeys:
        if meta != key:
          if meta.lower() == key.lower():
            return self._reportError("Key syntax error %s, should be %s" % (key, meta), name = 'SIDProduction')
      if not metaFCkeys.count(key):
        return self._reportError("Key %s not found in metadata keys, allowed are %s" % (key, metaFCkeys))
    #if not metadata.has_key("ProdID"):
    #  return self._reportError("Input metadata dictionary must contain at least a key 'ProdID' as reference")
    res = client.findDirectoriesByMetadata(metadata)
    if not res['OK']:
      return self._reportError("Error looking up the catalog for available directories")
    elif len(res['Value']) < 1:
      return self._reportError('Could not find any directory corresponding to the query issued')
    dirs = res['Value'].values()
    for mdir in dirs:
      res = self.fc.getDirectoryMetadata(mdir)
      if not res['OK']:
        return self._reportError("Error looking up the catalog for directory metadata")
      compatmeta = res['Value']
      compatmeta.update(metadata)
      
    if compatmeta.has_key('EvtType'):
      if type(compatmeta['EvtType']) in types.StringTypes:
        self.evttype  = compatmeta['EvtType']
      if type(compatmeta['EvtType']) == type([]):
        self.evttype = compatmeta['EvtType'][0]
    else:
      return self._reportError("EvtType is not in the metadata, it has to be!")
    if compatmeta.has_key('NumberOfEvents'):
      if type(compatmeta['NumberOfEvents']) == type([]):
        self.nbevts = int(compatmeta['NumberOfEvents'][0])
      else:
        self.nbevts = int(compatmeta['NumberOfEvents'])

    
    if compatmeta.has_key("Energy"):
      if type(compatmeta["Energy"]) in types.StringTypes:
        self.energycat = compatmeta["Energy"]
      if type(compatmeta["Energy"]) == type([]):
        self.energycat = compatmeta["Energy"][0]

    if compatmeta.has_key("Polarisation"):
      if type(compatmeta["Polarisation"]) in types.StringTypes:
        self.polarization = compatmeta["Polarisation"]
      if type(compatmeta["Polarisation"]) == type([]):
        self.polarization = compatmeta["Polarisation"][0]

    if compatmeta.has_key("MachineParams"):
      if type(compatmeta["MachineParams"]) in types.StringTypes:
        self.machineTuning = compatmeta["MachineParams"]
      if type(compatmeta["MachineParams"]) == type([]):
        self.machineparams = compatmeta["MachineParams"][0]
    gendata = False    
    if compatmeta.has_key('Datatype'):
      if type(compatmeta['Datatype']) in types.StringTypes:
        self.datatype = compatmeta['Datatype']
        if compatmeta['Datatype'] == 'GEN':
          gendata = True
      if type(compatmeta['Datatype']) == type([]):
        self.datatype = compatmeta['Datatype'][0]
        if compatmeta['Datatype'][0] == 'GEN':
          gendata = True

    if compatmeta.has_key("DetectorModel") and not gendata:
      if type(compatmeta["DetectorModel"]) in types.StringTypes:
        self.detector = compatmeta["DetectorModel"]
      if type(compatmeta["DetectorModel"]) == type([]):
        self.detector = compatmeta["DetectorModel"][0]

    self.basename = self.evttype+"_"+self.polarization
        
    self.energy = Decimal(self.energycat)  
    
    self.inputBKSelection = metadata
    self.prodparameters['nbevts'] = self.nbevts 
    self.prodparameters["FCInputQuery"] = self.inputBKSelection

    self.inputdataquery = True
    return S_OK()