Python ClassAd.insertAttributeVectorString方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import insertAttributeVectorString [as 别名]
  def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ):
    """This method sends jobs to the task queue agent and if candidate sites
       are defined, updates job JDL accordingly.
    """

    reqJDL = classAdJob.get_expression( 'JobRequirements' )
    classAddReq = ClassAd( reqJDL )

    if siteCandidates:
      classAddReq.insertAttributeVectorString( 'Sites', siteCandidates )
    if bannedSites:
      classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites )

    if classAdJob.lookupAttribute( "SubmitPools" ):
      classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) )

    if classAdJob.lookupAttribute( "GridMiddleware" ):
      classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) )

    if classAdJob.lookupAttribute( "PilotTypes" ):
      classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) )
    #HAck to migrate old jobs to new ones.
    #DELETE ON 08/09
    else:
      if classAdJob.lookupAttribute( "PilotType" ):
        classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) )

    if classAdJob.lookupAttribute( "JobType" ):
      jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ]
      classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes )

    #Required CE's requirements
    gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ]
    if gridCEs:
      classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs )

    if siteCandidates:
      sites = ','.join( siteCandidates )
      classAdJob.insertAttributeString( "Site", sites )

    reqJDL = classAddReq.asJDL()
    classAdJob.insertAttributeInt( 'JobRequirements', reqJDL )

    jdl = classAdJob.asJDL()
    result = self.jobDB.setJobJDL( job, jdl )
    if not result['OK']:
      return result

    if siteCandidates:
      if len( siteCandidates ) == 1:
        self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) )
        self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] )
      elif bannedSites:
        remainingSites = []
        for site in siteCandidates:
          if not site in bannedSites:
            remainingSites.append( site )
        if remainingSites:
          if len( remainingSites ) == 1:
            self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) )
            self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] )
          else:
            self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) )
            result = self.jobDB.getJobAttribute(job,'Site')
            siteGroup = "Multiple"  
            if result['OK']:
              if result['Value'].startswith('Group'):
                siteGroup = result['Value']              
            self.jobDB.setJobAttribute( job, 'Site', siteGroup )
      else:
        self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) )
        result = self.jobDB.getJobAttribute(job,'Site')
        siteGroup = "Multiple"  
        if result['OK']:
          if result['Value'].startswith('Group'):
            siteGroup = result['Value']              
        self.jobDB.setJobAttribute( job, 'Site', siteGroup )
    else:
      self.log.verbose( 'All sites are eligible for job %s' % job )
      self.jobDB.setJobAttribute( job, 'Site', 'ANY' )

    return self.setNextOptimizer( job )

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import insertAttributeVectorString [as 别名]

#.........这里部分代码省略.........
        finalInputData = extraFiles
        if currentFiles:
          finalInputData = currentFiles + ';' + extraFiles
        uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
        paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
        self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
      else:
        paramsDict['InputData'] = {}
        paramsDict['InputData']['value'] = extraFiles
        paramsDict['InputData']['type'] = 'JDL'

    # Handle here the Parametric values
    if self.parametric:
      for pType in ['InputData', 'InputSandbox']:
        if self.parametric.has_key( pType ):
          if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
            pData = self.parametric[pType]
            # List of lists case
            currentFiles = paramsDict[pType]['value'].split( ';' )
            tmpList = []
            if type( pData[0] ) == list:
              for pElement in pData:
                tmpList.append( currentFiles + pElement )
            else:
              for pElement in pData:
                tmpList.append( currentFiles + [pElement] )
            self.parametric[pType] = tmpList

          paramsDict[pType] = {}
          paramsDict[pType]['value'] = "%s"
          paramsDict[pType]['type'] = 'JDL'
          self.parametric['files'] = self.parametric[pType]
          arguments.append( ' -p Parametric' + pType + '=%s' )
          break

      if self.parametric.has_key( 'files' ):
        paramsDict['Parameters'] = {}
        paramsDict['Parameters']['value'] = self.parametric['files']
        paramsDict['Parameters']['type'] = 'JDL'
      if self.parametric.has_key( 'GenericParameters' ):
        paramsDict['Parameters'] = {}
        paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
        paramsDict['Parameters']['type'] = 'JDL'
        arguments.append( ' -p ParametricParameters=%s' )
    ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
    classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )

    #Add any JDL parameters to classad obeying lists with ';' rule
    requirements = False
    for name, props in paramsDict.items():
      ptype = props['type']
      value = props['value']
      if name.lower() == 'requirements' and ptype == 'JDL':
        self.log.verbose( 'Found existing requirements: %s' % ( value ) )
        requirements = True

      if re.search( '^JDL', ptype ):
        if type( value ) == list:
          if type( value[0] ) == list:
            classadJob.insertAttributeVectorStringList( name, value )
          else:
            classadJob.insertAttributeVectorString( name, value )
        elif value == "%s":
          classadJob.insertAttributeInt( name, value )
        elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
          classadJob.insertAttributeString( name, value )
        else:
          classadJob.insertAttributeVectorString( name, value.split( ';' ) )

    if not requirements:
      reqtsDict = self.reqParams
      exprn = ''
      plus = ''
      for name, props in paramsDict.items():
        ptype = paramsDict[name]['type']
        value = paramsDict[name]['value']
        if not ptype == 'dict':
          if ptype == 'JDLReqt':
            if value and not value.lower() == 'any':
              plus = ' && '
              if re.search( ';', value ):
                for val in value.split( ';' ):
                  exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
              else:
                exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus

      if len( plus ):
        exprn = exprn[:-len( plus )]
      if not exprn:
        exprn = 'true'
      self.log.verbose( 'Requirements: %s' % ( exprn ) )
      #classadJob.set_expression('Requirements', exprn)

    self.addToInputSandbox.remove( scriptname )
    self.addToOutputSandbox.remove( self.stdout )
    self.addToOutputSandbox.remove( self.stderr )
    jdl = classadJob.asJDL()
    start = jdl.find( '[' )
    end = jdl.rfind( ']' )
    return jdl[( start + 1 ):( end - 1 )]

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import insertAttributeVectorString [as 别名]

#.........这里部分代码省略.........
                self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
            else:
                paramsDict["InputSandbox"] = {}
                paramsDict["InputSandbox"]["value"] = extraFiles
                paramsDict["InputSandbox"]["type"] = "JDL"

        if self.addToOutputSandbox:
            extraFiles = ";".join(self.addToOutputSandbox)
            if paramsDict.has_key("OutputSandbox"):
                currentFiles = paramsDict["OutputSandbox"]["value"]
                finalOutputSandbox = currentFiles + ";" + extraFiles
                uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
                paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
                self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
            else:
                paramsDict["OutputSandbox"] = {}
                paramsDict["OutputSandbox"]["value"] = extraFiles
                paramsDict["OutputSandbox"]["type"] = "JDL"

        if self.addToInputData:
            extraFiles = ";".join(self.addToInputData)
            if paramsDict.has_key("InputData"):
                currentFiles = paramsDict["InputData"]["value"]
                finalInputData = extraFiles
                if currentFiles:
                    finalInputData = currentFiles + ";" + extraFiles
                uniqueInputData = uniqueElements(finalInputData.split(";"))
                paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
                self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
            else:
                paramsDict["InputData"] = {}
                paramsDict["InputData"]["value"] = extraFiles
                paramsDict["InputData"]["type"] = "JDL"

        # Handle here the Parametric values
        if self.parametric:
            for pType in ["InputData", "InputSandbox"]:
                if self.parametric.has_key(pType):
                    if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
                        pData = self.parametric[pType]
                        # List of lists case
                        currentFiles = paramsDict[pType]["value"].split(";")
                        tmpList = []
                        if type(pData[0]) == list:
                            for pElement in pData:
                                tmpList.append(currentFiles + pElement)
                        else:
                            for pElement in pData:
                                tmpList.append(currentFiles + [pElement])
                        self.parametric[pType] = tmpList

                    paramsDict[pType] = {}
                    paramsDict[pType]["value"] = "%s"
                    paramsDict[pType]["type"] = "JDL"
                    self.parametric["files"] = self.parametric[pType]
                    arguments.append(" -p Parametric" + pType + "=%s")
                    break

            if self.parametric.has_key("files"):
                paramsDict["Parameters"] = {}
                paramsDict["Parameters"]["value"] = self.parametric["files"]
                paramsDict["Parameters"]["type"] = "JDL"
            if self.parametric.has_key("GenericParameters"):
                paramsDict["Parameters"] = {}
                paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
                paramsDict["Parameters"]["type"] = "JDL"
                arguments.append(" -p ParametricParameters=%s")
        ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
        classadJob.insertAttributeString("Arguments", " ".join(arguments))

        # Add any JDL parameters to classad obeying lists with ';' rule
        for name, props in paramsDict.items():
            ptype = props["type"]
            value = props["value"]
            if name.lower() == "requirements" and ptype == "JDL":
                self.log.verbose("Found existing requirements: %s" % (value))

            if re.search("^JDL", ptype):
                if type(value) == list:
                    if type(value[0]) == list:
                        classadJob.insertAttributeVectorStringList(name, value)
                    else:
                        classadJob.insertAttributeVectorString(name, value)
                elif value == "%s":
                    classadJob.insertAttributeInt(name, value)
                elif not re.search(";", value) or name == "GridRequirements":  # not a nice fix...
                    classadJob.insertAttributeString(name, value)
                else:
                    classadJob.insertAttributeVectorString(name, value.split(";"))

        for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
            try:
                self.addToInputSandbox.remove(fToBeRemoved)
            except ValueError:
                pass

        jdl = classadJob.asJDL()
        start = jdl.find("[")
        end = jdl.rfind("]")
        return jdl[(start + 1) : (end - 1)]