Python ClassAd.asJDL方法代码示例

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def submitJob( self, jdl ):
    """ Submit one job specified by its JDL to WMS
    """

    if os.path.exists( jdl ):
      fic = open ( jdl, "r" )
      jdlString = fic.read()
      fic.close()
    else:
      # If file JDL does not exist, assume that the JDL is passed as a string
      jdlString = jdl

    # Check the validity of the input JDL
    jdlString = jdlString.strip()
    if jdlString.find( "[" ) != 0:
      jdlString = "[%s]" % jdlString
    classAdJob = ClassAd( jdlString )
    if not classAdJob.isOK():
      return S_ERROR( 'Invalid job JDL' )

    # Check the size and the contents of the input sandbox
    result = self.__uploadInputSandbox( classAdJob )
    if not result['OK']:
      return result

    # Submit the job now and get the new job ID
    if not self.jobManager:
      self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
                                    useCertificates = self.useCertificates,
                                    timeout = self.timeout )
    result = self.jobManager.submitJob( classAdJob.asJDL() )
    if 'requireProxyUpload' in result and result['requireProxyUpload']:
      gLogger.warn( "Need to upload the proxy" )
    return result

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
 def __saveJobJDLRequest( self, jobID, jobJDL ):
   """Save job JDL local to JobAgent.
   """
   classAdJob = ClassAd( jobJDL )
   classAdJob.insertAttributeString( 'LocalCE', self.ceName )
   jdlFileName = jobID + '.jdl'
   jdlFile = open( jdlFileName, 'w' )
   jdl = classAdJob.asJDL()
   jdlFile.write( jdl )
   jdlFile.close()

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
 def __saveJobJDLRequest(self, jobID, jobJDL):
     """Save job JDL local to JobAgent.
 """
     classAdJob = ClassAd(jobJDL)
     classAdJob.insertAttributeString("LocalCE", self.ceName)
     jdlFileName = jobID + ".jdl"
     jdlFile = open(jdlFileName, "w")
     jdl = classAdJob.asJDL()
     jdlFile.write(jdl)
     jdlFile.close()
     return S_OK(jdlFileName)

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def _processResourceDescription( self, resourceDescription ):
    """ Check and form the resource description dictionary

        resourceDescription is a ceDict coming from a JobAgent, for example.
    """

    resourceDict = {}
    if isinstance( resourceDescription, basestring ):
      classAdAgent = ClassAd( resourceDescription )
      if not classAdAgent.isOK():
        raise ValueError( 'Illegal Resource JDL' )
      self.log.verbose( classAdAgent.asJDL() )

      for name in singleValueDefFields:
        if classAdAgent.lookupAttribute( name ):
          if name == 'CPUTime':
            resourceDict[name] = classAdAgent.getAttributeInt( name )
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      for name in multiValueMatchFields:
        if classAdAgent.lookupAttribute( name ):
          if name == 'SubmitPool':
            resourceDict[name] = classAdAgent.getListFromExpression( name )
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      # Check if a JobID is requested
      if classAdAgent.lookupAttribute( 'JobID' ):
        resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
        if classAdAgent.lookupAttribute( k ):
          resourceDict[ k ] = classAdAgent.getAttributeString( k )

    else:
      for name in singleValueDefFields:
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      for name in multiValueMatchFields:
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      if resourceDescription.has_key( 'JobID' ):
        resourceDict['JobID'] = resourceDescription['JobID']

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
                 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
        if k in resourceDescription:
          resourceDict[ k ] = resourceDescription[ k ]

    return resourceDict

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def __processResourceDescription( self, resourceDescription ):
    # Check and form the resource description dictionary
    resourceDict = {}
    if type( resourceDescription ) in StringTypes:
      classAdAgent = ClassAd( resourceDescription )
      if not classAdAgent.isOK():
        return S_ERROR( 'Illegal Resource JDL' )
      gLogger.verbose( classAdAgent.asJDL() )

      for name in gTaskQueueDB.getSingleValueTQDefFields():
        if classAdAgent.lookupAttribute( name ):
          if name == 'CPUTime':
            resourceDict[name] = classAdAgent.getAttributeInt( name )
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      for name in gTaskQueueDB.getMultiValueMatchFields():
        if classAdAgent.lookupAttribute( name ):
          if name == 'SubmitPool':
            resourceDict[name] = classAdAgent.getListFromExpression( name )      
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      # Check if a JobID is requested
      if classAdAgent.lookupAttribute( 'JobID' ):
        resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
        if classAdAgent.lookupAttribute( k ):
          resourceDict[ k ] = classAdAgent.getAttributeString( k )
          
    else:
      for name in gTaskQueueDB.getSingleValueTQDefFields():
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      for name in gTaskQueueDB.getMultiValueMatchFields():
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      if resourceDescription.has_key( 'JobID' ):
        resourceDict['JobID'] = resourceDescription['JobID']

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
                 'PilotReference', 'PilotInfoReportedFlag', 'PilotBenchmark' ):
        if k in resourceDescription:
          resourceDict[ k ] = resourceDescription[ k ]

    return resourceDict

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
 def matchJob(self, resourceJDL):
   """  Use Matcher service to retrieve a MPI job from Task Queue.
        Returns: JobID, NumProc required, JDL and MPI flavor
        Input: resourceJDL
        Output: result = {JobID, JobJDL, NumProc, MPIFlavor}
   """
   print "S37"
   matcher = RPCClient('WorkloadManagement/Matcher', timeout = 600)
   dictMatchMPI = {'Setup':'EELA-Production', 'CPUTime':6000, 'JobType':'MPI'}
   result = matcher.getMatchingTaskQueues(dictMatchMPI)
   if not result['OK']:
     print "S38"
     gLogger.info("-------------------------------------------------------------------")
     gLogger.error ("Here I have to call to get normal job")
     gLogger.info("-------------------------------------------------------------------")
     gLogger.error (("Match not found: %s") % (result['Message']))
     gLogger.info("-------------------------------------------------------------------")
     return S_ERROR()
   else:
     if result['Value'] == {}:
       gLogger.info("-------------------------------------------------------------------")
       gLogger.info("Value == Empty")
       gLogger.info("-------------------------------------------------------------------")
       return S_ERROR()
   mpiTaskQueue = result['Value']
   classAdAgent = ClassAd(resourceJDL)
   classAdAgent.insertAttributeString('JobType', 'MPI')
   resourceJDL = str(classAdAgent.asJDL())
   result = matcher.requestJob(resourceJDL)
   if not result['OK']:
     gLogger.error (("Request Job Error: %s") % (result['Message']))
     return S_ERROR()
   elif result['OK']==False:
     gLogger.error (("Request Job False: %s") % (result['Message']))
     return S_ERROR()
   else:
     gLogger.error (("Request Job OK"))
   jobJDL = result['Value']['JDL']
   ### Review how to optimize this part (Importante)
   jobID1 = ClassAd(jobJDL)
   jobID = jobID1.getAttributeString('JobID')
   numProc = jobID1.getAttributeString('CPUNumber')
   mpiFlavor = jobID1.getAttributeString('Flavor')
   result = {'JobID':jobID, 'JobJDL':jobJDL, 'NumProc': numProc, 
             'MPIFlavor': mpiFlavor}
   print "S39"
   print result
   return S_OK(result)

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
    def submitJob(self, jdl):
        """ Submit one job specified by its JDL to WMS
    """

        if not self.jobManagerClient:
            jobManager = RPCClient(
                "WorkloadManagement/JobManager", useCertificates=self.useCertificates, timeout=self.timeout
            )
        else:
            jobManager = self.jobManagerClient
        if os.path.exists(jdl):
            fic = open(jdl, "r")
            jdlString = fic.read()
            fic.close()
        else:
            # If file JDL does not exist, assume that the JDL is
            # passed as a string
            jdlString = jdl

        # Check the validity of the input JDL
        jdlString = jdlString.strip()
        if jdlString.find("[") != 0:
            jdlString = "[%s]" % jdlString
        classAdJob = ClassAd(jdlString)
        if not classAdJob.isOK():
            return S_ERROR("Invalid job JDL")

        # Check the size and the contents of the input sandbox
        result = self.__uploadInputSandbox(classAdJob)
        if not result["OK"]:
            return result

        # Submit the job now and get the new job ID
        result = jobManager.submitJob(classAdJob.asJDL())

        if not result["OK"]:
            return result
        jobID = result["Value"]
        if "requireProxyUpload" in result and result["requireProxyUpload"]:
            gProxyManager.uploadProxy()

        # print "Sandbox uploading"
        return S_OK(jobID)

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def submitJob( self, jdl, jobDescriptionObject = None ):
    """ Submit one job specified by its JDL to WMS
    """

    if os.path.exists( jdl ):
      fic = open ( jdl, "r" )
      jdlString = fic.read()
      fic.close()
    else:
      # If file JDL does not exist, assume that the JDL is passed as a string
      jdlString = jdl

    jdlString = jdlString.strip()

    # Strip of comments in the jdl string
    newJdlList = []
    for line in jdlString.split('\n'):
      if not line.strip().startswith( '#' ):
        newJdlList.append( line )
    jdlString = '\n'.join( newJdlList )

    # Check the validity of the input JDL
    if jdlString.find( "[" ) != 0:
      jdlString = "[%s]" % jdlString
    classAdJob = ClassAd( jdlString )
    if not classAdJob.isOK():
      return S_ERROR( 'Invalid job JDL' )

    # Check the size and the contents of the input sandbox
    result = self.__uploadInputSandbox( classAdJob, jobDescriptionObject )
    if not result['OK']:
      return result

    # Submit the job now and get the new job ID
    if not self.jobManager:
      self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
                                    useCertificates = self.useCertificates,
                                    timeout = self.timeout )
    result = self.jobManager.submitJob( classAdJob.asJDL() )
    if 'requireProxyUpload' in result and result['requireProxyUpload']:
      gLogger.warn( "Need to upload the proxy" )
    return result

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]

#.........这里部分代码省略.........
        finalInputData = extraFiles
        if currentFiles:
          finalInputData = currentFiles + ';' + extraFiles
        uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
        paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
        self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
      else:
        paramsDict['InputData'] = {}
        paramsDict['InputData']['value'] = extraFiles
        paramsDict['InputData']['type'] = 'JDL'

    # Handle here the Parametric values
    if self.parametric:
      for pType in ['InputData', 'InputSandbox']:
        if self.parametric.has_key( pType ):
          if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
            pData = self.parametric[pType]
            # List of lists case
            currentFiles = paramsDict[pType]['value'].split( ';' )
            tmpList = []
            if type( pData[0] ) == list:
              for pElement in pData:
                tmpList.append( currentFiles + pElement )
            else:
              for pElement in pData:
                tmpList.append( currentFiles + [pElement] )
            self.parametric[pType] = tmpList

          paramsDict[pType] = {}
          paramsDict[pType]['value'] = "%s"
          paramsDict[pType]['type'] = 'JDL'
          self.parametric['files'] = self.parametric[pType]
          arguments.append( ' -p Parametric' + pType + '=%s' )
          break

      if self.parametric.has_key( 'files' ):
        paramsDict['Parameters'] = {}
        paramsDict['Parameters']['value'] = self.parametric['files']
        paramsDict['Parameters']['type'] = 'JDL'
      if self.parametric.has_key( 'GenericParameters' ):
        paramsDict['Parameters'] = {}
        paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
        paramsDict['Parameters']['type'] = 'JDL'
        arguments.append( ' -p ParametricParameters=%s' )
    ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
    classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )

    #Add any JDL parameters to classad obeying lists with ';' rule
    requirements = False
    for name, props in paramsDict.items():
      ptype = props['type']
      value = props['value']
      if name.lower() == 'requirements' and ptype == 'JDL':
        self.log.verbose( 'Found existing requirements: %s' % ( value ) )
        requirements = True

      if re.search( '^JDL', ptype ):
        if type( value ) == list:
          if type( value[0] ) == list:
            classadJob.insertAttributeVectorStringList( name, value )
          else:
            classadJob.insertAttributeVectorString( name, value )
        elif value == "%s":
          classadJob.insertAttributeInt( name, value )
        elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
          classadJob.insertAttributeString( name, value )
        else:
          classadJob.insertAttributeVectorString( name, value.split( ';' ) )

    if not requirements:
      reqtsDict = self.reqParams
      exprn = ''
      plus = ''
      for name, props in paramsDict.items():
        ptype = paramsDict[name]['type']
        value = paramsDict[name]['value']
        if not ptype == 'dict':
          if ptype == 'JDLReqt':
            if value and not value.lower() == 'any':
              plus = ' && '
              if re.search( ';', value ):
                for val in value.split( ';' ):
                  exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
              else:
                exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus

      if len( plus ):
        exprn = exprn[:-len( plus )]
      if not exprn:
        exprn = 'true'
      self.log.verbose( 'Requirements: %s' % ( exprn ) )
      #classadJob.set_expression('Requirements', exprn)

    self.addToInputSandbox.remove( scriptname )
    self.addToOutputSandbox.remove( self.stdout )
    self.addToOutputSandbox.remove( self.stderr )
    jdl = classadJob.asJDL()
    start = jdl.find( '[' )
    end = jdl.rfind( ']' )
    return jdl[( start + 1 ):( end - 1 )]

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def export_submitJob( self, jobDesc ):
    """ Submit a single job to DIRAC WMS
    """

    if self.peerUsesLimitedProxy:
      return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" )

    # Check job submission permission
    result = self.jobPolicy.getJobPolicy()
    if not result['OK']:
      return S_ERROR( 'Failed to get job policies' )
    policyDict = result['Value']
    if not policyDict[ RIGHT_SUBMIT ]:
      return S_ERROR( 'Job submission not authorized' )

    #jobDesc is JDL for now
    jobDesc = jobDesc.strip()
    if jobDesc[0] != "[":
      jobDesc = "[%s" % jobDesc
    if jobDesc[-1] != "]":
      jobDesc = "%s]" % jobDesc

    # Check if the job is a parameteric one
    jobClassAd = ClassAd( jobDesc )
    parametricJob = False
    if jobClassAd.lookupAttribute( 'Parameters' ):
      parametricJob = True
      if jobClassAd.isAttributeList( 'Parameters' ):
        parameterList = jobClassAd.getListFromExpression( 'Parameters' )
      else:
        pStep = 0
        pFactor = 1
        pStart = 1
        nParameters = jobClassAd.getAttributeInt( 'Parameters' )
        if not nParameters:
          value = jobClassAd.get_expression( 'Parameters' )
          return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value )

        if jobClassAd.lookupAttribute( 'ParameterStart' ):
          value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' )
          try:
            pStart = int( value )
          except:
            try:
              pStart = float( value )
            except:
              return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value )

        if jobClassAd.lookupAttribute( 'ParameterStep' ):
          pStep = jobClassAd.getAttributeInt( 'ParameterStep' )
          if not pStep:
            pStep = jobClassAd.getAttributeFloat( 'ParameterStep' )
            if not pStep:
              value = jobClassAd.get_expression( 'ParameterStep' )
              return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value )
        if jobClassAd.lookupAttribute( 'ParameterFactor' ):
          pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' )
          if not pFactor:
            pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' )
            if not pFactor:
              value = jobClassAd.get_expression( 'ParameterFactor' )
              return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value )

        parameterList = list()
        parameterList.append( pStart )
        for i in range( nParameters - 1 ):
          parameterList.append( parameterList[i] * pFactor + pStep )


      if len( parameterList ) > self.maxParametricJobs:
        return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs )

      jobDescList = []
      nParam = len(parameterList) - 1
      for n,p in enumerate(parameterList):
        newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam))))
        newClassAd = ClassAd(newJobDesc)
        for attr in ['Parameters','ParameterStep','ParameterFactor']:
          newClassAd.deleteAttribute(attr)
        if type( p ) == type ( ' ' ) and p.startswith('{'):
          newClassAd.insertAttributeInt( 'Parameter',str(p) )
        else:
          newClassAd.insertAttributeString( 'Parameter', str( p ) )
        newClassAd.insertAttributeInt( 'ParameterNumber', n )
        newJDL = newClassAd.asJDL()
        jobDescList.append( newJDL )
    else:
      jobDescList = [ jobDesc ]

    jobIDList = []
    for jobDescription in jobDescList:
      result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup )
      if not result['OK']:
        return result

      jobID = result['JobID']
      gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) )

      gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' )

#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def __sendJobToTaskQueue( self, job, classAdJob, siteCandidates, bannedSites ):
    """This method sends jobs to the task queue agent and if candidate sites
       are defined, updates job JDL accordingly.
    """

    reqJDL = classAdJob.get_expression( 'JobRequirements' )
    classAddReq = ClassAd( reqJDL )

    if siteCandidates:
      classAddReq.insertAttributeVectorString( 'Sites', siteCandidates )
    if bannedSites:
      classAddReq.insertAttributeVectorString( 'BannedSites', bannedSites )

    if classAdJob.lookupAttribute( "SubmitPools" ):
      classAddReq.set_expression( 'SubmitPools', classAdJob.get_expression( 'SubmitPools' ) )

    if classAdJob.lookupAttribute( "GridMiddleware" ):
      classAddReq.set_expression( 'GridMiddleware', classAdJob.get_expression( 'GridMiddleware' ) )

    if classAdJob.lookupAttribute( "PilotTypes" ):
      classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotTypes' ) )
    #HAck to migrate old jobs to new ones.
    #DELETE ON 08/09
    else:
      if classAdJob.lookupAttribute( "PilotType" ):
        classAddReq.set_expression( 'PilotTypes', classAdJob.get_expression( 'PilotType' ) )

    if classAdJob.lookupAttribute( "JobType" ):
      jobTypes = [ jt for jt in classAdJob.getListFromExpression( 'JobType' ) if jt ]
      classAddReq.insertAttributeVectorString( 'JobTypes', jobTypes )

    #Required CE's requirements
    gridCEs = [ ce for ce in classAdJob.getListFromExpression( 'GridRequiredCEs' ) if ce ]
    if gridCEs:
      classAddReq.insertAttributeVectorString( 'GridCEs', gridCEs )

    if siteCandidates:
      sites = ','.join( siteCandidates )
      classAdJob.insertAttributeString( "Site", sites )

    reqJDL = classAddReq.asJDL()
    classAdJob.insertAttributeInt( 'JobRequirements', reqJDL )

    jdl = classAdJob.asJDL()
    result = self.jobDB.setJobJDL( job, jdl )
    if not result['OK']:
      return result

    if siteCandidates:
      if len( siteCandidates ) == 1:
        self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, siteCandidates[0] ) )
        self.jobDB.setJobAttribute( job, 'Site', siteCandidates[0] )
      elif bannedSites:
        remainingSites = []
        for site in siteCandidates:
          if not site in bannedSites:
            remainingSites.append( site )
        if remainingSites:
          if len( remainingSites ) == 1:
            self.log.verbose( 'Individual site candidate for job %s is %s' % ( job, remainingSites[0] ) )
            self.jobDB.setJobAttribute( job, 'Site', remainingSites[0] )
          else:
            self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( remainingSites ) ) )
            result = self.jobDB.getJobAttribute(job,'Site')
            siteGroup = "Multiple"  
            if result['OK']:
              if result['Value'].startswith('Group'):
                siteGroup = result['Value']              
            self.jobDB.setJobAttribute( job, 'Site', siteGroup )
      else:
        self.log.verbose( 'Site candidates for job %s are %s' % ( job, str( siteCandidates ) ) )
        result = self.jobDB.getJobAttribute(job,'Site')
        siteGroup = "Multiple"  
        if result['OK']:
          if result['Value'].startswith('Group'):
            siteGroup = result['Value']              
        self.jobDB.setJobAttribute( job, 'Site', siteGroup )
    else:
      self.log.verbose( 'All sites are eligible for job %s' % job )
      self.jobDB.setJobAttribute( job, 'Site', 'ANY' )

    return self.setNextOptimizer( job )

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]

#.........这里部分代码省略.........
                self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
            else:
                paramsDict["InputSandbox"] = {}
                paramsDict["InputSandbox"]["value"] = extraFiles
                paramsDict["InputSandbox"]["type"] = "JDL"

        if self.addToOutputSandbox:
            extraFiles = ";".join(self.addToOutputSandbox)
            if paramsDict.has_key("OutputSandbox"):
                currentFiles = paramsDict["OutputSandbox"]["value"]
                finalOutputSandbox = currentFiles + ";" + extraFiles
                uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
                paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
                self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
            else:
                paramsDict["OutputSandbox"] = {}
                paramsDict["OutputSandbox"]["value"] = extraFiles
                paramsDict["OutputSandbox"]["type"] = "JDL"

        if self.addToInputData:
            extraFiles = ";".join(self.addToInputData)
            if paramsDict.has_key("InputData"):
                currentFiles = paramsDict["InputData"]["value"]
                finalInputData = extraFiles
                if currentFiles:
                    finalInputData = currentFiles + ";" + extraFiles
                uniqueInputData = uniqueElements(finalInputData.split(";"))
                paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
                self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
            else:
                paramsDict["InputData"] = {}
                paramsDict["InputData"]["value"] = extraFiles
                paramsDict["InputData"]["type"] = "JDL"

        # Handle here the Parametric values
        if self.parametric:
            for pType in ["InputData", "InputSandbox"]:
                if self.parametric.has_key(pType):
                    if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
                        pData = self.parametric[pType]
                        # List of lists case
                        currentFiles = paramsDict[pType]["value"].split(";")
                        tmpList = []
                        if type(pData[0]) == list:
                            for pElement in pData:
                                tmpList.append(currentFiles + pElement)
                        else:
                            for pElement in pData:
                                tmpList.append(currentFiles + [pElement])
                        self.parametric[pType] = tmpList

                    paramsDict[pType] = {}
                    paramsDict[pType]["value"] = "%s"
                    paramsDict[pType]["type"] = "JDL"
                    self.parametric["files"] = self.parametric[pType]
                    arguments.append(" -p Parametric" + pType + "=%s")
                    break

            if self.parametric.has_key("files"):
                paramsDict["Parameters"] = {}
                paramsDict["Parameters"]["value"] = self.parametric["files"]
                paramsDict["Parameters"]["type"] = "JDL"
            if self.parametric.has_key("GenericParameters"):
                paramsDict["Parameters"] = {}
                paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
                paramsDict["Parameters"]["type"] = "JDL"
                arguments.append(" -p ParametricParameters=%s")
        ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
        classadJob.insertAttributeString("Arguments", " ".join(arguments))

        # Add any JDL parameters to classad obeying lists with ';' rule
        for name, props in paramsDict.items():
            ptype = props["type"]
            value = props["value"]
            if name.lower() == "requirements" and ptype == "JDL":
                self.log.verbose("Found existing requirements: %s" % (value))

            if re.search("^JDL", ptype):
                if type(value) == list:
                    if type(value[0]) == list:
                        classadJob.insertAttributeVectorStringList(name, value)
                    else:
                        classadJob.insertAttributeVectorString(name, value)
                elif value == "%s":
                    classadJob.insertAttributeInt(name, value)
                elif not re.search(";", value) or name == "GridRequirements":  # not a nice fix...
                    classadJob.insertAttributeString(name, value)
                else:
                    classadJob.insertAttributeVectorString(name, value.split(";"))

        for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
            try:
                self.addToInputSandbox.remove(fToBeRemoved)
            except ValueError:
                pass

        jdl = classadJob.asJDL()
        start = jdl.find("[")
        end = jdl.rfind("]")
        return jdl[(start + 1) : (end - 1)]

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]

#.........这里部分代码省略.........
        arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
      else:
        self.log.warn( 'Job Mode defined with null value' )
    if paramsDict.has_key( 'JobConfigArgs' ):
      if paramsDict['JobConfigArgs']['value']:
        arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
      else:
        self.log.warn( 'JobConfigArgs defined with null value' )
    if self.parametricWFArguments:
      for name, value in self.parametricWFArguments.items():
        arguments.append( "-p %s='%s'" % ( name, value ) )

    classadJob.insertAttributeString( 'Executable', self.executable )
    self.addToOutputSandbox.append( self.stderr )
    self.addToOutputSandbox.append( self.stdout )

    #Extract i/o sandbox parameters from steps and any input data parameters
    #to do when introducing step-level api...

    #To add any additional files to input and output sandboxes
    if self.addToInputSandbox:
      extraFiles = ';'.join( self.addToInputSandbox )
      if paramsDict.has_key( 'InputSandbox' ):
        currentFiles = paramsDict['InputSandbox']['value']
        finalInputSandbox = currentFiles + ';' + extraFiles
        uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
        paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
        self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
      else:
        paramsDict['InputSandbox'] = {}
        paramsDict['InputSandbox']['value'] = extraFiles
        paramsDict['InputSandbox']['type'] = 'JDL'

    if self.addToOutputSandbox:
      extraFiles = ';'.join( self.addToOutputSandbox )
      if paramsDict.has_key( 'OutputSandbox' ):
        currentFiles = paramsDict['OutputSandbox']['value']
        finalOutputSandbox = currentFiles + ';' + extraFiles
        uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
        paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
        self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
      else:
        paramsDict['OutputSandbox'] = {}
        paramsDict['OutputSandbox']['value'] = extraFiles
        paramsDict['OutputSandbox']['type'] = 'JDL'

    if self.addToInputData:
      extraFiles = ';'.join( self.addToInputData )
      if paramsDict.has_key( 'InputData' ):
        currentFiles = paramsDict['InputData']['value']
        finalInputData = extraFiles
        if currentFiles:
          finalInputData = currentFiles + ';' + extraFiles
        uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
        paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
        self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
      else:
        paramsDict['InputData'] = {}
        paramsDict['InputData']['value'] = extraFiles
        paramsDict['InputData']['type'] = 'JDL'

    # Handle parameter sequences
    if self.numberOfParameters > 0:
      paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments )

    classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )

    #Add any JDL parameters to classad obeying lists with ';' rule
    for name, props in paramsDict.iteritems():
      ptype = props['type']
      value = props['value']
      if isinstance( value, basestring) and re.search( ';', value ):
        value = value.split( ';' )
      if name.lower() == 'requirements' and ptype == 'JDL':
        self.log.verbose( 'Found existing requirements: %s' % ( value ) )

      if re.search( '^JDL', ptype ):
        if isinstance( value, list ):
          if isinstance( value[0], list ):
            classadJob.insertAttributeVectorStringList( name, value )
          else:
            classadJob.insertAttributeVectorInt( name, value )
        elif isinstance( value, basestring ) and value:
          classadJob.insertAttributeInt( name, value )
        elif isinstance( value, ( int, long, float ) ):
          classadJob.insertAttributeInt( name, value )

    if self.numberOfParameters > 0:
      classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters )

    for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
      try:
        self.addToInputSandbox.remove( fToBeRemoved )
      except ValueError:
        pass

    jdl = classadJob.asJDL()
    start = jdl.find( '[' )
    end = jdl.rfind( ']' )
    return jdl[( start + 1 ):( end - 1 )]

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
def generateParametricJobs( jobClassAd ):
  """ Generate a series of ClassAd job descriptions expanding
      job parameters

  :param jobClassAd: ClassAd job description object
  :return: list of ClassAd job description objects
  """
  if not jobClassAd.lookupAttribute( 'Parameters' ):
    return S_OK( [ jobClassAd.asJDL() ] )

  result = getParameterVectorLength( jobClassAd )
  if not result['OK']:
    return result
  nParValues = result['Value']
  if nParValues is None:
    return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters')

  parameterDict = {}
  attributes = jobClassAd.getAttributes()
  for attribute in attributes:
    for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]:
      if attribute.startswith( key ):
        seqID = '0' if not '.' in attribute else attribute.split( '.' )[1]
        parameterDict.setdefault( seqID, {} )
        if key == 'Parameters':
          if jobClassAd.isAttributeList( attribute ):
            parList = jobClassAd.getListFromExpression( attribute )
            if len( parList ) != nParValues:
              return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
            parameterDict[seqID]['ParameterList'] = parList
          else:
            if attribute != "Parameters":
              return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
            nPar = jobClassAd.getAttributeInt( attribute )
            if nPar is None:
              value = jobClassAd.get_expression( attribute )
              return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % ( attribute, value ) )
            parameterDict[seqID]['Parameters'] = nPar
        else:
          value = jobClassAd.getAttributeInt( attribute )
          if value is None:
            value = jobClassAd.getAttributeFloat( attribute )
            if value is None:
              value = jobClassAd.get_expression( attribute )
              return S_ERROR( 'Illegal value for %s JDL field: %s' % ( attribute, value ) )
          parameterDict[seqID][key] = value

  if '0' in parameterDict and not parameterDict.get( '0' ):
    parameterDict.pop( '0' )

  parameterLists = {}
  for seqID in parameterDict:
    parList = __getParameterSequence( nParValues,
                                      parList = parameterDict[seqID].get( 'ParameterList', [] ),
                                      parStart = parameterDict[seqID].get( 'ParameterStart', 1 ),
                                      parStep = parameterDict[seqID].get( 'ParameterStep', 0 ),
                                      parFactor = parameterDict[seqID].get( 'ParameterFactor', 1 )
                                    )
    if not parList:
      return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )

    parameterLists[seqID] = parList

  jobDescList = []
  jobDesc = jobClassAd.asJDL()
  # Width of the sequential parameter number
  zLength = len( str( nParValues - 1 ) )
  for n in range( nParValues ):
    newJobDesc = jobDesc
    newJobDesc = newJobDesc.replace( '%n', str( n ).zfill( zLength ) )
    newClassAd = ClassAd( newJobDesc )
    for seqID in parameterLists:
      parameter = parameterLists[seqID][n]
      for attribute in newClassAd.getAttributes():
        __updateAttribute( newClassAd, attribute, seqID, str( parameter ) )

    for seqID in parameterLists:
      for attribute in ['Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor']:
        if seqID == '0':
          newClassAd.deleteAttribute( attribute )
        else:
          newClassAd.deleteAttribute( '%s.%s' % ( attribute, seqID ) )

      parameter = parameterLists[seqID][n]
      if seqID == '0':
        attribute = 'Parameter'
      else:
        attribute = 'Parameter.%s' % seqID
      if isinstance( parameter, basestring) and parameter.startswith( '{' ):
        newClassAd.insertAttributeInt( attribute, str( parameter ) )
      else:
        newClassAd.insertAttributeString( attribute, str( parameter ) )

    newClassAd.insertAttributeInt( 'ParameterNumber', n )
    newJDL = newClassAd.asJDL()
    jobDescList.append( newJDL )

  return S_OK( jobDescList )

# 需要导入模块: from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd [as 别名]
# 或者: from DIRAC.Core.Utilities.ClassAd.ClassAdLight.ClassAd import asJDL [as 别名]
  def _processResourceDescription( self, resourceDescription ):
    """ Check and form the resource description dictionary

        resourceDescription is a ceDict coming from a JobAgent, for example.
    """

    resourceDict = {}
    if isinstance( resourceDescription, basestring ):
      classAdAgent = ClassAd( resourceDescription )
      if not classAdAgent.isOK():
        raise ValueError( 'Illegal Resource JDL' )
      self.log.verbose( classAdAgent.asJDL() )

      for name in singleValueDefFields:
        if classAdAgent.lookupAttribute( name ):
          if name == 'CPUTime':
            resourceDict[name] = classAdAgent.getAttributeInt( name )
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      for name in multiValueMatchFields:
        if classAdAgent.lookupAttribute( name ):
          if name == 'SubmitPool':
            resourceDict[name] = classAdAgent.getListFromExpression( name )
          else:
            resourceDict[name] = classAdAgent.getAttributeString( name )

      # Check if a JobID is requested
      if classAdAgent.lookupAttribute( 'JobID' ):
        resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
        if classAdAgent.lookupAttribute( k ):
          resourceDict[ k ] = classAdAgent.getAttributeString( k )

    else:
      for name in singleValueDefFields:
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      for name in multiValueMatchFields:
        if resourceDescription.has_key( name ):
          resourceDict[name] = resourceDescription[name]

      if 'JobID' in resourceDescription:
        resourceDict['JobID'] = resourceDescription['JobID']

      # Convert MaxRAM and NumberOfCores parameters into a list of tags
      maxRAM = resourceDescription.get( 'MaxRAM' )
      nCores = resourceDescription.get( 'NumberOfProcessors' )
      for param, key in [ ( maxRAM, 'GB' ), ( nCores, 'Cores' ) ]:
        if param:
          try:
            intValue = int( param )/1000
            if intValue <= 128:
              paramList = range( 1, intValue + 1 )
              paramTags = [ '%d%s' % ( par, key ) for par in paramList ]
              resourceDict.setdefault( "Tag", [] ).extend( paramTags )
          except ValueError:
            pass
      if 'Tag' in resourceDict:
        resourceDict['Tag'] = list( set( resourceDict['Tag'] ) )

      for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
                 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
        if k in resourceDescription:
          resourceDict[ k ] = resourceDescription[ k ]

    return resourceDict