Python ClassAdLight.ClassAd类代码示例

 def getJobDefinition(self, job, jobDef=False):
   """ Retrieve JDL of the Job and return jobDef dictionary
   """
   if not jobDef:
     jobDef = {}
   # If not jdl in jobinfo load it
   if 'jdl' not in jobDef:
     if self.requiredJobInfo == 'jdlOriginal':
       result = self.jobDB.getJobJDL(job, original=True)
       if not result['OK']:
         self.log.error("No JDL for job", "%s" % job)
         return S_ERROR("No JDL for job")
       jobDef['jdl'] = result['Value']
     if self.requiredJobInfo == 'jdl':
       result = self.jobDB.getJobJDL(job)
       if not result['OK']:
         self.log.error("No JDL for job", "%s" % job)
         return S_ERROR("No JDL for job")
       jobDef['jdl'] = result['Value']
   # Load the classad if needed
   if 'jdl' in jobDef and 'classad' not in jobDef:
     try:
       classad = ClassAd(jobDef['jdl'])
     except BaseException:
       self.log.debug("Cannot load JDL")
       return S_ERROR('Illegal Job JDL')
     if not classad.isOK():
       self.log.debug("Warning: illegal JDL for job %s, will be marked problematic" % (job))
       return S_ERROR('Illegal Job JDL')
     jobDef['classad'] = classad
   return S_OK(jobDef)

  def submitJob( self, jdl ):
    """ Submit one job specified by its JDL to WMS
    """

    if os.path.exists( jdl ):
      fic = open ( jdl, "r" )
      jdlString = fic.read()
      fic.close()
    else:
      # If file JDL does not exist, assume that the JDL is passed as a string
      jdlString = jdl

    # Check the validity of the input JDL
    jdlString = jdlString.strip()
    if jdlString.find( "[" ) != 0:
      jdlString = "[%s]" % jdlString
    classAdJob = ClassAd( jdlString )
    if not classAdJob.isOK():
      return S_ERROR( 'Invalid job JDL' )

    # Check the size and the contents of the input sandbox
    result = self.__uploadInputSandbox( classAdJob )
    if not result['OK']:
      return result

    # Submit the job now and get the new job ID
    if not self.jobManager:
      self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
                                    useCertificates = self.useCertificates,
                                    timeout = self.timeout )
    result = self.jobManager.submitJob( classAdJob.asJDL() )
    if 'requireProxyUpload' in result and result['requireProxyUpload']:
      gLogger.warn( "Need to upload the proxy" )
    return result

 def __saveJobJDLRequest( self, jobID, jobJDL ):
   """Save job JDL local to JobAgent.
   """
   classAdJob = ClassAd( jobJDL )
   classAdJob.insertAttributeString( 'LocalCE', self.ceName )
   jdlFileName = jobID + '.jdl'
   jdlFile = open( jdlFileName, 'w' )
   jdl = classAdJob.asJDL()
   jdlFile.write( jdl )
   jdlFile.close()

  def __getProcessingType( self, jobID ):
    """ Get the Processing Type from the JDL, until it is promoted to a real Attribute
"""
    processingType = 'unknown'
    result = self.jobDB.getJobJDL( jobID, original = True )
    if not result['OK']:
      return processingType
    classAdJob = ClassAd( result['Value'] )
    if classAdJob.lookupAttribute( 'ProcessingType' ):
      processingType = classAdJob.getAttributeString( 'ProcessingType' )
    return processingType

    def __getProcessingType(self, jobID):
        """ Get the Processing Type from the JDL, until it is promoted to a real Attribute
"""
        processingType = "unknown"
        result = self.jobDB.getJobJDL(jobID, original=True)
        if not result["OK"]:
            return processingType
        classAdJob = ClassAd(result["Value"])
        if classAdJob.lookupAttribute("ProcessingType"):
            processingType = classAdJob.getAttributeString("ProcessingType")
        return processingType

 def __saveJobJDLRequest(self, jobID, jobJDL):
     """Save job JDL local to JobAgent.
 """
     classAdJob = ClassAd(jobJDL)
     classAdJob.insertAttributeString("LocalCE", self.ceName)
     jdlFileName = jobID + ".jdl"
     jdlFile = open(jdlFileName, "w")
     jdl = classAdJob.asJDL()
     jdlFile.write(jdl)
     jdlFile.close()
     return S_OK(jdlFileName)

  def test_SimpleProgression(self):

    clad = ClassAd( TEST_JDL_SIMPLE_PROGRESSION )
    nParam = getNumberOfParameters( clad )

    self.assertEqual( nParam, 3 )

    result = generateParametricJobs( clad )
    self.assert_( result['OK'] )

    jobDescList = result['Value']
    self.assertEqual( nParam, len( jobDescList ) )

    # Check the definition of the 2nd job
    jobClassAd = ClassAd( jobDescList[1] )
    self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '3' )
    self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )

def test_SimpleParametricJob():

  job = Job()
  job.setExecutable('myExec')
  job.setLogLevel('DEBUG')
  parList = [1, 2, 3]
  job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
  inputDataList = [
      [
          '/lhcb/data/data1',
          '/lhcb/data/data2'
      ],
      [
          '/lhcb/data/data3',
          '/lhcb/data/data4'
      ],
      [
          '/lhcb/data/data5',
          '/lhcb/data/data6'
      ]
  ]
  job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)

  jdl = job._toJDL()

  try:
    with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()

  assert jdl == expected

  clad = ClassAd('[' + jdl + ']')

  arguments = clad.getAttributeString('Arguments')
  job_id = clad.getAttributeString('JOB_ID')
  inputData = clad.getAttributeString('InputData')

  assert job_id == '%(JOB_ID)s'
  assert inputData == '%(InputData)s'
  assert 'jobDescription.xml' in arguments
  assert '-o LogLevel=DEBUG' in arguments
  assert'-p JOB_ID=%(JOB_ID)s' in arguments
  assert'-p InputData=%(InputData)s' in arguments

  def insertJobInQueue( self, job, classAdJob ):
    """ Check individual job and add to the Task Queue eventually.
    """

    jobReq = classAdJob.get_expression( "JobRequirements" )
    classAdJobReq = ClassAd( jobReq )
    jobReqDict = {}
    for name in self.taskQueueDB.getSingleValueTQDefFields():
      if classAdJobReq.lookupAttribute( name ):
        if name == 'CPUTime':
          jobReqDict[name] = classAdJobReq.getAttributeInt( name )
        else:
          jobReqDict[name] = classAdJobReq.getAttributeString( name )

    for name in self.taskQueueDB.getMultiValueTQDefFields():
      if classAdJobReq.lookupAttribute( name ):
        jobReqDict[name] = classAdJobReq.getListFromExpression( name )

    jobPriority = classAdJobReq.getAttributeInt( 'UserPriority' )

    result = self.taskQueueDB.insertJob( job, jobReqDict, jobPriority )
    if not result[ 'OK' ]:
      self.log.error( "Cannot insert job %s in task queue: %s" % ( job, result[ 'Message' ] ) )
      # Force removing the job from the TQ if it was actually inserted
      result = self.taskQueueDB.deleteJob( job )
      if result['OK']:
        if result['Value']:
          self.log.info( "Job %s removed from the TQ" % job )
      return S_ERROR( "Cannot insert in task queue" )
    return S_OK()

    def submitJob(self, jdl):
        """ Submit one job specified by its JDL to WMS
    """

        if not self.jobManagerClient:
            jobManager = RPCClient(
                "WorkloadManagement/JobManager", useCertificates=self.useCertificates, timeout=self.timeout
            )
        else:
            jobManager = self.jobManagerClient
        if os.path.exists(jdl):
            fic = open(jdl, "r")
            jdlString = fic.read()
            fic.close()
        else:
            # If file JDL does not exist, assume that the JDL is
            # passed as a string
            jdlString = jdl

        # Check the validity of the input JDL
        jdlString = jdlString.strip()
        if jdlString.find("[") != 0:
            jdlString = "[%s]" % jdlString
        classAdJob = ClassAd(jdlString)
        if not classAdJob.isOK():
            return S_ERROR("Invalid job JDL")

        # Check the size and the contents of the input sandbox
        result = self.__uploadInputSandbox(classAdJob)
        if not result["OK"]:
            return result

        # Submit the job now and get the new job ID
        result = jobManager.submitJob(classAdJob.asJDL())

        if not result["OK"]:
            return result
        jobID = result["Value"]
        if "requireProxyUpload" in result and result["requireProxyUpload"]:
            gProxyManager.uploadProxy()

        # print "Sandbox uploading"
        return S_OK(jobID)

  def test_SimpleBunch(self):

    clad = ClassAd( TEST_JDL_SIMPLE_BUNCH )
    result = getParameterVectorLength( clad )
    self.assertTrue( result['OK'] )
    nParam = result['Value']

    self.assertEqual( nParam, 3 )

    result = generateParametricJobs( clad )
    self.assertTrue(result['OK'])

    jobDescList = result['Value']
    self.assertEqual( nParam, len( jobDescList ) )

    # Check the definition of the 2nd job
    jobClassAd = ClassAd( jobDescList[1] )
    self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '5' )
    self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )

  def submitNewBigJob( self ):

    result = jobDB.getJobJDL( str( self.__jobID ) , True )
    classAdJob = ClassAd( result['Value'] )
    executableFile = ""
    if classAdJob.lookupAttribute( 'Executable' ):
      executableFile = classAdJob.getAttributeString( 'Executable' )

    tempPath = self.__tmpSandBoxDir
    dirac = Dirac()
    if not os.path.exists( tempPath ):
      os.makedirs( tempPath )

    settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
    self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
    moveData = self.__tmpSandBoxDir + "/InputSandbox" + str( self.__jobID )

    HiveV1Cli = HiveV1Client( self.__User , self.__publicIP )
    returned = HiveV1Cli.dataCopy( moveData, self.__tmpSandBoxDir )
    self.log.info( 'Copy the job contain to the Hadoop Master with HIVE: ', returned )

    jobInfo = jobDB.getJobAttributes( self.__jobID )
    if not jobInfo['OK']:
      return S_ERROR( jobInfo['Value'] )
    proxy = ""
    jobInfo = jobInfo['Value']
    if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
      proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
    else:
      proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )

    HiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile
    HiveJobOutput = str( self.__jobID ) + "_" + executableFile + "_out"

    returned = HiveV1Cli.jobSubmit( tempPath, HiveJob, proxy['chain'], HiveJobOutput )
    self.log.info( 'Launch Hadoop-Hive job to the Master: ', returned )

    if not returned['OK']:
      return S_ERROR( returned['Message'] )
    else:
      self.log.info( 'Hadoop-Hive Job ID: ', returned['Value'] )

    return S_OK( returned['Value'] )

  def submitJob( self, jdl, jobDescriptionObject = None ):
    """ Submit one job specified by its JDL to WMS
    """

    if os.path.exists( jdl ):
      fic = open ( jdl, "r" )
      jdlString = fic.read()
      fic.close()
    else:
      # If file JDL does not exist, assume that the JDL is passed as a string
      jdlString = jdl

    jdlString = jdlString.strip()

    # Strip of comments in the jdl string
    newJdlList = []
    for line in jdlString.split('\n'):
      if not line.strip().startswith( '#' ):
        newJdlList.append( line )
    jdlString = '\n'.join( newJdlList )

    # Check the validity of the input JDL
    if jdlString.find( "[" ) != 0:
      jdlString = "[%s]" % jdlString
    classAdJob = ClassAd( jdlString )
    if not classAdJob.isOK():
      return S_ERROR( 'Invalid job JDL' )

    # Check the size and the contents of the input sandbox
    result = self.__uploadInputSandbox( classAdJob, jobDescriptionObject )
    if not result['OK']:
      return result

    # Submit the job now and get the new job ID
    if not self.jobManager:
      self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
                                    useCertificates = self.useCertificates,
                                    timeout = self.timeout )
    result = self.jobManager.submitJob( classAdJob.asJDL() )
    if 'requireProxyUpload' in result and result['requireProxyUpload']:
      gLogger.warn( "Need to upload the proxy" )
    return result

  def test_SimpleParametricJob( self ):

    job = Job()
    job.setExecutable( 'myExec' )
    job.setLogLevel( 'DEBUG' )
    parList = [1,2,3]
    job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
    inputDataList = [
      [
        '/lhcb/data/data1',
        '/lhcb/data/data2'
      ],
      [
        '/lhcb/data/data3',
        '/lhcb/data/data4'
      ],
      [
        '/lhcb/data/data5',
        '/lhcb/data/data6'
      ]
    ]
    job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )

    jdl = job._toJDL()

    print jdl

    clad = ClassAd( '[' + jdl + ']' )

    arguments = clad.getAttributeString( 'Arguments' )
    job_id = clad.getAttributeString( 'JOB_ID' )
    inputData = clad.getAttributeString( 'InputData' )

    print "arguments", arguments

    self.assertEqual( job_id, '%(JOB_ID)s' )
    self.assertEqual( inputData, '%(InputData)s' )
    self.assertIn( 'jobDescription.xml', arguments )
    self.assertIn( '-o LogLevel=DEBUG', arguments )
    self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
    self.assertIn( '-p InputData=%(InputData)s', arguments )

 def matchJob(self, resourceJDL):
   """  Use Matcher service to retrieve a MPI job from Task Queue.
        Returns: JobID, NumProc required, JDL and MPI flavor
        Input: resourceJDL
        Output: result = {JobID, JobJDL, NumProc, MPIFlavor}
   """
   print "S37"
   matcher = RPCClient('WorkloadManagement/Matcher', timeout = 600)
   dictMatchMPI = {'Setup':'EELA-Production', 'CPUTime':6000, 'JobType':'MPI'}
   result = matcher.getMatchingTaskQueues(dictMatchMPI)
   if not result['OK']:
     print "S38"
     gLogger.info("-------------------------------------------------------------------")
     gLogger.error ("Here I have to call to get normal job")
     gLogger.info("-------------------------------------------------------------------")
     gLogger.error (("Match not found: %s") % (result['Message']))
     gLogger.info("-------------------------------------------------------------------")
     return S_ERROR()
   else:
     if result['Value'] == {}:
       gLogger.info("-------------------------------------------------------------------")
       gLogger.info("Value == Empty")
       gLogger.info("-------------------------------------------------------------------")
       return S_ERROR()
   mpiTaskQueue = result['Value']
   classAdAgent = ClassAd(resourceJDL)
   classAdAgent.insertAttributeString('JobType', 'MPI')
   resourceJDL = str(classAdAgent.asJDL())
   result = matcher.requestJob(resourceJDL)
   if not result['OK']:
     gLogger.error (("Request Job Error: %s") % (result['Message']))
     return S_ERROR()
   elif result['OK']==False:
     gLogger.error (("Request Job False: %s") % (result['Message']))
     return S_ERROR()
   else:
     gLogger.error (("Request Job OK"))
   jobJDL = result['Value']['JDL']
   ### Review how to optimize this part (Importante)
   jobID1 = ClassAd(jobJDL)
   jobID = jobID1.getAttributeString('JobID')
   numProc = jobID1.getAttributeString('CPUNumber')
   mpiFlavor = jobID1.getAttributeString('Flavor')
   result = {'JobID':jobID, 'JobJDL':jobJDL, 'NumProc': numProc, 
             'MPIFlavor': mpiFlavor}
   print "S39"
   print result
   return S_OK(result)