Python Job.Job类代码示例

  def execute(self):
    """ Execute it.
    """
    jobDescription = self.paramDict['Job']
    prodID = self.paramDict['TransformationID']
    jobID = self.paramDict['TaskID']
    inputData = self.paramDict['InputData']
    
    job = Job(jobDescription)
    commons = job._getParameters() #pylint: disable=protected-access
    code = job.workflow.createCode()
    outputList = []
    for line in code.split("\n"):
      if line.count("listoutput"):
        outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used

    commons['outputList'] = outputList
    commons['PRODUCTION_ID'] = prodID
    commons['JOB_ID'] = jobID
    if inputData:
      commons['InputData'] = inputData

    result = constructProductionLFNs(commons)
    if not result['OK']:
      LOG.error(result['Message'])
      return result

    if commons['JobType'] in ILDJOBTYPES and commons['InputData']:
      for index, outputFile in enumerate( result['Value']['ProductionOutputData'] ):
        outputFileILD = getProdFilenameFromInput( commons['InputData'], outputFile, prodID, jobID )
        result['Value']['ProductionOutputData'][index] = outputFileILD
        LOG.debug("Changed output file name from '%s' to '%s' " % (outputFile, outputFileILD))


    return result

  def execute(self):
    """ Execute it.
    """
    jobDescription = self.paramDict['Job']
    prodID = self.paramDict['TransformationID']
    jobID = self.paramDict['TaskID']
    inputData = self.paramDict['InputData']
    
    job = Job(jobDescription)
    commons = job._getParameters()
    code = job.workflow.createCode()
    outputList = []
    for line in code.split("\n"):
      if line.count("listoutput"):
        outputList += eval(line.split("#")[0].split("=")[-1])

    commons['outputList'] = outputList
    commons['PRODUCTION_ID'] = prodID
    commons['JOB_ID'] = jobID
    if inputData:
      commons['InputData'] = inputData

    gLogger.debug(commons)
    result = constructProductionLFNs(commons)
    if not result['OK']:
      gLogger.error(result['Message'])
    return result

  def test_execute( self ):
    """ this one tests that I can execute a job that requires multi-processing
    """

    j = Job()

    j.setName( "MP-test" )
    j.setExecutable( self.mpExe )
    j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
    j.setTag( 'MultiProcessor' )
    res = j.runLocal( self.d )
    self.assertTrue( res['OK'] )

 def __init__(self, script = None):
   DiracJob.__init__(self, script)
   self.log = gLogger.getSubLogger("ILCJob")
   self.applicationlist = []
   self.inputsandbox = []
   self.outputsandbox = []
   self.check = True
   self.systemConfig = ''
   self.stepnumber = 0
   self.steps = []
   self.nbevts = 0
   self.energy = 0
   self.setSystemConfig('x86_64-slc5-gcc43-opt')

  def test_execute(self):

    j = Job()

    j.setName("helloWorld-test")
    j.setExecutable(self.exeScriptLocation)
    j.setLogLevel('DEBUG')
    res = j.runLocal(self.d)
    self.assertTrue(res['OK'])

def test_SimpleParametricJob():

  job = Job()
  job.setExecutable('myExec')
  job.setLogLevel('DEBUG')
  parList = [1, 2, 3]
  job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
  inputDataList = [
      [
          '/lhcb/data/data1',
          '/lhcb/data/data2'
      ],
      [
          '/lhcb/data/data3',
          '/lhcb/data/data4'
      ],
      [
          '/lhcb/data/data5',
          '/lhcb/data/data6'
      ]
  ]
  job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)

  jdl = job._toJDL()

  try:
    with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWF.jdl') as fd:
      expected = fd.read()

  assert jdl == expected

  clad = ClassAd('[' + jdl + ']')

  arguments = clad.getAttributeString('Arguments')
  job_id = clad.getAttributeString('JOB_ID')
  inputData = clad.getAttributeString('InputData')

  assert job_id == '%(JOB_ID)s'
  assert inputData == '%(InputData)s'
  assert 'jobDescription.xml' in arguments
  assert '-o LogLevel=DEBUG' in arguments
  assert'-p JOB_ID=%(JOB_ID)s' in arguments
  assert'-p InputData=%(InputData)s' in arguments

def helloWorldJob():
  job = Job()
  job.setName("helloWorld")
  exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
  job.setInputSandbox(exeScriptLocation)
  job.setExecutable(exeScriptLocation, "", "helloWorld.log")
  return job

def helloWorldJob():
  job = Job()
  job.setName( "helloWorld" )
  exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0]
  job.setInputSandbox( exeScriptLocation )
  job.setExecutable( exeScriptLocation, "", "helloWorld.log" )
  return job

  def test_SimpleParametricJob( self ):

    job = Job()
    job.setExecutable( 'myExec' )
    job.setLogLevel( 'DEBUG' )
    parList = [1,2,3]
    job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
    inputDataList = [
      [
        '/lhcb/data/data1',
        '/lhcb/data/data2'
      ],
      [
        '/lhcb/data/data3',
        '/lhcb/data/data4'
      ],
      [
        '/lhcb/data/data5',
        '/lhcb/data/data6'
      ]
    ]
    job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )

    jdl = job._toJDL()

    print jdl

    clad = ClassAd( '[' + jdl + ']' )

    arguments = clad.getAttributeString( 'Arguments' )
    job_id = clad.getAttributeString( 'JOB_ID' )
    inputData = clad.getAttributeString( 'InputData' )

    print "arguments", arguments

    self.assertEqual( job_id, '%(JOB_ID)s' )
    self.assertEqual( inputData, '%(InputData)s' )
    self.assertIn( 'jobDescription.xml', arguments )
    self.assertIn( '-o LogLevel=DEBUG', arguments )
    self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
    self.assertIn( '-p InputData=%(InputData)s', arguments )

""" simple hello world job
"""

from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers

j = Job()

j.setName( "helloWorld-test" )

j.setExecutable( "exe-script.py", "", "Executable.log" )

# <-- user settings
j.setCPUTime( 172800 )
tier1s = DMSHelpers().getTiers( tier = ( 0, 1 ) )
j.setBannedSites( tier1s )
# user settings -->


# print j.workflow

# submit the job to dirac
result = Dirac().submitJob(j)
print result

            "n_non_gamma" : f.getNumberOfNonGammas(),
            #
            "ismc"        : int(f.isMC())
            }

        file_dict["%s.txt" % (fn)] = metadata

    # Update the user.
    print("*")
    print("* Uploading the following files:")
    for fn in file_dict.keys():
        print("*-> '%s'" % (fn))
    print("*")

    ## The DIRAC job to submit.
    j = Job(stdout='StdOut', stderr='StdErr')

    # Set the name of the job (viewable in the web portal).
    j.setName(jobname)

    # As we're just copying the input sandbox to the storage element
    # via OutputData, we'll just list the files as a check for the
    # output written to StdOut.
    j.setExecutable('/bin/ls -l')

    # Here we add the names of the temporary copies of the frame data
    # files in the dataset to the input sandbox. These will be uploaded
    # to the grid with the job...
    j.setInputSandbox(file_dict.keys())

    #...and added to the desired storage element with the corresponding

        # if file already in GRID storage, skip
        # (you cannot overwrite it there, delete it and resubmit)
        # (assumes tail and wave will always be written out together)
        if '/'.join([output_path.strip('/'), output_filename_wave]) in GRID_filelist:
            print("\n{} already on GRID SE\n".format(job_name))
            continue

        if NJobs == 0:
            print("maximum number of jobs to submit reached")
            print("breaking loop now")
            break
        else:
            NJobs -= 1

        j = Job()
        # runtime in seconds times 8 (CPU normalisation factor)
        j.setCPUTime(6 * 3600 * 8)
        j.setName(job_name)
        j.setInputSandbox(input_sandbox)

        if banned_sites:
            j.setBannedSites(banned_sites)
        # j.setDestination( 'LCG.IN2P3-CC.fr' )

        # mr_filter loses its executable property by uploading it to the GRID SE; reset
        j.setExecutable('chmod', '+x mr_filter')

        j.setExecutable('ls -lah')

        for run_file in run_filelist:

  def test_execute( self ):

    job = Job()

    job.setName( "helloWorld-test" )
    job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
                       arguments = "This is an argument",
                       logFile = "aLogFileForTest.txt" ,
                       parameters=[('executable', 'string', '', "Executable Script"),
                                   ('arguments', 'string', '', 'Arguments for executable Script'),
                                   ( 'applicationLog', 'string', '', "Log file name" ),
                                   ( 'someCustomOne', 'string', '', "boh" )],
                       paramValues = [( 'someCustomOne', 'aCustomValue' )] )
    job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
    job.setOwner( 'ownerName' )
    job.setOwnerGroup( 'ownerGroup' )
    job.setName( 'jobName' )
    job.setJobGroup( 'jobGroup' )
    job.setType( 'jobType' )
    job.setDestination( 'DIRAC.someSite.ch' )
    job.setCPUTime( 12345 )
    job.setLogLevel( 'DEBUG' )

    res = job.runLocal( self.d )
    self.assertTrue( res['OK'] )

def parametricJob():
  job = Job()
  job.setName("parametric_helloWorld_%n")
  exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
  job.setInputSandbox(exeScriptLocation)
  job.setParameterSequence("args", ['one', 'two', 'three'])
  job.setParameterSequence("iargs", [1, 2, 3])
  job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
  return job

def helloWorldJob():
  job = Job()
  job.setName( "helloWorld" )
  job.setInputSandbox( '../../Integration/exe-script.py' )
  job.setExecutable( "exe-script.py", "", "helloWorld.log" )
  return job