Python Dirac.getOutputSandbox方法代码示例

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
  else:
    outputDir = group

if outputDir:
  if not os.path.exists( outputDir ):
    os.makedirs( outputDir )
else:
  outputDir = os.getcwd()

jobs = [ str( job ) for job in jobs ]
doneJobs = os.listdir( outputDir )
todoJobs = [ job for job in jobs if not job in doneJobs ]

for job in todoJobs:

  result = dirac.getOutputSandbox( job, outputDir = outputDir )

  jobDir = str( job )
  if outputDir:
    jobDir = os.path.join( outputDir, job )
  if result['OK']:
    if os.path.exists( jobDir ):
      print 'Job output sandbox retrieved in %s/' % ( jobDir )
  else:
    if os.path.exists( '%s' % jobDir ):
      shutil.rmtree( jobDir )
    errorList.append( ( job, result['Message'] ) )
    exitCode = 2

for error in errorList:
  print "ERROR %s: %s" % error

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
jobName = sys.argv[1]

finalStatus = ['Done', 'Failed']

dirac = Dirac()

idstr = open("%s/jobIdList.txt"%jobName, 'r').readlines()
ids = map(int, idstr)
print "found %s jobs"%(len(ids))

res = dirac.getJobSummary(ids)
if not res['OK']:
  print res['Message']
  sys.exit(1)

metadata = res['Value']

for jid in ids:
  jobMeta = metadata.get( jid, None )
  if not jobMeta :
    print "No metadata for job ", jid
    continue

  status = jobMeta['Status']
  print "%s %s" % ( jid, status )
  if status  in finalStatus:
    outputDir =  '%s/%s'%(jobName,status)
    if not os.path.exists( "%s/%s" % ( outputDir, jid ) ):
      print "Retrieving sandbox"
      res = dirac.getOutputSandbox( jid, outputDir = outputDir )

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
                    # check if 'jobmeta.inf' is present (if not it's not a PIPELINE job )
                    if not os.path.isfile(dir_temp + "/InputSandbox" + j + "/jobmeta.inf"):
                        print "WARNING : not a pipeline task"
                        # notify the job as "already handled"
                        jobid_handled.append(j)
                    else:

                        # Get the working dir of the task from 'jobmeta.inf'
                        file_jobmeta = open(dir_temp + "/InputSandbox" + j + "/jobmeta.inf", "r")
                        workdir = file_jobmeta.readline().splitlines()[0]
                        # TEST workdir = dir_temp+"/"+file_jobmeta.readline().splitlines()[0]
                        file_jobmeta.close()

                        # retrieve the OUTPUT sandbox
                        res = d.getOutputSandbox(j, dir_temp)
                        if not res["OK"]:
                            print res["Message"]
                            sys.stderr.write(
                                time.strftime("%d/%m/%y %H:%M", time.localtime())
                                + " => "
                                + j
                                + " => "
                                + res["Message"]
                                + "\n"
                            )
                        elif not os.path.isfile(dir_temp + "/" + j + "/jobmeta.inf"):
                            print "ERROR : no jobmeta.inf file in the outpusandbox"
                            sys.stderr.write(
                                time.strftime("%d/%m/%y %H:%M", time.localtime())
                                + " => "

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]

#.........这里部分代码省略.........
      os.environ[ 'X509_USER_PROXY' ] = oldProxy
    LOCK.release()

    return result


  def getTestResult( self, elementName, vo, jobID, submissionTime ):
    """
      download output sandbox and judge the test status from the log file.
    """

    isFinish = False

    res = self.__getJobOutput( jobID, vo )
    if not res[ 'OK' ]:
      return res
    output = res[ 'Value' ]
    status = res[ 'Status' ]

    resDict = { 'CompletionTime' : None, 'Status' : None, 'Log' : None, 'ApplicationTime' : None }
    utcNow = datetime.utcnow().replace( microsecond = 0 )

    if output:
      isFinish = True
      resDict[ 'CompletionTime' ] = utcNow
      log = output[ 'Log' ]
      if not output[ 'Download' ]:
        resDict[ 'Status' ] = 'Unknown'
        resDict[ 'Log' ] = 'Fail to download log file for job %s: %s' % ( jobID, log )
      else:
        resDict[ 'Log' ] = log
        resDict[ 'Status' ] = self._judge( log )
        resDict[ 'AppliactionTime' ] = self.__getAppRunningTime( log )

    else:
      if utcNow - submissionTime >= timedelta( seconds = self.timeout ):
        isFinish = True
        if elementName.split( '.' )[ 0 ] == 'CLOUD':
          site = elementName
        else:
          site = BESUtils.getSiteForCE( elementName )
        jobCount = self.wmsAdmin.getSiteSummaryWeb( { 'Site' : site }, [], 0, 0 )
        if not jobCount[ 'OK' ]:
          return jobCount
        params = jobCount[ 'Value' ][ 'ParameterNames' ]
        records = jobCount[ 'Value' ][ 'Records' ][ 0 ]
        run = records[ params.index( 'Running' ) ]
        done = records[ params.index( 'Done' ) ]
        if status == 'Waiting' and run == 0 and done == 0:
          resDict[ 'Status' ] = 'Bad'
          resDict[ 'Log' ] = 'The test job is waiting for %d seconds, but no running and done jobs at this site.' % self.timeout
        else:
          if run != 0:
            resDict[ 'Status' ] = 'Busy'
            resDict[ 'Log' ] = 'Site %s is too busy to execute this test job, job status is %s' % ( site, status )
          else:
            resDict[ 'Status' ] = 'Unknown'
            resDict[ 'Log' ] = 'Test did not complete within the timeout of %d seconds, job status is %s' % ( self.timeout, status )
        self.dirac.kill( jobID )

    if not isFinish:
      return S_OK()
    else:
      return S_OK( resDict )


  def __getJobOutput( self, jobID, vo ):
    status = self.dirac.status( jobID )
    if not status[ 'OK' ]:
      return status
    status = status[ 'Value' ][ jobID ][ 'Status' ]

    if status in ( 'Done', 'Failed' ):
      LOCK.acquire()
      proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
      if not proxyPath[ 'OK' ]:
        LOCK.release()
        return proxyPath
      proxyPath = proxyPath[ 'Value' ]
      oldProxy = os.environ.get( 'X509_USER_PROXY' )
      os.environ[ 'X509_USER_PROXY' ] = proxyPath
      outputRes = self.dirac.getOutputSandbox( jobID, self.__logPath )
      if oldProxy is None:
        del os.environ[ 'X509_USER_PROXY' ]
      else:
        os.environ[ 'X509_USER_PROXY' ] = oldProxy
      LOCK.release()

      if not outputRes[ 'OK' ]:
        ret = S_OK( { 'Download'  : False, 'Log' : outputRes[ 'Message' ] } )
      else:
        try:
          logfile = open( '%s/%d/Script1_CodeOutput.log' % ( self.__logPath, jobID ), 'r' )
          log = logfile.read()
          logfile.close()
        except IOError, e:
          raise IOError
        os.system( 'rm -rf %s/%d' % ( self.__logPath, jobID ) )
        ret = S_OK( { 'Download' : True, 'Log' : log } )
    else:

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}

result = dirac.selectJobs( status = status, minorStatus = minorStatus, applicationStatus = appStatus,
                           site = site, owner = owner, jobGroup = jobGroup, date = date )
if result['OK']:
  jobs = result['Value']
else:
  print "Error in selectJob", result['Message']
  DIRAC.exit( 2 )

for job in jobs:

  result = dirac.getOutputSandbox( job )
  if result['OK']:
    if os.path.exists( '%s' % job ):

      lines = []
      try:
        lines = open( os.path.join( job, filename ) ).readlines()
      except Exception, x:
        errorList.append( ( job, x ) )
      for line in lines:
        if line.count( searchstring ):
          resultDict[job] = line
      rmtree( "%s" % ( job ) )
  else:
    errorList.append( ( job, result['Message'] ) )
    exitCode = 2

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac

if (len(sys.argv) < 2) :
    print 'the input file with ids should be specified'
    print 'if the 3rd argument is <get_output> the job output sandbox will be downloaded'
    sys.exit(os.EX_USAGE)

list = sys.argv[1]

get_output = False
if (len(sys.argv) > 2):
    if (sys.argv[2] == 'get_output'): get_output = True

id_list_file = open(list, 'r')

for line in id_list_file:
##    line = line.strip().decode("utf-8").replace("True","true").replace("False","false")
    line = line.replace("True","true").replace("False","false")
    line = line.replace("'","\"")
    j = json.loads(line)

    dirac = Dirac()
    print dirac.status(j['Value'])

    if get_output: print dirac.getOutputSandbox(j['Value'])

id_list_file.close()

# 需要导入模块: from DIRAC.Interfaces.API.Dirac import Dirac [as 别名]
# 或者: from DIRAC.Interfaces.API.Dirac.Dirac import getOutputSandbox [as 别名]
#!/bin/env python

import sys

from DIRAC.Core.Base import Script
Script.parseCommandLine()

from DIRAC.Interfaces.API.Job import Job
from DIRAC.Interfaces.API.Dirac import Dirac

dirac = Dirac()
jobid = sys.argv[1]
print dirac.getOutputSandbox(jobid)