Python JobDB.JobDB类代码示例

def initSites():
  '''
    Initializes Sites statuses taking their values from the "SiteMask" table of "JobDB" database.
  '''

  jobClient = JobDB()
  rssClient = ResourceStatusDB()

  sites = jobClient.getAllSiteMaskStatus()

  if not sites[ 'OK' ]:
    subLogger.error( sites[ 'Message' ] )
    DIRACExit( 1 )

  for site, elements in sites['Value'].iteritems():
    table  = { 'table': 'SiteStatus' }
    insert = { 'Status': elements[0], 'Reason': 'Synchronized', 'Name': site, 'DateEffective': elements[1], 'TokenExpiration': Datetime,
             'ElementType': 'Site', 'StatusType': 'all', 'LastCheckTime': None, 'TokenOwner': elements[2] }

    result = rssClient.addIfNotThere(insert, table)

    if not result[ 'OK' ]:
      subLogger.error( result[ 'Message' ] )
      DIRACExit( 1 )

  return S_OK()

    def doNew(self, masterParams=None):

        hosts = masterParams

        sql = """
select JP.Value, J.Status, J.Site, count(*) from Jobs J, JobParameters JP 
where J.JobID = JP.JobID and JP.Name = 'HostName' 
and J.EndExecTime >= DATE_SUB(UTC_TIMESTAMP(),INTERVAL 24 HOUR) 
group by JP.Value, J.Status
"""

        jobDB = JobDB()
        queryRes = jobDB._query(sql)
        if not queryRes["OK"]:
            return queryRes
        records = queryRes["Value"]

        hostJobs = {}

        for record in records:
            hostName = record[0]
            status = record[1]
            if status != "Done" and status != "Failed":
                continue
            if hostName not in hostJobs:
                hostJobs[hostName] = {"Site": record[2], "Done": 0, "Failed": 0}
            hostJobs[hostName][record[1]] = record[3]

        uniformResult = []

        for host, hostDict in hostJobs.items():
            hostDict["Host"] = host
            try:
                hosts.remove(host)
            except ValueError:
                pass
            if hostDict["Done"] == 0 and hostDict["Failed"] == 0:
                hostDict["Efficiency"] = 0.0
            else:
                hostDict["Efficiency"] = (
                    math.floor(float(hostDict["Done"]) / (hostDict["Done"] + hostDict["Failed"]) * 1000) / 10
                )
            uniformResult.append(hostDict)

        if len(hosts) != 0:
            deleteRes = self.rmClient.deleteWorkNodeCache(host=hosts)
            if not deleteRes["OK"]:
                return deleteRes

        storeRes = self._storeCommand(uniformResult)
        if not storeRes["OK"]:
            return storeRes

        return S_OK(uniformResult)

def mysql_querry(querry):
    if USE_PURE_MYSQL:
        import MySQLdb#@UnresolvedImport
        db = MySQLdb.connect(host="diracdb2.ihep.ac.cn", # your host, usually localhost
                         user="monitor", # your username
                         passwd="###DIRAC_DB_PASS###", # your password
                         db="JobDB")
        cur = db.cursor()
        cur.execute(querry)
        data = cur.fetchall()
        cur.close()
        return data
    else:
        db = JobDB()
        return db._query( querry )['Value']

class JobDBTestCase( unittest.TestCase ):
  """ Base class for the JobDB test cases
  """

  def setUp( self ):
    gLogger.setLevel( 'DEBUG' )
    self.jobDB = JobDB()

  def tearDown( self ):
    result = self.jobDB.selectJobs( {} )
    self.assert_( result['OK'], 'Status after selectJobs' )
    jobs = result['Value']
    for job in jobs:
      result = self.jobDB.removeJobFromDB( job )
      self.assert_( result['OK'] )

class JobDBTest( unittest.TestCase ):

  def setUp( self ):

    def mockInit(self ):
      self.log = MagicMock()
      self.logger = MagicMock()
      self._connected = True

    from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
    with patch( MODULE_NAME+".JobDB.__init__", new=mockInit):
      self.jobDB = JobDB()
    self.jobDB._query = MagicMock( name="Query" )
    self.jobDB._escapeString = MagicMock( return_value=S_OK() )

  def tearDown( self ):
    pass


  def test_getInputData( self ):
    self.jobDB._query.return_value = S_OK( (( '/vo/user/lfn1',), ('LFN:/vo/user/lfn2',)) )
    result = self.jobDB.getInputData( 1234 )
    print result
    self.assertTrue( result['OK'] )
    self.assertEqual( result['Value'], [ '/vo/user/lfn1', '/vo/user/lfn2' ] )

class JobDBTestCase(unittest.TestCase):
  """ Base class for the JobDB test cases
  """
  
  def setUp(self):
    print
    self.jobDB = JobDB('Test',20)
    
  def createJob(self):
    result = self.jobDB.getJobID()
    jobID = result['Value']
    jdlfile = open("test.jdl","r")
    jdl = jdlfile.read()
    jdlfile.close()
    result = self.jobDB.insertJobIntoDB(jobID,jdl) 
    return jobID  

  def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ):
    """ c'tor
    """
    if pilotAgentsDB:
      self.pilotAgentsDB = pilotAgentsDB
    else:
      self.pilotAgentsDB = PilotAgentsDB()
    if jobDB:
      self.jobDB = jobDB
    else:
      self.jobDB = JobDB()
    if tqDB:
      self.tqDB = tqDB
    else:
      self.tqDB = TaskQueueDB()
    if jlDB:
      self.jlDB = jlDB
    else:
      self.jlDB = JobLoggingDB()

    if opsHelper:
      self.opsHelper = opsHelper
    else:
      self.opsHelper = Operations()

    self.log = gLogger.getSubLogger( "Matcher" )

    self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper )

  def initialize( self, jobDB = False, logDB = False ):
    """ Initialization of the Optimizer Agent.
    """
    if not jobDB:
      self.jobDB = JobDB()
    else:
      self.jobDB = jobDB
    if not logDB:
      self.logDB = JobLoggingDB()
    else:
      self.logDB = logDB

    trailing = "Agent"
    optimizerName = self.am_getModuleParam( 'agentName' )
    if optimizerName[ -len( trailing ):].find( trailing ) == 0:
      optimizerName = optimizerName[ :-len( trailing ) ]
    self.am_setModuleParam( 'optimizerName', optimizerName )

    self.startingMinorStatus = self.am_getModuleParam( 'optimizerName' )
    self.startingMajorStatus = "Checking"
    self.failedStatus = self.am_getOption( "FailedJobStatus" , 'Failed' )
    self.requiredJobInfo = 'jdl'
    self.am_setOption( "PollingTime", 30 )

    return self.initializeOptimizer()

  def initialize( self ):
    """ Sets defaults
    """

    self.am_setOption( "PollingTime", 120 )
    self.jobDB = JobDB()
    self.taskQueueDB = TaskQueueDB()
    self.jobLoggingDB = JobLoggingDB()
    # self.sandboxDB = SandboxDB( 'SandboxDB' )
    agentTSTypes = self.am_getOption('ProductionTypes', [])
    if agentTSTypes:
      self.prod_types = agentTSTypes
    else:
      self.prod_types = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
    gLogger.info( "Will exclude the following Production types from cleaning %s" % ( ', '.join( self.prod_types ) ) )
    self.maxJobsAtOnce = self.am_getOption( 'MaxJobsAtOnce', 500 )
    self.jobByJob = self.am_getOption( 'JobByJob', False )
    self.throttlingPeriod = self.am_getOption('ThrottlingPeriod', 0.)
    
    self.removeStatusDelay['Done'] = self.am_getOption( 'RemoveStatusDelay/Done', 7 )
    self.removeStatusDelay['Killed'] = self.am_getOption( 'RemoveStatusDelay/Killed', 7 )
    self.removeStatusDelay['Failed'] = self.am_getOption( 'RemoveStatusDelay/Failed', 7 )
    self.removeStatusDelay['Any'] = self.am_getOption( 'RemoveStatusDelay/Any', -1 )

    return S_OK()

 def initialize( self ):
   """ Standard constructor
   """
   self.jobDB = JobDB()
   self.jobLoggingDB = JobLoggingDB()
   self._optimizers = {}
   self.am_setOption( "PollingTime", 30 )
   return S_OK()

  def initialize( self ):
    """Sets default parameters
"""
    self.jobDB = JobDB()
    self.logDB = JobLoggingDB()
    self.am_setOption( 'PollingTime', 60 * 60 )
    if not self.am_getOption( 'Enable', True ):
      self.log.info( 'Stalled Job Agent running in disabled mode' )
    return S_OK()

class JobHistoryAgent( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  def initialize( self ):

    self.jobDB = JobDB()

    for status in MONITOR_STATUS:
      for site in MONITOR_SITES:
        gLogger.verbose( "Registering activity %s-%s" % ( status, site ) )
        gLogger.verbose( "Jobs in %s state at %s" % ( status, site ) )
        gMonitor.registerActivity( "%s-%s" % ( status, site ), "Jobs in %s state at %s" % ( status, site ),
                                  "JobHistoryAgent", "Jobs/minute", gMonitor.OP_MEAN )

    self.last_update = 0
    self.resultDB = None
    self.reportPeriod = 60
    return S_OK()

  def execute( self ):
    """ Main execution method
    """
    delta = time.time() - self.last_update
    if delta > self.reportPeriod:
      result = self.jobDB.getCounters( 'Jobs', ['Status', 'Site'], {}, '' )
      if not result['OK']:
        return S_ERROR( 'Failed to get data from the Job Database' )
      self.resultDB = result['Value']
      self.last_update = time.time()

    totalDict = {}
    for status in MONITOR_STATUS:
      totalDict[status] = 0

    for row in self.resultDB:
      site = row[0]['Site']
      status = row[0]['Status']
      count = row[1]
      if site in MONITOR_SITES and status in MONITOR_STATUS:
        gLogger.verbose( "Adding mark %s-%s: " % ( status, site ) + str( count ) )
        gMonitor.addMark( "%s-%s" % ( status, site ), count )
      if status in totalDict:
        totalDict[status] += count

    for status in MONITOR_STATUS:
      gLogger.verbose( "Adding mark %s-All sites: " % status + str( totalDict[status] ) )
      gMonitor.addMark( "%s-All sites" % status, totalDict[status] )

    return S_OK()

  def initialize( self ):
    """Sets defaults
    """

    self.am_setOption( 'PollingTime', 120 )
    self.am_setOption( 'GridEnv', '' )
    self.am_setOption( 'PilotStalledDays', 3 )
    self.pilotDB = PilotAgentsDB()
    self.diracadmin = DiracAdmin()
    self.jobDB = JobDB()
    return S_OK()

  def setUp( self ):

    def mockInit(self ):
      self.log = MagicMock()
      self.logger = MagicMock()
      self._connected = True

    from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
    with patch( MODULE_NAME+".JobDB.__init__", new=mockInit):
      self.jobDB = JobDB()
    self.jobDB._query = MagicMock( name="Query" )
    self.jobDB._escapeString = MagicMock( return_value=S_OK() )

    def getDataFromDB(self):
           db = JobDB()
           cmd = 'SELECT * FROM Jobs'
           res = db._query( cmd ) 

           #Week data
           cmd = "select J.Site, JP.Value, sum(1),  S.total from Jobs J, JobParameters JP, (select JP.Value as host, sum(1) as total from Jobs J, JobParameters JP where J.JobID=JP.JobID and JP.Name='HostName' group by JP.Value) S where J.JobID=JP.JobID and JP.Name='HostName' and J.Status='Done' and S.host=JP.Value group by JP.Value order by J.Site asc"

           result = mysql_querry( cmd )
           #print result
           data = {}
           for i in result:
             data[(i[0], i[1])]=[int(i[2]), int(i[3])]

           #TwoDays data
           cmd = "select J.Site, JP.Value, sum(1), S.total from Jobs J, JobParameters JP, (select JP.Value as host, sum(1) as total from Jobs J, JobParameters JP where J.JobID=JP.JobID and JP.Name='HostName' and J.SubmissionTime>= DATE_SUB(UTC_TIMESTAMP(),INTERVAL 48 HOUR) group by JP.Value) S where J.JobID=JP.JobID and JP.Name='HostName' and J.Status='Done' and S.host=JP.Value and J.SubmissionTime>= DATE_SUB(UTC_TIMESTAMP(),INTERVAL 48 HOUR) group by JP.Value order by J.Site asc"

           result = mysql_querry( cmd )
           for i in result:
             if (i[0],i[1]) not in data:
                data[(i[0],i[1])] = [0,0]
                print "Strange behavior: for ", (i[0],i[1]), " was no week data"
             data[(i[0], i[1])] += [int(i[2]), int(i[3])]


           #OneDay data
           cmd = "select J.Site, JP.Value, sum(1),  S.total from Jobs J, JobParameters JP, (select JP.Value as host, sum(1) as total from Jobs J, JobParameters JP where J.JobID=JP.JobID and JP.Name='HostName' and J.SubmissionTime>= DATE_SUB(UTC_TIMESTAMP(),INTERVAL 24 HOUR) group by JP.Value) S where J.JobID=JP.JobID and JP.Name='HostName' and J.Status='Done' and S.host=JP.Value and J.SubmissionTime>= DATE_SUB(UTC_TIMESTAMP(),INTERVAL 24 HOUR) group by JP.Value order by J.Site asc"

           result = mysql_querry( cmd )
           for i in result:
             if (i[0],i[1]) not in data:
                data[(i[0],i[1])] = [0,0]
                print "Strange behavior: for ", (i[0],i[1]), " was no week data"
             data[(i[0], i[1])] += [int(i[2]), int(i[3])]

           prejson = [[x[0], x[1]]+data[x] for x in data]
           for i in range(0, len(prejson)):
             prejson[i] += [0 for j in range(0, 8-len(prejson[i]))] 

           return prejson