Python WMInit.setLogging方法代码示例

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
    def __call__(self, filesetToProcess):
        """
        The algorithm itself
        """

        # Get configuration
        initObj = WMInit()
        initObj.setLogging()
        initObj.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        myThread = threading.currentThread()

        daofactory = DAOFactory(package = "WMCore.WMBS" , \
              logger = myThread.logger, \
              dbinterface = myThread.dbi)

        locationNew = daofactory(classname = "Locations.New")
        getFileLoc = daofactory(classname = "Files.GetLocation")
        fileInFileset = daofactory(classname = "Files.InFileset")


        logging.debug("DBSFeeder is processing %s" % \
                 filesetToProcess.name)
        logging.debug("the filesetBase name  %s" \
       % (filesetToProcess.name).split(":")[0])

        LASTIME = filesetToProcess.lastUpdate

        # Get the start Run if asked
        startRun = (filesetToProcess.name).split(":")[3]

        # get list of files
        tries = 1

        while True:

            try:

                blocks = self.dbsReader.getFiles(\
              (filesetToProcess.name).split(":")[0])
                now = time.time()
                logging.debug("DBS queries done ...")

                break

            except DBSReaderError, ex:
                logging.error("DBS error: %s, cannot get files for %s" % \
                      (str(ex), filesetToProcess.name))
                # Close fileset
                filesetToProcess.markOpen(False)
                return

            # connection error, retry
            except DbsConnectionError, ex:
                logging.error("Unable to connect to DBS, retrying: " + \
                      str(ex))
                if tries > self.connectionAttempts: #too many errors - bail out
                    return
                tries = tries + 1

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
if __name__ == '__main__':

    if len(sys.argv) < 2:
        usage()
        sys.exit(1)

    datasetPath = sys.argv[1]

    if len(sys.argv) < 3:  #No config path sent along
        configPath = os.getenv('WMAGENT_CONFIG', None)
    else:
        configPath = sys.argv[2]
        if not os.path.isfile(configPath):
            configPath = os.getenv('WMAGENT_CONFIG', None)

    if configPath:
        config = loadConfigurationFile(configPath)
    else:
        print "Error!  No config could be found"
        sys.exit(2)


    wmInit = WMInit()
    wmInit.setLogging()
    wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                 config.CoreDatabase.dialect,
                                 config.CoreDatabase.socket)

    migrateFileBlocks = MigrateFileBlocks(config = config)
    migrateFileBlocks.migrateDataset(datasetPath = datasetPath)

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
    def __call__(self, filesetToProcess):
        """
        The algorithm itself
        """

        # Get configuration
        initObj = WMInit()
        initObj.setLogging()
        initObj.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        myThread = threading.currentThread()

        daofactory = DAOFactory(package = "WMCore.WMBS" , \
              logger = myThread.logger, \
              dbinterface = myThread.dbi)

        lastFileset = daofactory(classname = "Fileset.ListFilesetByTask")
        lastWorkflow = daofactory(classname = "Workflow.LoadFromTask")
        subsRun = daofactory(\
classname = "Subscriptions.LoadFromFilesetWorkflow")
        successJob = daofactory(classname = "Subscriptions.SucceededJobs")
        allJob = daofactory(classname = "Subscriptions.Jobs")
        fileInFileset = daofactory(classname = "Files.InFileset")


        # Get the start Run if asked
        startRun = (filesetToProcess.name).split(":")[3]
        logging.debug("the T0Feeder is processing %s" % \
                 filesetToProcess.name)
        logging.debug("the fileset name %s" % \
         (filesetToProcess.name).split(":")[0])

        fileType = (filesetToProcess.name).split(":")[2]
        crabTask = filesetToProcess.name.split(":")[0]
        LASTIME = filesetToProcess.lastUpdate

        tries = 1
        while True:

            try:

                myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
                requestResult = myRequester.get("/tier0/runs")

            except:

                logging.debug("T0Reader call error...")
                if tries == self.maxRetries:
                    return
                else:
                    tries += 1
                    continue


            logging.debug("T0ASTRunChain feeder queries done ...")
            now = time.time()

            break


        for listRun in requestResult[0]:


            if startRun != 'None' and int(listRun['run']) >= int(startRun):
                if listRun['status'] =='CloseOutExport' or listRun\
        ['status']=='Complete' or listRun['status']=='CloseOutT1Skimming':

                    crabWorkflow = lastWorkflow.execute(task=crabTask)

                    crabFileset = lastFileset.execute\
                                (task=crabTask)

                    crabrunFileset = Fileset(\
    name = crabFileset[0]["name"].split(':')[0].split\
   ('-Run')[0]+ '-Run' + str(listRun['run']) + ":" + \
     ":".join(crabFileset[0]['name'].split(':')[1:]) )

                    if crabrunFileset.exists() > 0:

                        crabrunFileset.load()
                        currSubs = subsRun.execute\
           (crabrunFileset.id, crabWorkflow[0]['id'])

                        if currSubs:

                            listsuccessJob = successJob.execute(\
                                 subscription=currSubs['id'])
                            listallJob = allJob.execute(\
                                subscription=currSubs['id'])

                            if len(listsuccessJob) == len(listallJob):

                                for currid in listsuccessJob:
                                    currjob = Job( id = currid )
                                    currjob.load()

                                    logging.debug("Reading FJR %s" %currjob['fwjr_path'])

                                    jobReport = readJobReport(currjob['fwjr_path'])
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
class TestInit:
    """
    A set of initialization steps used in many tests.
    Test can call the methods from this class to 
    initialize their default environment so to 
    minimize code duplication.
    """ 

    def __init__(self, testClassName = "Unknown Class"):
        self.testClassName = testClassName
        self.testDir = None
        self.currModules = []
        global hasDatabase
        self.hasDatabase = hasDatabase
        if self.hasDatabase:
            self.init = WMInit()
        self.deleteTmp = True
    
    def __del__(self):
        if self.deleteTmp:
            self.delWorkDir()
        self.attemptToCloseDBConnections()

    
    def delWorkDir(self):
        if (self.testDir != None):
            try:
                shutil.rmtree( self.testDir )
            except:
                # meh, if it fails, I guess something weird happened
                pass

    def setLogging(self, logLevel = logging.INFO):
        """
        Sets logging parameters
        """
        # remove old logging instances.
        return
        logger1 = logging.getLogger()
        logger2 = logging.getLogger(self.testClassName)
        for logger in [logger1, logger2]:
            for handler in logger.handlers:
                handler.close()
                logger.removeHandler(handler)

        self.init.setLogging(self.testClassName, self.testClassName,
                             logExists = False, logLevel = logLevel)
    
    def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
        self.testDir = tempfile.mkdtemp()
        if config:
            config.section_("General")
            config.General.workDir = self.testDir
        os.environ['TESTDIR'] = self.testDir
        if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
            deleteOnDestruction = True
            logging.info("Generated testDir - %s" % self.testDir)
        if setTmpDir:
            os.environ['TMPDIR'] = self.testDir

        self.deleteTmp = deleteOnDestruction
        return self.testDir
        
    def getBackendFromDbURL(self, dburl):
        dialectPart = dburl.split(":")[0]
        if dialectPart == 'mysql':
            return 'MySQL'
        elif dialectPart == 'sqlite':
            return 'SQLite'
        elif dialectPart == 'oracle':
            return 'Oracle'
        else:
            raise RuntimeError, "Unrecognized dialect %s" % dialectPart
        
    def eraseEverythingInDatabase(self):
        if not self.hasDatabase:
            return
        if trashDatabases:
            dbi = self.getDBInterface()
            dialect = self.coreConfig.CoreDatabase.dialect
            formatter = DBFormatter(logging.getLogger(''), dbi)
            if (dialect == 'MySQL'):
                formatter.sql = r"SHOW TABLES"
                result = formatter.execute()
                formatter.sql = "SET foreign_key_checks = 0"
                formatter.execute()
                allTables = []
                for oneTable in result:
                    allTables.append( oneTable[0] )
                    
                if len(allTables) == 0:
                    return
                
                query = "DROP TABLE IF EXISTS `%s`" % ("`,`".join(allTables))
                formatter.sql = query
                formatter.execute()

                formatter.sql = "SET foreign_key_checks = 1"
                formatter.execute()
                
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
class TestInit(object):
    """
    A set of initialization steps used in many tests.
    Test can call the methods from this class to
    initialize their default environment so to
    minimize code duplication.
    """

    def __init__(self, testClassName = "Unknown Class"):
        self.testClassName = testClassName
        self.testDir = None
        self.currModules = []
        global hasDatabase
        self.hasDatabase = hasDatabase
        if self.hasDatabase:
            self.init = WMInit()
        self.deleteTmp = True

    def __del__(self):
        if self.deleteTmp:
            self.delWorkDir()
        self.attemptToCloseDBConnections()


    def delWorkDir(self):
        if self.testDir != None:
            try:
                shutil.rmtree( self.testDir )
            except:
                # meh, if it fails, I guess something weird happened
                pass

    def setLogging(self, logLevel = logging.INFO):
        """
        Sets logging parameters
        """
        # remove old logging instances.
        return
        logger1 = logging.getLogger()
        logger2 = logging.getLogger(self.testClassName)
        for logger in [logger1, logger2]:
            for handler in logger.handlers:
                handler.close()
                logger.removeHandler(handler)

        self.init.setLogging(self.testClassName, self.testClassName,
                             logExists = False, logLevel = logLevel)

    def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
        self.testDir = tempfile.mkdtemp()
        if config:
            config.section_("General")
            config.General.workDir = self.testDir
        os.environ['TESTDIR'] = self.testDir
        if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
            deleteOnDestruction = True
            logging.info("Generated testDir - %s" % self.testDir)
        if setTmpDir:
            os.environ['TMPDIR'] = self.testDir

        self.deleteTmp = deleteOnDestruction
        return self.testDir

    def getBackendFromDbURL(self, dburl):
        dialectPart = dburl.split(":")[0]
        if dialectPart == 'mysql':
            return 'MySQL'
        elif dialectPart == 'oracle':
            return 'Oracle'
        elif dialectPart == 'http':
            return 'CouchDB'
        else:
            raise RuntimeError("Unrecognized dialect %s" % dialectPart)

    def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
        """
        Set up the database connection by retrieving the environment
        parameters.

        The destroyAllDatabase option is for testing ONLY.  Never flip that switch
        on in any other instance where you don't know what you're doing.
        """
        if not self.hasDatabase:
            return
        config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
        self.coreConfig = config
        self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                        config.CoreDatabase.dialect,
                                        config.CoreDatabase.socket)

        if trashDatabases or destroyAllDatabase:
            self.clearDatabase()

        # Have to check whether or not database is empty
        # If the database is not empty when we go to set the schema, abort!
        try:
            result = self.init.checkDatabaseContents()
        except OperationalError:
            logging.debug("Error checking DB contents, assume DB does not exist")
            return
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
class CrabJobCreatorComponent:
    """
    _CrabJobCreatorComponent_
    
    """
    
################################
#   Standard Component Core Methods      
################################
    def __init__(self, **args):
        self.args = {}
        
        self.args.setdefault('Logfile', None)
        self.args.setdefault('CacheDir', None)
        self.args.setdefault('ProxiesDir', None)
        self.args.setdefault('CopyTimeOut', '')

        # SE support parameters
        # Protocol = local cannot be the default. Any default allowd 
        # for this parameter... it must be defined from config file. 
        self.args.setdefault('Protocol', '')
        self.args.setdefault('storageName', 'localhost')
        self.args.setdefault('storagePort', '')
        self.args.setdefault('storagePath', self.args["CacheDir"])

        # specific delegation strategy for glExex
        self.args.setdefault('glExecDelegation', 'false')

        self.args.setdefault('PollInterval',60)
        self.args.setdefault("HeartBeatDelay", "00:05:00")
        self.args.update(args)

        if len(self.args["HeartBeatDelay"]) != 8:
            self.HeartBeatDelay="00:05:00"
        else:
            self.HeartBeatDelay=self.args["HeartBeatDelay"]
        
        # define log file
        if self.args['Logfile'] == None:
            self.args['Logfile'] = os.path.join(self.args['ComponentDir'],
                                                "ComponentLog")
        # create log handler
        logHandler = RotatingFileHandler(self.args['Logfile'],
                                         "a", 1000000, 7)
        # define log format
        logFormatter = logging.Formatter("%(asctime)s:%(message)s")
        logHandler.setFormatter(logFormatter)
        logging.getLogger().addHandler(logHandler)
        logging.getLogger().setLevel(logging.INFO)

        ## volatile properties
        self.wdir = self.args['ComponentDir']
        self.maxThreads = int( self.args.get('maxThreads', 5) )
        self.timePoolDB = self.args['PollInterval']

        # shared sessions 
        self.blDBsession = BossLiteAPI('MySQL', dbConfig, makePool=True)
        self.sessionPool = self.blDBsession.bossLiteDB.getPool()
        self.workerPool = self.blDBsession.bossLiteDB.getPool()

        # Get configuration
        self.init = WMInit()
        self.init.setLogging()
        self.init.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        self.myThread = threading.currentThread()
        self.factory = WMFactory("msgService", "WMCore.MsgService."+ \
                             self.myThread.dialect)
        self.newMsgService = self.myThread.factory['msgService']\
                          .loadObject("MsgService")

        self.ms = MessageService()

        self.workerCfg = {}
        logging.info(" ")
        logging.info("Starting component...")
    
    def startComponent(self):
        """
        _startComponent_
        Start up the component
        """
        # Registration in oldMsgService 
        self.ms.registerAs("CrabJobCreatorComponent")
        
        self.ms.subscribeTo("JobFailed")
        self.ms.subscribeTo("JobSuccess")
        self.ms.subscribeTo("CrabJobCreatorComponent:EndDebug")

        # Registration in new MsgService
        self.myThread.transaction.begin()
        self.newMsgService.registerAs("CrabJobCreatorComponent")
        self.myThread.transaction.commit()

        self.ms.subscribeTo("CrabJobCreatorComponent:HeartBeat")
        self.ms.remove("CrabJobCreatorComponent:HeartBeat")
        self.ms.publish("CrabJobCreatorComponent:HeartBeat","",self.HeartBeatDelay)
        self.ms.commit()
        self.workerCfg = self.prepareBaseStatus()      
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.WMInit import WMInit [as 别名]
# 或者: from WMCore.WMInit.WMInit import setLogging [as 别名]
class TestInit:
    """
    A set of initialization steps used in many tests.
    Test can call the methods from this class to 
    initialize their default environment so to 
    minimize code duplication.
    """ 

    def __init__(self, testClassName = "Unknown Class"):
        self.testClassName = testClassName
        self.testDir = None
        self.currModules = []
        global hasDatabase
        self.hasDatabase = hasDatabase
        if self.hasDatabase:
            self.init = WMInit()
        self.deleteTmp = True
    
    def __del__(self):
        if self.deleteTmp:
            self.delWorkDir()
        self.attemptToCloseDBConnections()

    
    def delWorkDir(self):
        if (self.testDir != None):
            try:
                shutil.rmtree( self.testDir )
            except:
                # meh, if it fails, I guess something weird happened
                pass

    def setLogging(self, logLevel = logging.INFO):
        """
        Sets logging parameters
        """
        # remove old logging instances.
        return
        logger1 = logging.getLogger()
        logger2 = logging.getLogger(self.testClassName)
        for logger in [logger1, logger2]:
            for handler in logger.handlers:
                handler.close()
                logger.removeHandler(handler)

        self.init.setLogging(self.testClassName, self.testClassName,
                             logExists = False, logLevel = logLevel)
    
    def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
        self.testDir = tempfile.mkdtemp()
        if config:
            config.section_("General")
            config.General.workDir = self.testDir
        os.environ['TESTDIR'] = self.testDir
        if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
            deleteOnDestruction = True
            logging.info("Generated testDir - %s" % self.testDir)
        if setTmpDir:
            os.environ['TMPDIR'] = self.testDir

        self.deleteTmp = deleteOnDestruction
        return self.testDir
        
    def getBackendFromDbURL(self, dburl):
        dialectPart = dburl.split(":")[0]
        if dialectPart == 'mysql':
            return 'MySQL'
        elif dialectPart == 'sqlite':
            return 'SQLite'
        elif dialectPart == 'oracle':
            return 'Oracle'
        elif dialectPart == 'http':
            return 'CouchDB'
        else:
            raise RuntimeError, "Unrecognized dialect %s" % dialectPart
        
    def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
        """
        Set up the database connection by retrieving the environment
        parameters.

        The destroyAllDatabase option is for testing ONLY.  Never flip that switch
        on in any other instance where you don't know what you're doing.
        """
        if not self.hasDatabase:
            return        
        config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
        self.coreConfig = config
        self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                        config.CoreDatabase.dialect,
                                        config.CoreDatabase.socket)

        if trashDatabases or destroyAllDatabase:
            self.clearDatabase()

        # Have to check whether or not database is empty 
        # If the database is not empty when we go to set the schema, abort! 
        result = self.init.checkDatabaseContents() 
        if len(result) > 0: 
            msg =  "Database not empty, cannot set schema !\n" 
#.........这里部分代码省略.........