Python WMInit.WMInit类代码示例

def main():
    config = loadConfigurationFile(os.environ['WMAGENT_CONFIG'])
    config.CoreDatabase.dialect = 'oracle'
    init = WMInit()
    init.setDatabaseConnection(config.CoreDatabase.connectUrl,
                               config.CoreDatabase.dialect)
    couchDB = Database('wmagent_jobdump/fwjrs', '')
    couchDB2 = Database('wmagent_jobdump/jobs', '')
    myThread = threading.currentThread()
    daofactory = DAOFactory(package = "WMCore.WMBS",
                            logger = logging,
                            dbinterface = myThread.dbi)
    getJobsDAO = daofactory(classname = "Jobs.GetAllJobs")
    completedJobs = getJobsDAO.execute(state = 'complete')
    candidates = []
    while len(completedJobs):
        candidates = []
        chunk = completedJobs[:500]
        completedJobs = completedJobs[500:]
        result = couchDB.loadView('FWJRDump', 'outputByJobID', keys = chunk)
        rows = result['rows']
        for entry in rows:
            candidates.append(entry['key'])
        for jobId in candidates:
            doc = couchDB2.document(str(jobId))
            last = max(map(int, doc['states'].keys()))
            lastState = doc['states'][str(last)]['newstate']
            if lastState == 'success':
                print jobId

    def __call__(self, filesetToProcess):
        """
        The algorithm itself
        """

        # Get configuration
        initObj = WMInit()
        initObj.setLogging()
        initObj.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        myThread = threading.currentThread()

        daofactory = DAOFactory(package = "WMCore.WMBS" , \
              logger = myThread.logger, \
              dbinterface = myThread.dbi)

        locationNew = daofactory(classname = "Locations.New")
        getFileLoc = daofactory(classname = "Files.GetLocation")
        fileInFileset = daofactory(classname = "Files.InFileset")


        logging.debug("DBSFeeder is processing %s" % \
                 filesetToProcess.name)
        logging.debug("the filesetBase name  %s" \
       % (filesetToProcess.name).split(":")[0])

        LASTIME = filesetToProcess.lastUpdate

        # Get the start Run if asked
        startRun = (filesetToProcess.name).split(":")[3]

        # get list of files
        tries = 1

        while True:

            try:

                blocks = self.dbsReader.getFiles(\
              (filesetToProcess.name).split(":")[0])
                now = time.time()
                logging.debug("DBS queries done ...")

                break

            except DBSReaderError, ex:
                logging.error("DBS error: %s, cannot get files for %s" % \
                      (str(ex), filesetToProcess.name))
                # Close fileset
                filesetToProcess.markOpen(False)
                return

            # connection error, retry
            except DbsConnectionError, ex:
                logging.error("Unable to connect to DBS, retrying: " + \
                      str(ex))
                if tries > self.connectionAttempts: #too many errors - bail out
                    return
                tries = tries + 1

def main():
    myPhedex = PhEDEx()
    config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
    config.CoreDatabase.dialect = "mysql"
    init = WMInit()
    init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket)
    myThread = threading.currentThread()
    daofactory = DAOFactory(package="WMComponent.PhEDExInjector.Database", logger=logging, dbinterface=myThread.dbi)

    getUninjectedDAO = daofactory(classname="GetUninjectedFiles")
    uninjectedFiles = getUninjectedDAO.execute()
    for location in uninjectedFiles:
        for dataset in uninjectedFiles[location]:
            for block in uninjectedFiles[location][dataset]:
                result = myPhedex.getReplicaInfoForFiles(dataset=dataset, block=block)
                phedexBlock = result["phedex"]["block"]
                if not phedexBlock:
                    continue
                phedexBlock = phedexBlock[0]
                filesInjected = [x["name"] for x in phedexBlock["file"]]
                for fileInfo in uninjectedFiles[location][dataset][block]["files"]:
                    lfn = fileInfo["lfn"]
                    if lfn in filesInjected:
                        print lfn
    return 0

def connectToDB():
    """
    _connectToDB_
    
    Connect to the database specified in the WMAgent config.
    """
    if not os.environ.has_key("WMAGENT_CONFIG"):
        print "Please set WMAGENT_CONFIG to point at your WMAgent configuration."
        sys.exit(1)
        
    if not os.path.exists(os.environ["WMAGENT_CONFIG"]):
        print "Can't find config: %s" % os.environ["WMAGENT_CONFIG"]
        sys.exit(1)

    wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
    
    if not hasattr(wmAgentConfig, "CoreDatabase"):
        print "Your config is missing the CoreDatabase section."

    socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
    connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
    (dialect, junk) = connectUrl.split(":", 1)

    myWMInit = WMInit()
    myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
                                   socketLoc = socketLoc)
    return

 def __init__(self, testClassName = "Unknown Class"):
     self.testClassName = testClassName
     self.testDir = None
     self.currModules = []
     global hasDatabase
     self.hasDatabase = hasDatabase
     if self.hasDatabase:
         self.init = WMInit()
     self.deleteTmp = True

if __name__ == '__main__':

    if len(sys.argv) < 2:
        usage()
        sys.exit(1)

    datasetPath = sys.argv[1]

    if len(sys.argv) < 3:  #No config path sent along
        configPath = os.getenv('WMAGENT_CONFIG', None)
    else:
        configPath = sys.argv[2]
        if not os.path.isfile(configPath):
            configPath = os.getenv('WMAGENT_CONFIG', None)

    if configPath:
        config = loadConfigurationFile(configPath)
    else:
        print "Error!  No config could be found"
        sys.exit(2)


    wmInit = WMInit()
    wmInit.setLogging()
    wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                 config.CoreDatabase.dialect,
                                 config.CoreDatabase.socket)

    migrateFileBlocks = MigrateFileBlocks(config = config)
    migrateFileBlocks.migrateDataset(datasetPath = datasetPath)

    def __call__(self, filesetToProcess):
        """
        The algorithm itself
        """
        global LOCK


        # Get configuration
        initObj = WMInit()
        initObj.setLogging()
        initObj.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        myThread = threading.currentThread()

        daofactory = DAOFactory(package = "WMCore.WMBS" , \
              logger = myThread.logger, \
              dbinterface = myThread.dbi)

        locationNew = daofactory(classname = "Locations.New")
        getFileLoc = daofactory(classname = "Files.GetLocation")


        logging.debug("the T0Feeder is processing %s" % \
                 filesetToProcess.name)
        logging.debug("the fileset name %s" % \
         (filesetToProcess.name).split(":")[0])

        startRun = (filesetToProcess.name).split(":")[3]
        fileType = (filesetToProcess.name).split(":")[2]

        # url builder
        primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
        processedDataset = ((filesetToProcess.name).split(":")[0]).split('/')[2]
        dataTier = (((filesetToProcess.name\
            ).split(":")[0]).split('/')[3]).split('-')[0]

        # Fisrt call to T0 db for this fileset
        # Here add test for the closed fileset
        LASTIME = filesetToProcess.lastUpdate

        url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
              (fileType, LASTIME, primaryDataset,processedDataset, dataTier)

        tries = 1
        while True:

            try:

                myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
                requestResult = myRequester.get(\
             url+"/"+"?return_type=text/json%2Bdas")
                newFilesList = requestResult[0]["results"]

            except:

                logging.debug("T0Reader call error...")
                if tries == self.maxRetries:
                    return
                else:
                    tries += 1
                    continue

            logging.debug("T0ASTRun queries done ...")
            now = time.time()
            filesetToProcess.last_update = now
            LASTIME = int(newFilesList['end_time']) + 1

            break



        # process all files
        if len(newFilesList['files']):

            LOCK.acquire()

            try:
                locationNew.execute(siteName = "caf.cern.ch", seName = "caf.cern.ch")
            except Exception as e:
                logging.debug("Error when adding new location...")
                logging.debug(e)
                logging.debug( format_exc() )

            for files in newFilesList['files']:

                # Assume parents aren't asked
                newfile = File(str(files['lfn']), \
           size = files['file_size'], events = files['events'])


                try:
                    if newfile.exists() == False :
                        newfile.create()

                    else:
                        newfile.loadData()

                    #Add run test if already exist
                    for run in files['runs']:
#.........这里部分代码省略.........

    def testB_Database(self):
        """
        _Database_

        Testing the database stuff.
        """

        init = WMInit()
        url     = os.environ.get("DATABASE")
        dialect = os.environ.get("DIALECT")
        sock    = os.environ.get("DBSOCK", None)

        init.setDatabaseConnection(url, dialect, sock)

        try:
            # Initial clear should work
            myThread = threading.currentThread()
            init.clearDatabase()

            # Clear one after another should work
            init.setSchema(modules = ['WMCore.WMBS'])
            init.clearDatabase()
            init.setSchema(modules = ['WMCore.WMBS'])
            init.clearDatabase()

            # Clear non-existant DB should work
            # Drop the database, and then make sure the database gets recreated
            a = myThread.dbi.engine.url.database
            dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
            myThread.dbi.processData("DROP DATABASE %s" % dbName)
            dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
            self.assertEqual(dbName, None)
            init.clearDatabase()
            dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
            self.assertEqual(dbName, a)


            init.setSchema(modules = ['WMCore.WMBS'])
            myThread.transaction.begin()
            myThread.transaction.processData("SELECT * FROM wmbs_job")
            init.clearDatabase()
            dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
            self.assertEqual(dbName, a)
            myThread.transaction.begin()
            init.setSchema(modules = ['WMCore.WMBS'])
            myThread.transaction.commit()
        except:
            init.clearDatabase()
            raise

        init.clearDatabase()        

        return        

class TestInit(object):
    """
    A set of initialization steps used in many tests.
    Test can call the methods from this class to
    initialize their default environment so to
    minimize code duplication.
    """

    def __init__(self, testClassName = "Unknown Class"):
        self.testClassName = testClassName
        self.testDir = None
        self.currModules = []
        global hasDatabase
        self.hasDatabase = hasDatabase
        if self.hasDatabase:
            self.init = WMInit()
        self.deleteTmp = True

    def __del__(self):
        if self.deleteTmp:
            self.delWorkDir()
        self.attemptToCloseDBConnections()


    def delWorkDir(self):
        if self.testDir != None:
            try:
                shutil.rmtree( self.testDir )
            except:
                # meh, if it fails, I guess something weird happened
                pass

    def setLogging(self, logLevel = logging.INFO):
        """
        Sets logging parameters
        """
        # remove old logging instances.
        return
        logger1 = logging.getLogger()
        logger2 = logging.getLogger(self.testClassName)
        for logger in [logger1, logger2]:
            for handler in logger.handlers:
                handler.close()
                logger.removeHandler(handler)

        self.init.setLogging(self.testClassName, self.testClassName,
                             logExists = False, logLevel = logLevel)

    def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
        self.testDir = tempfile.mkdtemp()
        if config:
            config.section_("General")
            config.General.workDir = self.testDir
        os.environ['TESTDIR'] = self.testDir
        if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
            deleteOnDestruction = True
            logging.info("Generated testDir - %s" % self.testDir)
        if setTmpDir:
            os.environ['TMPDIR'] = self.testDir

        self.deleteTmp = deleteOnDestruction
        return self.testDir

    def getBackendFromDbURL(self, dburl):
        dialectPart = dburl.split(":")[0]
        if dialectPart == 'mysql':
            return 'MySQL'
        elif dialectPart == 'oracle':
            return 'Oracle'
        elif dialectPart == 'http':
            return 'CouchDB'
        else:
            raise RuntimeError("Unrecognized dialect %s" % dialectPart)

    def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
        """
        Set up the database connection by retrieving the environment
        parameters.

        The destroyAllDatabase option is for testing ONLY.  Never flip that switch
        on in any other instance where you don't know what you're doing.
        """
        if not self.hasDatabase:
            return
        config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
        self.coreConfig = config
        self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                        config.CoreDatabase.dialect,
                                        config.CoreDatabase.socket)

        if trashDatabases or destroyAllDatabase:
            self.clearDatabase()

        # Have to check whether or not database is empty
        # If the database is not empty when we go to set the schema, abort!
        try:
            result = self.init.checkDatabaseContents()
        except OperationalError:
            logging.debug("Error checking DB contents, assume DB does not exist")
            return
#.........这里部分代码省略.........

    def __init__(self, **args):
        self.args = {}
        
        self.args.setdefault('Logfile', None)
        self.args.setdefault('CacheDir', None)
        self.args.setdefault('ProxiesDir', None)
        self.args.setdefault('CopyTimeOut', '')

        # SE support parameters
        # Protocol = local cannot be the default. Any default allowd 
        # for this parameter... it must be defined from config file. 
        self.args.setdefault('Protocol', '')
        self.args.setdefault('storageName', 'localhost')
        self.args.setdefault('storagePort', '')
        self.args.setdefault('storagePath', self.args["CacheDir"])

        # specific delegation strategy for glExex
        self.args.setdefault('glExecDelegation', 'false')

        self.args.setdefault('PollInterval',60)
        self.args.setdefault("HeartBeatDelay", "00:05:00")
        self.args.update(args)

        if len(self.args["HeartBeatDelay"]) != 8:
            self.HeartBeatDelay="00:05:00"
        else:
            self.HeartBeatDelay=self.args["HeartBeatDelay"]
        
        # define log file
        if self.args['Logfile'] == None:
            self.args['Logfile'] = os.path.join(self.args['ComponentDir'],
                                                "ComponentLog")
        # create log handler
        logHandler = RotatingFileHandler(self.args['Logfile'],
                                         "a", 1000000, 7)
        # define log format
        logFormatter = logging.Formatter("%(asctime)s:%(message)s")
        logHandler.setFormatter(logFormatter)
        logging.getLogger().addHandler(logHandler)
        logging.getLogger().setLevel(logging.INFO)

        ## volatile properties
        self.wdir = self.args['ComponentDir']
        self.maxThreads = int( self.args.get('maxThreads', 5) )
        self.timePoolDB = self.args['PollInterval']

        # shared sessions 
        self.blDBsession = BossLiteAPI('MySQL', dbConfig, makePool=True)
        self.sessionPool = self.blDBsession.bossLiteDB.getPool()
        self.workerPool = self.blDBsession.bossLiteDB.getPool()

        # Get configuration
        self.init = WMInit()
        self.init.setLogging()
        self.init.setDatabaseConnection(os.getenv("DATABASE"), \
            os.getenv('DIALECT'), os.getenv("DBSOCK"))

        self.myThread = threading.currentThread()
        self.factory = WMFactory("msgService", "WMCore.MsgService."+ \
                             self.myThread.dialect)
        self.newMsgService = self.myThread.factory['msgService']\
                          .loadObject("MsgService")

        self.ms = MessageService()

        self.workerCfg = {}
        logging.info(" ")
        logging.info("Starting component...")

class TestInit:
    """
    A set of initialization steps used in many tests.
    Test can call the methods from this class to 
    initialize their default environment so to 
    minimize code duplication.
    """ 

    def __init__(self, testClassName = "Unknown Class"):
        self.testClassName = testClassName
        self.testDir = None
        self.currModules = []
        global hasDatabase
        self.hasDatabase = hasDatabase
        if self.hasDatabase:
            self.init = WMInit()
        self.deleteTmp = True
    
    def __del__(self):
        if self.deleteTmp:
            self.delWorkDir()
        self.attemptToCloseDBConnections()

    
    def delWorkDir(self):
        if (self.testDir != None):
            try:
                shutil.rmtree( self.testDir )
            except:
                # meh, if it fails, I guess something weird happened
                pass

    def setLogging(self, logLevel = logging.INFO):
        """
        Sets logging parameters
        """
        # remove old logging instances.
        return
        logger1 = logging.getLogger()
        logger2 = logging.getLogger(self.testClassName)
        for logger in [logger1, logger2]:
            for handler in logger.handlers:
                handler.close()
                logger.removeHandler(handler)

        self.init.setLogging(self.testClassName, self.testClassName,
                             logExists = False, logLevel = logLevel)
    
    def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
        self.testDir = tempfile.mkdtemp()
        if config:
            config.section_("General")
            config.General.workDir = self.testDir
        os.environ['TESTDIR'] = self.testDir
        if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
            deleteOnDestruction = True
            logging.info("Generated testDir - %s" % self.testDir)
        if setTmpDir:
            os.environ['TMPDIR'] = self.testDir

        self.deleteTmp = deleteOnDestruction
        return self.testDir
        
    def getBackendFromDbURL(self, dburl):
        dialectPart = dburl.split(":")[0]
        if dialectPart == 'mysql':
            return 'MySQL'
        elif dialectPart == 'sqlite':
            return 'SQLite'
        elif dialectPart == 'oracle':
            return 'Oracle'
        elif dialectPart == 'http':
            return 'CouchDB'
        else:
            raise RuntimeError, "Unrecognized dialect %s" % dialectPart
        
    def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
        """
        Set up the database connection by retrieving the environment
        parameters.

        The destroyAllDatabase option is for testing ONLY.  Never flip that switch
        on in any other instance where you don't know what you're doing.
        """
        if not self.hasDatabase:
            return        
        config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
        self.coreConfig = config
        self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
                                        config.CoreDatabase.dialect,
                                        config.CoreDatabase.socket)

        if trashDatabases or destroyAllDatabase:
            self.clearDatabase()

        # Have to check whether or not database is empty 
        # If the database is not empty when we go to set the schema, abort! 
        result = self.init.checkDatabaseContents() 
        if len(result) > 0: 
            msg =  "Database not empty, cannot set schema !\n" 
#.........这里部分代码省略.........

 def __init__(self, testClassName):
     self.testClassName = testClassName
     self.testDir = None
     self.currModules = []
     self.init = WMInit()
     self.deleteTmp = True

else:
    stri=time.ctime().split()
    stri1=stri[2]
    stri2=stri[3].replace(":","")  
    arguments["ProcessingVersion"] =  '%s_%s'%(stri1,stri2)

wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])

if not hasattr(wmAgentConfig, "CoreDatabase"):
    print "Your config is missing the CoreDatabase section."

socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)

myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
                               socketLoc = socketLoc)

workloadName = "CmsRunAnalysis-%s" % arguments["ProcessingVersion"]
workloadFile = "CmsRunAnalysis-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)

cmsRunAna = AnalysisWorkloadFactory()
workload = cmsRunAna(workloadName, arguments)

taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()

workload.save(os.path.join(workloadName, workloadFile))