Python JSONRequests.encode方法代码示例

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import encode [as 别名]
class DBSBlock:
    """
    DBSBlock

    Class for holding all the necessary equipment for a DBSBlock
    """

    def __init__(self, name, location, das):
        """
        Just the necessary objects

        Expects:
          name:  The blockname in full
          location: The SE-name of the site the block is at
        """
        
        
        self.data      = {'dataset_conf_list':    [],   # List of dataset configurations
                          'file_conf_list':       [],   # List of files, with the configuration for each
                          'files':                [],   # List of file objects
                          'block':                {},   # Dict of block info
                          'block_parent_list':    [],   # List of block parents
                          'processing_era':       {},   # Dict of processing era info
                          'ds_parent_list':       [],   # List of parent datasets
                          'acquisition_era':      {},   # Dict of acquisition era information
                          'primds':               {},   # Dict of primary dataset info
                          'dataset':              {},   # Dict of processed dataset info
                          'physics_group_name':   {},   # Physics Group Name
                          'file_parent_list':     []}   # List of file parents
                          
        self.files     = [] 
        self.encoder   = JSONRequests()
        self.status    = 'Open'
        self.inBuff    = False
        self.startTime = time.time()
        self.name      = name
        self.location  = location
        self.das       = das

        self.data['block']['block_name']       = name
        self.data['block']['origin_site_name'] = location
        self.data['block']['open_for_writing'] = 0  # If we're sending a block, it better be open

        return                      


    def encode(self):
        """
        _encode_

        Turn this into a JSON object for transmission
        to DBS
        """

        return self.encoder.encode(data = self.data)



    def addFile(self, dbsFile):
        """
        _addFile_
        
        Add a DBSBufferFile object to our block
        """

        
        
        if dbsFile['id'] in [x['id'] for x in self.files]:
            msg =  "Duplicate file inserted into DBSBlock: %i\n" % (dbsFile['id'])
            msg += "Ignoring this file for now!\n"
            logging.error(msg)
            logging.debug("Block length: %i" % len(self.files))
            l = [x['id'] for x in self.files]
            l.sort()
            logging.debug("First file: %s    Last file: %s" % (l[0], l[-1]))
            return
        
        self.files.append(dbsFile)
        # Assemble information for the file itself
        fileDict = {}
        fileDict['file_type']              =  'EDM'
        fileDict['logical_file_name']      = dbsFile['lfn']
        fileDict['file_size']              = dbsFile['size']
        fileDict['event_count']            = dbsFile['events']
        # Do the checksums
        for cktype in dbsFile['checksums'].keys():
            cksum = dbsFile['checksums'][cktype]
            if cktype.lower() == 'cksum':
                fileDict['check_sum'] = cksum
            elif cktype.lower() == 'adler32':
                fileDict['adler32'] = cksum
            elif cktype.lower() == 'md5':
                fileDict['md5'] = cksum

        # Do the runs
        lumiList = []
        for run in dbsFile.getRuns():
            for lumi in run.lumis:
                lumiList.append({'lumi_section_num': lumi, 'run_num': run.run})
        fileDict['file_lumi_list'] = lumiList
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import encode [as 别名]

#.........这里部分代码省略.........
            count += 1


        return

    def _subProcessName(self, slaveClassName, sequence):
        """ subProcessName for heartbeat
            could change to use process ID as a suffix
        """
        return "%s_%s" % (slaveClassName, sequence + 1)

    def __del__(self):
        """
        __del__

        Kill all the workers processes by sending them an invalid JSON object.
        This will cause them to shut down.
        """
        self.close()
        return

    def close(self):
        """
        _close_

        Close shuts down all the active systems by:

        a) Sending STOP commands for all workers
        b) Closing the pipes
        c) Shutting down the workers themselves
        """
        for i in range(self.nSlaves):
            try:
                encodedWork = self.jsonHandler.encode('STOP')
                self.sender.send(encodedWork)
            except Exception as ex:
                # Might be already failed.  Nothing you can
                # really do about that.
                logging.error("Failure killing running process: %s" % str(ex))
                pass

        try:
            self.sender.close()
        except:
            # We can't really do anything if we fail
            pass
        try:
            self.sink.close()
        except:
            # We can't do anything if we fail
            pass

        # Now close the workers by hand
        for worker in self.workers:
            try:
                worker.join()
            except Exception as ex:
                try:
                    worker.terminate()
                except Exception as ex2:
                    logging.error("Failure to join or terminate process")
                    logging.error(str(ex))
                    logging.error(str(ex2))
                    continue
        self.workers = []
        return

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import encode [as 别名]
class DBSBufferBlock:
    """
    _DBSBufferBlock_

    """

    def __init__(self, name, location, datasetpath):
        """
        Just the necessary objects

        Expects:
          name:  The blockname in full
          location: The PNN of the site the block is at
        """


        self.data      = {'dataset_conf_list':    [],   # List of dataset configurations
                          'file_conf_list':       [],   # List of files, the configuration for each
                          'files':                [],   # List of file objects
                          'block':                {},   # Dict of block info
                          'processing_era':       {},   # Dict of processing era info
                          'acquisition_era':      {},   # Dict of acquisition era information
                          'primds':               {},   # Dict of primary dataset info
                          'dataset':              {},   # Dict of processed dataset info
                          'file_parent_list':     [],   # List of file parents
                          'close_settings':       {}}   # Dict of info about block close settings

        self.files        = []
        self.encoder      = JSONRequests()
        self.status       = 'Open'
        self.inBuff       = False
        self.startTime    = time.time()
        self.name         = name
        self.location     = location
        self.datasetpath  = datasetpath
        self.workflows    = set()

        self.data['block']['block_name']       = name
        self.data['block']['origin_site_name'] = location
        self.data['block']['open_for_writing'] = 1

        self.data['block']['create_by'] = "WMAgent"
        self.data['block']['creation_date'] = int(time.time())
        self.data['block']['block_size'] = 0
        self.data['block']['file_count'] = 0
        self.data['block']['block_events'] = 0

        self.data['close_settings'] = {}
        self.data['close_settings']['block_close_max_wait_time'] = None
        self.data['close_settings']['block_close_max_events'] = None
        self.data['close_settings']['block_close_max_size'] = None
        self.data['close_settings']['block_close_max_files'] = None
        return


    def encode(self):
        """
        _encode_

        Turn this into a JSON object for transmission
        to DBS
        """

        return self.encoder.encode(data = self.data)



    def addFile(self, dbsFile, datasetType, primaryDatasetType):
        """
        _addFile_

        Add a DBSBufferFile object to our block
        """
        if dbsFile['id'] in [x['id'] for x in self.files]:
            msg =  "Duplicate file inserted into DBSBufferBlock: %i\n" % (dbsFile['id'])
            msg += "Ignoring this file for now!\n"
            logging.error(msg)
            logging.debug("Block length: %i" % len(self.files))
            l = sorted([x['id'] for x in self.files])
            logging.debug("First file: %s    Last file: %s" % (l[0], l[-1]))
            return

        for setting in self.data['close_settings']:
            if self.data['close_settings'][setting] is None:
                self.data['close_settings'][setting] = dbsFile[setting]

        self.workflows.add(dbsFile['workflow'])

        self.files.append(dbsFile)
        self.data['block']['block_size'] += int(dbsFile['size'])
        self.data['block']['file_count'] += 1
        self.data['block']['block_events'] += int(dbsFile['events'])

        # Assemble information for the file itself
        fileDict = {}
        fileDict['file_type']              =  'EDM'
        fileDict['logical_file_name']      = dbsFile['lfn']
        fileDict['file_size']              = dbsFile['size']
        fileDict['event_count']            = dbsFile['events']
        fileDict['last_modified_by'] = "WMAgent"
#.........这里部分代码省略.........

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import encode [as 别名]
        try:
            logging.error(input)
            output = slaveClass(input)
        except Exception as ex:
            crashMessage = "Slave process crashed with exception: " + str(ex)
            crashMessage += "\nStacktrace:\n"

            stackTrace = traceback.format_tb(sys.exc_info()[2], None)
            for stackFrame in stackTrace:
                crashMessage += stackFrame

            logging.error(crashMessage)
            try:
                output        = {'type': 'ERROR', 'msg': crashMessage}
                encodedOutput = jsonHandler.encode(output)
                sender.send(encodedOutput)
                logging.error("Sent error message and now breaking")
                break
            except Exception as ex:
                logging.error("Failed to send error message")
                logging.error(str(ex))
                del jsonHandler
                sys.exit(1)

        if output != None:
            if type(output) == list:
                for item in output:
                    encodedOutput = jsonHandler.encode(item)
                    sender.send(encodedOutput)
            else:

# 需要导入模块: from WMCore.Services.Requests import JSONRequests [as 别名]
# 或者: from WMCore.Services.Requests.JSONRequests import encode [as 别名]
class ProcessPool:
    def __init__(self, slaveClassName, totalSlaves, componentDir,
                 config, slaveInit = None, namespace = None):
        """
        __init__

        Constructor for the process pool.  The slave class name must be based
        inside the WMComponent namespace.  For examples, the JobAccountant would
        pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker
        class.  All log files will be stored in the component directory that is
        passed in.  Each slave will have its own log file.

        Note that the config is only used to determine database connection
        parameters.  It is not passed to the slave class.  The slaveInit
        parameter will be serialized and passed to the slave class's
        constructor.
        """
        self.enqueueIndex = 0
        self.dequeueIndex = 0
        self.runningWork  = 0

        #Use the Services.Requests JSONizer, which handles __to_json__ calls
        self.jsonHandler = JSONRequests()
        
        # heartbeat should be registered at this point
        if getattr(config.Agent, "useHeartbeat", True):
            self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave"))
            
        self.slaveClassName = slaveClassName
        self.componentDir   = componentDir
        self.config         = config
        # Grab the python version from the current version
        # Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X
        majorVersion = sys.version_info[0]
        minorVersion = sys.version_info[1]

        if majorVersion and minorVersion:
            self.versionString = "python%i.%i" % (majorVersion, minorVersion)
        else:
            self.versionString = "python2.4"

        self.workers = []
        self.nSlaves = totalSlaves
        self.slaveInit = slaveInit
        self.namespace = namespace


        # Now actually create the slaves
        self.createSlaves()


        return


    def createSlaves(self):
        """
        _createSlaves_

        Create the slaves by using the values from __init__()
        Moving it into a separate function allows us to restart
        all of them.
        """

        totalSlaves    = self.nSlaves
        slaveClassName = self.slaveClassName
        config         = self.config
        slaveInit      = self.slaveInit
        namespace      = self.namespace
        
        slaveArgs = [self.versionString, __file__, self.slaveClassName]
        if hasattr(config.CoreDatabase, "socket"):
            socket = config.CoreDatabase.socket
        else:
            socket = None

        (connectDialect, junk) = config.CoreDatabase.connectUrl.split(":", 1)
        if connectDialect.lower() == "mysql":
            dialect = "MySQL"
        elif connectDialect.lower() == "oracle":
            dialect = "Oracle"
        elif connectDialect.lower() == "sqlite":
            dialect = "SQLite"

        dbConfig = {"dialect": dialect,
                    "connectUrl": config.CoreDatabase.connectUrl,
                    "socket": socket,
                    "componentDir": self.componentDir}
        if namespace:
            # Then add a namespace to the config
            dbConfig['namespace'] = namespace
        encodedDBConfig = self.jsonHandler.encode(dbConfig)

        if slaveInit == None:
            encodedSlaveInit = None
        else:
            encodedSlaveInit = self.jsonHandler.encode(slaveInit)
        
        count = 0     
        while totalSlaves > 0:
            #For each worker you want create a slave process
#.........这里部分代码省略.........