Python ThreadPool.queueTask方法代码示例

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]
class TaskManager:
    log = logging.getLogger('TaskManager')

    def __init__(self, tpSize=config.getfloat('irs', 'thread_pool_size'), waitTimeout=3, maxTasks=config.getfloat('irs', 'max_tasks')):
        self.storage_repository = config.get('irs', 'repository')
        self.tp = ThreadPool(tpSize, waitTimeout, maxTasks)
        self._tasks = {}
        self._unqueuedTasks = []


    def queue(self, task):
        return self._queueTask(task, task.commit)

    def queueRecovery(self, task):
        return self._queueTask(task, task.recover)

    def _queueTask(self, task, method):
        try:
            self.log.debug("queueing task: %s", task.id)
            self._tasks[task.id] = task
            if not self.tp.queueTask(task.id, method):
                self.log.error("unable to queue task: %s", task.dumpTask())
                del self._tasks[task.id]
                raise se.AddTaskError()
            self.log.debug("task queued: %s", task.id)
        except Exception, ex:
            self.log.error("Could not queue task, encountered: %s", str(ex))
            raise
        return task.id

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]

#.........这里部分代码省略.........
        diff = newMaxId - self._numHosts
        if diff > 0:
            delta = MAILBOX_SIZE * diff * "\0"
            self._outgoingMail += delta
            self._incomingMail += delta
        elif diff < 0:
            delta = MAILBOX_SIZE * diff
            self._outgoingMail = self._outgoingMail[:-delta]
            self._incomingMail = self._incomingMail[:-delta]
        self._numHosts=newMaxId
        self._outMailLen = MAILBOX_SIZE * self._numHosts
        self._outLock.release()
        self._inLock.release()


    def _validateMailbox(self, mailbox, mailboxIndex):
        chkStart = MAILBOX_SIZE-CHECKSUM_BYTES
        chk = misc.checksum(mailbox[0 : chkStart], CHECKSUM_BYTES)
        pChk = struct.pack('<l',chk) # Assumes CHECKSUM_BYTES equals 4!!!
        if pChk != mailbox[chkStart : chkStart+CHECKSUM_BYTES]:
            self.log.error("SPM_MailMonitor: mailbox %s checksum failed, not clearing mailbox, clearing newMail.", str(mailboxIndex))
            return False
        elif pChk == pZeroChecksum: return False  # Ignore messages of empty mailbox
        return True


    def _handleRequests(self, newMail):

        send = False

        # run through all messages and check if new messages have arrived (since last read)
        for host in range(0, self._numHosts):
            # Check mailbox checksum
            mailboxStart = host * MAILBOX_SIZE

            isMailboxValidated = False

            for i in range(0, MESSAGES_PER_MAILBOX):

                msgId = host * SLOTS_PER_MAILBOX + i
                msgStart = msgId * MESSAGE_SIZE

                # First byte of message is message version.  Check message version, if 0 then message is empty and can be skipped
                if newMail[msgStart] in ['\0', '0']: continue

                # Most mailboxes are probably empty so it costs less to check that all messages start with 0 than
                # to validate the mailbox, therefor this is done after we find a non empty message in mailbox
                if not isMailboxValidated:
                    if not self._validateMailbox(newMail[mailboxStart : mailboxStart + MAILBOX_SIZE], host):
                        #Cleaning invalid mbx in newMail
                        newMail = newMail[:mailboxStart] + EMPTYMAILBOX +  newMail[mailboxStart + MAILBOX_SIZE:]
                        break
                    self.log.debug("SPM_MailMonitor: Mailbox %s validated, checking mail", host)
                    isMailboxValidated = True


                newMsg = newMail[msgStart : msgStart+MESSAGE_SIZE]
                msgOffset = msgId * MESSAGE_SIZE
                if newMsg == CLEAN_MESSAGE:
                    # Should probably put a setter on outgoingMail which would take the lock
                    self._outLock.acquire()
                    try:
                        self._outgoingMail = self._outgoingMail[0:msgOffset] + CLEAN_MESSAGE + self._outgoingMail[msgOffset+MESSAGE_SIZE : self._outMailLen]
                    finally:
                        self._outLock.release()
                    send = True
                    continue

                # Message isn't empty, check if its new
                isMessageNew = False
                for j in range(msgStart, msgStart + MESSAGE_SIZE):
                    if newMail[j] != self._incomingMail[j]:
                        isMessageNew = True
                        break

                # If search exhausted, i.e. message hasn't changed since last read, it can be skipped
                if not isMessageNew: continue

                # We only get here if there is a novel request
                try:
                    msgType = newMail[msgStart+1 : msgStart+5]
                    if msgType in self._messageTypes:
                        # Use message class to process request according to message specific logic
                        id = str(uuid.uuid4())
                        self.log.debug("SPM_MailMonitor: processing request: %s" % repr(newMail[msgStart : msgStart+MESSAGE_SIZE]))
                        res = self.tp.queueTask(id, runTask,
                                (self._messageTypes[msgType], msgId,
                                newMail[msgStart : msgStart+MESSAGE_SIZE])
                        )
                        if not res:
                            raise Exception()
                    else:
                        self.log.error("SPM_MailMonitor: unknown message type encountered: %s", msgType)
                except RuntimeError, e:
                    self.log.error("SPM_MailMonitor: exception: %s caught while handling message: %s",
                                    str(e), newMail[msgStart:msgStart + MESSAGE_SIZE])
                except:
                    self.log.error("SPM_MailMonitor: exception caught while handling message: %s",
                            newMail[msgStart:msgStart + MESSAGE_SIZE],
                            exc_info=True)

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]

#.........这里部分代码省略.........
        self._hostID = int(hostID)
        self._used_slots_array = [ 0 ] * MESSAGES_PER_MAILBOX
        self._outgoingMail = EMPTYMAILBOX
        self._incomingMail = EMPTYMAILBOX
        # TODO: add support for multiple paths (multiple mailboxes)
        self._spmStorageDir = config.get('irs', 'repository')
        self._inCmd = [ constants.EXT_DD,
                        'if=' + str(inbox),
                        'iflag=direct,fullblock',
                        'bs=' + str(BLOCK_SIZE),
                        'count=' + str(BLOCKS_PER_MAILBOX),
                        'skip=' + str(self._hostID*BLOCKS_PER_MAILBOX)
                        ]
        self._outCmd = [constants.EXT_DD,
                        'of=' + str(outbox),
                        'iflag=fullblock',
                        'oflag=direct',
                        'conv=notrunc',
                        'bs=' + str(BLOCK_SIZE),
                        'seek=' + str(self._hostID*BLOCKS_PER_MAILBOX)
                        ]
        self._init = False
        self._initMailbox() # Read initial mailbox state
        self._msgCounter = 0
        self._sendMail() # Clear outgoing mailbox
        threading.Thread.__init__(self)
        self.start()

    def _initMailbox(self):
        # Sync initial incoming mail state with storage view
        (rc, out, err) = misc.execCmd(self._inCmd, sudo=False, raw=True)
        if rc == 0:
            self._incomingMail = out
            self._init = True
        else:
            self.log.warning("HSM_MailboxMonitor - Could not initialize mailbox, will not accept requests until init succeeds")


    def immStop(self):
        self._stop = True


    def immFlush(self):
        self._flush = True


    def _handleResponses(self, newMsgs):
        rc = False

        for i in range(0, MESSAGES_PER_MAILBOX):
            # Skip checking non used slots
            if self._used_slots_array[i] == 0: continue

            # Skip empty return messages (messages with version 0)
            start = i*MESSAGE_SIZE

            # First byte of message is message version.
            # Check return message version, if 0 then message is empty
            if newMsgs[start] in ['\0', '0']: continue

            for j in range(start, start + MESSAGE_SIZE):
                if newMsgs[j] != self._incomingMail[j]: break

            # If search exhausted then message hasn't changed since last read and can be skipped
            if j == (start + MESSAGE_SIZE - 1): continue

            #
            # We only get here if there is a novel reply so we can remove the message from the active list
            #  and the outgoing mail and handle the reply
            #
            rc = True

            newMsg = newMsgs[start : start + MESSAGE_SIZE]

            if newMsg == CLEAN_MESSAGE:
                del self._activeMessages[i]
                self._used_slots_array[i] = 0
                self._msgCounter -= 1
                self._outgoingMail = self._outgoingMail[0 : start] + MESSAGE_SIZE * "\0" + self._outgoingMail[start + MESSAGE_SIZE : ]
                continue

            msg = self._activeMessages[i]
            self._activeMessages[i] = CLEAN_MESSAGE
            self._outgoingMail = self._outgoingMail[0 : start] + CLEAN_MESSAGE + self._outgoingMail[start + MESSAGE_SIZE : ]

            try:
                self.log.debug("HSM_MailboxMonitor(%s/%s) - Checking reply: %s", self._msgCounter, MESSAGES_PER_MAILBOX, repr(newMsg))
                msg.checkReply(newMsg)
                if msg.callback:
                    try:
                        id = str(uuid.uuid4())
                        if not self.tp.queueTask(id, runTask, (msg.callback, msg.volumeData)):
                            raise Exception()
                    except:
                        self.log.error("HSM_MailMonitor: exception caught while running msg callback, for message: %s, callback function: %s",
                                repr(msg.payload), msg.callback, exc_info=True)
            except RuntimeError, e:
                self.log.error("HSM_MailMonitor: exception: %s caught while checking reply for message: %s, reply: %s", \
                                str(e), repr(msg.payload), repr(newMsg))
            except:

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]

#.........这里部分代码省略.........
                    self._outLock.acquire()
                    try:
                        self._outgoingMail = \
                            self._outgoingMail[0:msgOffset] + CLEAN_MESSAGE + \
                            self._outgoingMail[msgOffset + MESSAGE_SIZE:
                                               self._outMailLen]
                    finally:
                        self._outLock.release()
                    send = True
                    continue

                # Message isn't empty, check if its new
                isMessageNew = False
                for j in range(msgStart, msgStart + MESSAGE_SIZE):
                    if newMail[j] != self._incomingMail[j]:
                        isMessageNew = True
                        break

                # If search exhausted, i.e. message hasn't changed since last
                # read, it can be skipped
                if not isMessageNew:
                    continue

                # We only get here if there is a novel request
                try:
                    msgType = newMail[msgStart + 1:msgStart + 5]
                    if msgType in self._messageTypes:
                        # Use message class to process request according to
                        # message specific logic
                        id = str(uuid.uuid4())
                        self.log.debug("SPM_MailMonitor: processing request: "
                                       "%s" % repr(newMail[
                                           msgStart:msgStart + MESSAGE_SIZE]))
                        res = self.tp.queueTask(
                            id, runTask, (self._messageTypes[msgType], msgId,
                                          newMail[msgStart:
                                                  msgStart + MESSAGE_SIZE])
                        )
                        if not res:
                            raise Exception()
                    else:
                        self.log.error("SPM_MailMonitor: unknown message type "
                                       "encountered: %s", msgType)
                except RuntimeError as e:
                    self.log.error("SPM_MailMonitor: exception: %s caught "
                                   "while handling message: %s", str(e),
                                   newMail[msgStart:msgStart + MESSAGE_SIZE])
                except:
                    self.log.error("SPM_MailMonitor: exception caught while "
                                   "handling message: %s",
                                   newMail[msgStart:msgStart + MESSAGE_SIZE],
                                   exc_info=True)

        self._incomingMail = newMail
        return send

    def _checkForMail(self):
        # Lock is acquired in order to make sure that neither _numHosts nor
        # incomingMail are changed during checkForMail
        self._inLock.acquire()
        try:
            # self.log.debug("SPM_MailMonitor -_checking for mail")
            cmd = self._inCmd + ['bs=' + str(self._outMailLen)]
            # self.log.debug("SPM_MailMonitor - reading incoming mail, "
            #               "command: " + str(cmd))
            (rc, in_mail, err) = misc.execCmd(cmd, raw=True)

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]

#.........这里部分代码省略.........
                continue

            #
            # We only get here if there is a novel reply so we can remove the
            # message from the active list and the outgoing mail and handle the
            # reply
            #
            rc = True

            newMsg = newMsgs[start:start + MESSAGE_SIZE]

            if newMsg == CLEAN_MESSAGE:
                del self._activeMessages[i]
                self._used_slots_array[i] = 0
                self._msgCounter -= 1
                self._outgoingMail = self._outgoingMail[0:start] + \
                    MESSAGE_SIZE * "\0" + self._outgoingMail[start +
                                                             MESSAGE_SIZE:]
                continue

            msg = self._activeMessages[i]
            self._activeMessages[i] = CLEAN_MESSAGE
            self._outgoingMail = self._outgoingMail[0:start] + \
                CLEAN_MESSAGE + self._outgoingMail[start + MESSAGE_SIZE:]

            try:
                self.log.debug("HSM_MailboxMonitor(%s/%s) - Checking reply: "
                               "%s", self._msgCounter, MESSAGES_PER_MAILBOX,
                               repr(newMsg))
                msg.checkReply(newMsg)
                if msg.callback:
                    try:
                        id = str(uuid.uuid4())
                        if not self.tp.queueTask(id, runTask, (msg.callback,
                                                 msg.volumeData)):
                            raise Exception()
                    except:
                        self.log.error("HSM_MailMonitor: exception caught "
                                       "while running msg callback, for "
                                       "message: %s, callback function: %s",
                                       repr(msg.payload), msg.callback,
                                       exc_info=True)
            except RuntimeError as e:
                self.log.error("HSM_MailMonitor: exception: %s caught while "
                               "checking reply for message: %s, reply: %s",
                               str(e), repr(msg.payload), repr(newMsg))
            except:
                self.log.error("HSM_MailMonitor: exception caught while "
                               "checking reply from SPM, request was: %s "
                               "reply: %s", repr(msg.payload), repr(newMsg),
                               exc_info=True)
        # Finished processing incoming mail, now save mail to compare against
        # next batch
        self._incomingMail = newMsgs
        return rc

    def _checkForMail(self):
        # self.log.debug("HSM_MailMonitor - checking for mail")
        # self.log.debug("Running command: " + str(self._inCmd))
        (rc, in_mail, err) = misc.execCmd(self._inCmd, raw=True)
        if rc:
            raise RuntimeError("_handleResponses.Could not read mailbox - rc "
                               "%s" % rc)
        if (len(in_mail) != MAILBOX_SIZE):
            raise RuntimeError("_handleResponses.Could not read mailbox - len "
                               "%s != %s" % (len(in_mail), MAILBOX_SIZE))

# 需要导入模块: from threadPool import ThreadPool [as 别名]
# 或者: from threadPool.ThreadPool import queueTask [as 别名]
class TaskManager:
    log = logging.getLogger('TaskManager')

    def __init__(self,
                 tpSize=config.getint('irs', 'thread_pool_size'),
                 waitTimeout=3,
                 maxTasks=config.getint('irs', 'max_tasks')):
        self.storage_repository = config.get('irs', 'repository')
        self.tp = ThreadPool(tpSize, waitTimeout, maxTasks)
        self._tasks = {}
        self._unqueuedTasks = []

    def queue(self, task):
        return self._queueTask(task, task.commit)

    def queueRecovery(self, task):
        return self._queueTask(task, task.recover)

    def _queueTask(self, task, method):
        try:
            self.log.debug("queuing task: %s", task.id)
            self._tasks[task.id] = task
            if not self.tp.queueTask(task.id, method):
                self.log.error("unable to queue task: %s", task.dumpTask())
                del self._tasks[task.id]
                raise se.AddTaskError()
            self.log.debug("task queued: %s", task.id)
        except Exception as ex:
            self.log.error("Could not queue task, encountered: %s", str(ex))
            raise
        return task.id

    def scheduleJob(self, type, store, task, jobName, func, *args):
        task.setTag(type)
        task.setPersistence(store, cleanPolicy=TaskCleanType.manual)
        task.setManager(self)
        task.setRecoveryPolicy("auto")
        task.addJob(Job(jobName, func, *args))
        self.log.debug("scheduled job %s for task %s ", jobName, task.id)

    def __getTask(self, taskID):
        Task.validateID(taskID)
        t = self._tasks.get(taskID, None)
        if t is None:
            raise se.UnknownTask(taskID)
        return t

    def prepareForShutdown(self):
        """ Prepare to shutdown. Stop all threads and asynchronous tasks
        """
        self.log.debug("Request to stop all tasks")

        # Clear the task queue and terminate all pooled threads
        for t in self._tasks:
            if hasattr(t, 'stop'):
                t.stop()
            self.log.info(str(t))

        self.tp.joinAll(waitForTasks=False, waitForThreads=False)

    def getTaskStatus(self, taskID):
        """ Internal return Task status for a given task.
        """
        self.log.debug("Entry. taskID: %s", taskID)
        t = self.__getTask(taskID)
        status = t.deprecated_getStatus()
        self.log.debug("Return. Response: %s", status)
        return status

    def getAllTasksStatuses(self, tag=None):
        """ Return Task status for all tasks by type.
        """
        self.log.debug("Entry.")
        subRes = {}
        for taskID, task in self._tasks.items():
            if not tag or tag in task.getTags():
                try:
                    subRes[taskID] = self.getTaskStatus(taskID)
                except se.UnknownTask:
                    # Return statuses for existing tasks only.
                    self.log.warn("Unknown task %s. "
                                  "Maybe task was already cleared.",
                                  taskID)
        self.log.debug("Return: %s", subRes)
        return subRes

    def getAllTasks(self):
        """
        Return Tasks for all public tasks.
        """
        self.log.debug("Entry.")
        subRes = {}
        for taskID, task in self._tasks.items():
            try:
                subRes[taskID] = task.getDetails()
            except se.UnknownTask:
                # Return info for existing tasks only.
                self.log.warn("Unknown task %s. Maybe task was already "
                              "cleared.", taskID)
        self.log.debug("Return: %s", subRes)
#.........这里部分代码省略.........