Python worker.Worker类代码示例

class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.worker = None
        self.statusBar().showMessage('ready')
        self.resize(250, 150)
        self.move(300, 300)
        self.setWindowTitle('刷起来')
        self.setWindowIcon(QIcon('icon.ico'))
        self.imagesPath = "./images/tp14/"
        self.toolBar = self.addToolBar('')

        GameStatus().window = self

        yaoguaifaxian_action = QAction(QIcon('./images/ui/yaoguaifaxian.jpg'), '妖怪发现', self)
        yaoguaifaxian_action.triggered.connect(self.yaoguaifaxian)

        exit_action = QAction(QIcon('./images/ui/exit.png'), '停止', self)
        exit_action.setShortcut('Ctrl+Q')
        exit_action.triggered.connect(self.stop_loop)


        self.toolBar.addAction(yaoguaifaxian_action)
        self.toolBar.addAction(exit_action)

        txt = QTextBrowser()
        txt.setContentsMargins(5, 5, 5, 5)
        self.setCentralWidget(txt)
        self.show()

    def add_text(self, text):
        self.centralWidget().append(text)
        sb = self.centralWidget().verticalScrollBar()
        sb.setValue(sb.maximum())
        print(text)

    def closeEvent(self, *args, **kwargs):
        self.stop_loop()
        print("关闭程序")

    def status_changed(self):
        if self.worker is None:
            self.statusBar().showMessage("就绪。")
        elif self.worker.stopped():
            self.statusBar().showMessage("已停止。")
        else:
            self.statusBar().showMessage("当前次数： " + str(self.worker.cnt))

    def yaoguaifaxian(self):
        if self.worker is not None and not self.worker.stopped():
            return

        self.worker = Worker(self.imagesPath)
        GameStatus().game_stage = GameStage.Yaoguaifaxian
        self.worker.start()

    def stop_loop(self):
        if self.worker is None:
            return
        self.worker.stop()

 def test_thread_safe(self):
     """
     These tests are related to:
     http://stackoverflow.com/q/3752618
     
     I'm not even sure if these tests are correct.
     """
     from worker import Worker
     
     with self.subTest("one-time listener"):
         a = Worker().start()
         @a.listen("test")
         def handler(event):
             a.unlisten(handler)
         a.fire("test")
         a.stop().join()
         self.assertNotIn(handler, a.listener_pool)
         self.assertEqual(a.listeners.get("test", []), [])
         
     with self.subTest("add listener in listener callback"):
         a = Worker().start()
         @a.listen("test")
         def _(event):
             @a.listen("test")
             def _(event):
                 pass
         a.fire("test")
         a.stop().join()
         self.assertEqual(len(a.listeners.get("test", [])), 2)

    def test_failed_job_machine_removal(self):
        from consuela import Consuela
        from job import Job
        from worker import Worker

        with mock.patch('consuela.terminate_worker') as worker_mock:

            cleaner = Consuela()
            cleaner.job_pub_sub = mock.MagicMock()
            cleaner.job_pub_sub.listen.return_value = [{'data': 'test'}]
            worker = Worker(None, None)
            worker.instance = 'some'
            cleaner.get_worker = mock.MagicMock()
            cleaner.get_worker.return_value = 'id', worker
            cleaner.client = mock.MagicMock()
            cleaner.client.exists.return_value = True
            cleaner.client.get.return_value = pickle.dumps(Job('failed', 'something'))
            cleaner.settings = mock.MagicMock()
            cleaner.settings.recycle_workers = True
            cleaner.recycle_worker = mock.MagicMock()
            cleaner.recycle_worker.return_value = False

            cleaner.run()

            assert cleaner.client.exists.call_count == 1
            assert worker_mock.call_count == 0

    def fork_all(self):
        """Create a fork for each worker. The number of workers per tube is
        specified in the tubes list passed to the constructor.
        """
        error_actions = ErrorActions(self.config['error_codes'])
        pids = []

        self.info('Parent process started with pid {}'.format(os.getpid()))

        for tube_config in self.config['tubes']:

            try: worker_count = tube_config['workers']
            except KeyError: worker_count = 1

            for i in range(worker_count):

                # fork the current process. The parent and the child both continue
                # from this point so we need to make sure that only the child
                # process adds workers to the pool.
                pid = os.fork()
                
                if pid == 0:
                    # child process
                    self.info('Child process started with pid {} on tube "{}"'.format(os.getpid(), tube_config['name']))
                    worker = Worker(os.getpid(), tube_config, self.config['connection'], error_actions)
                    worker.watch()

                    sys.exit()
                else:
                    pids.append(pid)

        for pid in pids:
            os.waitpid(pid, 0)
            self.info("Worker {} has exited.".format(pid))

    def test_request_to_booted(self):
        self.aws_mock.my_booted_machine = mock.MagicMock()
        self.aws_mock.my_booted_machine.return_value = 'instance', 'ip'

        from machine_midwife import MachineMidwife
        Apprentice = MachineMidwife.Apprentice
        from job import Job
        from worker import Worker

        apprentice = Apprentice()
        apprentice.settings = mock.MagicMock()
        apprentice.settings.aws_req_max_wait = 10
        apprentice.client = mock.MagicMock()
        apprentice.client.exists.return_value = True
        job = Job('requested', 'batch-')
        worker = Worker(None, None)
        worker.reservation = 'some'
        worker.request_time = datetime.now() - timedelta(minutes=5)

        apprentice.client.keys.return_value = ['jm-']
        apprentice.client.get.side_effect = [pickle.dumps(worker), pickle.dumps(job)]
        apprentice.client.set = mock.MagicMock()
        apprentice.client.publish = mock.MagicMock()

        apprentice.check_newborn()

        assert apprentice.client.keys.call_count == 1
        assert apprentice.client.get.call_count == 2
        assert apprentice.client.set.call_count == 2
        assert apprentice.client.publish.call_count == 1
        assert pickle.loads(apprentice.client.set.call_args_list[0][0][1]).instance == 'instance'
        assert pickle.loads(apprentice.client.set.call_args_list[1][0][1]).state == 'booted'

    def test_normal_machine_recycle(self):
        from consuela import Consuela
        from job import Job
        from worker import Worker

        cleaner = Consuela()
        cleaner.job_pub_sub = mock.MagicMock()
        cleaner.job_pub_sub.listen.return_value = [{'data': 'test'}]
        worker = Worker(None, None)
        worker.instance = 'some'
        cleaner.get_worker = mock.MagicMock()
        cleaner.get_worker.return_value = 'id', worker
        cleaner.client = mock.MagicMock()
        cleaner.client.exists.return_value = True
        cleaner.client.get.return_value = pickle.dumps(Job('finished', 'something'))
        cleaner.settings = mock.MagicMock()
        cleaner.settings.recycle_workers = True
        cleaner.recycle_worker = mock.MagicMock()
        cleaner.recycle_worker.return_value = True

        cleaner.run()

        assert cleaner.client.exists.call_count == 1
        assert cleaner.client.get.call_count == 1
        assert pickle.loads(cleaner.client.set.call_args_list[0][0][1]).job_id is None

    def test_delayed_machine_state_flow_for_requested_with_recycle(self):
        from machine_midwife import MachineMidwife
        from job import Job
        from worker import Worker

        midwife = MachineMidwife()
        midwife.apprentice = mock.MagicMock()
        midwife.settings = mock.MagicMock()
        midwife.client = mock.MagicMock()
        midwife.job_pub_sub = mock.MagicMock()
        midwife.job_pub_sub.listen.return_value = [{'data': 'test'}]
        midwife.client.exists.return_value = True
        job = Job('delayed', 'batch-')
        worker = Worker(None, 'batch-')
        worker.reservation = 'reservation'
        worker.request_time = datetime.now()
        midwife.client.keys.return_value = ['jm-']
        midwife.client.get.side_effect = [pickle.dumps(job), pickle.dumps(worker)]
        midwife.client.set = mock.MagicMock()
        midwife.client.publish = mock.MagicMock()

        midwife.run()

        assert midwife.client.exists.call_count == 2
        assert len(midwife.client.set.call_args_list) == 2
        assert pickle.loads(midwife.client.set.call_args_list[0][0][1]).job_id == 'test'
        assert pickle.loads(midwife.client.set.call_args_list[1][0][1]).state == 'booted'

    def test_no_wake_up_call_for_delayed(self):
        from machine_midwife import MachineMidwife
        Apprentice = MachineMidwife.Apprentice
        from job import Job
        from worker import Worker

        apprentice = Apprentice()
        apprentice.settings = mock.MagicMock()
        apprentice.settings.max_instances = 1
        apprentice.client = mock.MagicMock()
        apprentice.client.exists.return_value = True
        job = Job('delayed', 'batch-')
        apprentice.client.keys.side_effect = [['jm-1', 'jm-2'], ['job-']]
        w1 = Worker(None, None)
        w1.instance = 'a'
        w2 = Worker(None, None)
        w2.instance = 'b'
        apprentice.client.get.side_effect = [pickle.dumps(w1), pickle.dumps(w2), pickle.dumps(job)]
        apprentice.client.publish = mock.MagicMock()

        apprentice.rise_and_shine()

        assert apprentice.client.keys.call_count == 2
        assert apprentice.client.get.call_count == 3
        assert apprentice.client.publish.call_count == 0

class Controller(Parent):

    def models(self):
        self.workers = Worker(self.config)

    # BELOW THIS LINE ARE ALL CONTROLLER ACTIONS

    def new(self, body, resp):
        worker = self.workers.new()

        machine = ProcHandler(self.config, Writer(self.config))
        machine.start()

        worker.CONTROLQUEUE = machine.stopqueue
        worker.OUTPUTQUEUE = machine.subproc.queue

        self.workers.save(worker)

        resp.respond(worker.OUTPUTQUEUE)

    def delete(self, body, resp):
        worker = self.workers.find(body.data)
        pub = SimplePublisher(
            ''. self.config['Rabbit']['host'],
            self.config['Rabbit']['username'],
            self.config['Rabbit']['passwords'])
        pub.publish_msg(self, 'STOP', work.CONTROLQUEUE)

        resp.respond('DELETED')

def do_request( conn ):
    ## Envoi d'un message d'invite
    logger.info("demarrage d'une requete")
    conn.send("Ok send your job")
    run = False
    logger.info("Attente du job")
    try:
        j = conn.recv()
        run = True
    except:
        logger.info("Probleme reception job")

    ## ya pas eu d'erreur 
    if run:
        logger.info("Reception du job %s " % j)
        if isinstance(j, Job):
            conn.send('Job receive')
            logger.info("Creation Worker")
            w = Worker(j)
            logger.info("Lancement Worker")
            w.work()
            logger.info("Worker fini")
            conn.send('Job finish sending result ')
            conn.send(j)
            conn.send('result transmit')
            conn.send('OK see you soon')
            conn.close()
    else:
        logger.info("Erreur not a job %s " % j)
        conn.send('Error not a job')
        conn.close()

    def startWorker(self, featureCollection, attributes, pointProvider=None, hexagonProvider=None):
        # create a new worker instance
        worker = Worker(featureCollection, attributes, pointProvider, hexagonProvider)
        

        # configure the QgsMessageBar
        messageBar = self.iface.messageBar().createMessage('Reading IMAER data...', )
        progressBar = QtGui.QProgressBar()
        progressBar.setAlignment(QtCore.Qt.AlignLeft|QtCore.Qt.AlignVCenter)
        progressBar.setMinimum(0)
        progressBar.setMaximum(100)
        progressBar.setTextVisible(True)
        cancelButton = QtGui.QPushButton()
        cancelButton.setText('Cancel')
        cancelButton.clicked.connect(worker.kill)
        messageBar.layout().addWidget(progressBar)
        messageBar.layout().addWidget(cancelButton)
        self.iface.messageBar().pushWidget(messageBar, self.iface.messageBar().INFO)
        self.messageBar = messageBar
        self.progressBar = progressBar

        # start the worker in a new thread
        thread = QtCore.QThread(self)
        worker.moveToThread(thread)
        worker.finished.connect(self.workerFinished)
        worker.error.connect(self.workerError)
        worker.progress.connect(self.updateProgress)
        thread.started.connect(worker.run)
        thread.start()
        self.thread = thread
        self.worker = worker

    def __init__(self, node_id, data):
        """
        Constructor.

        @type node_id: Integer
        @param node_id: the unique id of this node; between 0 and N-1

        @type data: List of Integer
        @param data: a list containing this node's data
        """
        self.node_id = node_id
        self.data = data
        # temporary buffer for needed for scatter 
	self.copy = data[:]
        self.lock_copy = Lock()
        self.nodes = None
        self.lock_data = Lock()
        # list of threads (in this case 16 fo each node)
	self.thread_list = []
	# list with tasks that need to be computed
        self.thread_pool = []
        self.mutex = Lock()
        # condition used for put and get
	self.condition = Condition(self.mutex)
	# condition needed for checking if there are 
	# still tasks that need o be solved
        self.all_tasks_done = Condition(self.mutex)
	# number of unfinished tasks
        self.unfinished_tasks = 0
        # start the 16 threads
	for i in range(16):
            th = Worker(self, i)
            self.thread_list.append(th)
            th.start()

 def stop(self, force = True):
     """
     Stops all modules and shuts down the manager.
     """
     self.log().debug("Stopping")
     self.stopModules()
     Worker.stop(self, force)

def main():
    w = Worker('1133079545973309867')
    songs = w.get_songs()
    print "\n".join([x.simple_info() for x in songs])

    print '================='

    print "\n".join([x.detail_info() for x in songs])

 def run(self):
     # set thread ID
     self.tid = self._Thread__ident
     try:
         Worker.run(self)
     finally:
         # invalidate thread ID
         self.tid = None