Python BaseManager.start方法代码示例

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def start(self):
        BaseManager.register('get_dispatch_queue',
                             callable=self.get_dispatch_queue)
        BaseManager.register('get_finished_queue',
                             callable=self.get_finished_queue)

        manager = BaseManager(address=('0.0.0.0', 8000),
                              authkey=b'dispatcher')
        manager.start()

        dispatch_queue = manager.get_dispatch_queue()
        finished_queue = manager.get_finished_queue()

        job_id = 0
        unfinished = self.rnd * self.batch
        for rnd in range(self.rnd):
            for i in range(self.batch):
                job = Job(job_id=job_id)
                job_id += 1
                print('Dispatch Job {}'.format(job))
                dispatch_queue.put(job)

            while not dispatch_queue.empty():
                finished_job = finished_queue.get(60)
                unfinished -= 1
                print('Job finished {}'.format(finished_job))
        dispatch_queue.put(None)
        while unfinished > 0:
            finished_job = finished_queue.get(60)
            unfinished -= 1
            print('Job finished {}'.format(finished_job))
        manager.shutdown()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def start(self,skip=0):
        # 把派发作业队列和完成作业队列注册到网络上
        BaseManager.register('get_dispatched_job_queue', callable=self.get_dispatched_job_queue)
        BaseManager.register('get_finished_job_queue', callable=self.get_finished_job_queue)

        # 监听端口和启动服务
        manager = BaseManager(address=('0.0.0.0', rpc_port), authkey=authkey)
        manager.start()

        # 使用上面注册的方法获取队列
        dispatched_jobs = manager.get_dispatched_job_queue()
        finished_jobs = manager.get_finished_job_queue()

        job_id = 0
        module= self.project.modules[self.jobname];

        proj=json.loads(json.dumps(etl.convert_dict(self.project,self.project.__defaultdict__), ensure_ascii=False))
        while True:
            for task in etl.parallel_map(module):
                job_id = job_id + 1
                if job_id<skip:
                    continue
                job = ETLJob(proj, self.jobname, task, job_id);
                print('Dispatch job: %s' % job.id)
                dispatched_jobs.put(job)

            while not dispatched_jobs.empty():
                job = finished_jobs.get(60)
                print('Finished Job: %s, Count: %s' % (job.id, job.count))

            key=input('press any key to repeat,c to cancel')
            if key=='c':
                manager.shutdown()
                break

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def start(self):

        # 把派发作业队列和完成作业队列注册到网络上
        BaseManager.register(
            'get_dispatched_job_queue', callable=self.get_dispatched_job_queue)
        BaseManager.register(
            'get_finished_job_queue', callable=self.get_finished_job_queue)

        # 监听端口和启动服务
        manager = BaseManager(address=('0.0.0.0', 8888), authkey='jobs')
        manager.start()

        # 使用上面注册的方法获取队列
        dispatched_jobs = manager.get_dispatched_job_queue()
        finished_jobs = manager.get_finished_job_queue()

        # 这里一次派发10个作业，等到10个作业都运行完后，继续再派发10个作业
        job_id = 0
        while True:
            for i in range(0, 10):
                job_id = job_id + 1
                job = Job(job_id)
                print('Dispatch job: %s' % job.job_id)
                dispatched_jobs.put(job)

            while not dispatched_jobs.empty():
                job = finished_jobs.get(60)
                print('Finished Job: %s' % job.job_id)

        manager.shutdown()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
 def __init__(self):
     self.config.read('config.ini')
     BaseManager.register('Queue', queue.Queue)
     manager = BaseManager()
     manager.start()
     self.queue = manager.Queue()
     self.threads = {
         'sensorreading_submitter': {
             'last_run': None,
             'timeout': int(self.config.get('api', 'submit_sensorreadings_interval')),
             'thread': None
         },
         'desired_state_fetcher': {
             'last_run': None,
             'timeout': int(self.config.get('api', 'fetch_desired_states_interval')),
             'thread': None
         },
         'sensorreading_buffer': {
             'last_run': None,
             'timeout': int(self.config.get('api', 'sensorreading_buffer_flush_interval')),
             'thread': None
         },
         'maintenance': {
             'last_run': None,
             'timeout': int(self.config.get('api', 'maintenance_interval')),
             'thread': None
         }
     }

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
 def create(self, name, **ctor_args):
     ''' Create a :class:`ConsoleServer` and return a proxy for it. 
     '''
     manager = BaseManager()
     manager.register('ConsoleServer', ConsoleServer)
     manager.start()
     return manager.ConsoleServer()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
 def i_images_processing(self, vs): 
     
     """
     Interface function to launch processing VHRS images :func:`i_vhrs` and satellite images :func:`i_img_sat` in multi-processing.
     
     :param vs: Boolean variable to launch processing because of interface checkbox -> 0 or 1.
 
         - 0 means, not texture processing
         - 1 means, launch texture processing
     :type vs: int
     """
     
     # Multiprocessing
     mgr = BaseManager()
     mgr.register('defaultdict', defaultdict, DictProxy)
     mgr.start()
     self.out_ndvistats_folder_tab = mgr.defaultdict(list)
     
     p_img_sat = Process(target=self.i_img_sat)
     p_img_sat.start()
     if self.mp == 0:
         p_img_sat.join()
     
     if vs == 1:
         p_vhrs = Process(target=self.i_vhrs)#, args=(vs, ))
         p_vhrs.start()
         p_vhrs.join()
     
     if self.mp == 1:
         p_img_sat.join()
     
     # List of output raster path
     self.raster_path.append(self.out_ndvistats_folder_tab[0])
     # List of output raster band
     self.list_band_outraster.append(1)
     
     if vs == 1:
         self.raster_path.append(self.out_ndvistats_folder_tab['sfs'])
         self.list_band_outraster.append(4)
         self.raster_path.append(self.out_ndvistats_folder_tab['haralick'])
         self.list_band_outraster.append(2)
     
     # To slope, to extract scree
     if self.path_mnt != '':
         self.raster_path.append(self.path_mnt)
         self.list_band_outraster.append(1)
         
     self.raster_path.append(self.out_ndvistats_folder_tab[1])
     # example raster path tab :
     #                [path_folder_dpt + '/' + folder_processing + '/' + classif_year + '/Min_2014.TIF',\
     #                os.path.dirname(path_ortho) + '/Clip_buffer_surface_dep_18_IRCOrtho65_2m_sfs.TIF',\
     #                os.path.dirname(path_ortho) + '/Clip_buffer_surface_dep_18_IRCOrtho65_2m_haralick.TIF',\
     #                path_folder_dpt + '/' + folder_processing + '/' + classif_year + '/Max_2014.TIF']
     
     # List of output raster band
     self.list_band_outraster.append(1) #[1, 4, 2, 1]
     
     print("End of images processing !")

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def start(self):
        BaseManager.start(self)

        # 加载任务队列和结果队列
        if not self.task_queue:
            self.task_queue = self.get_task_queue()

        if not self.result_queue:
            self.result_queue = self.get_result_queue()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def _setup(self):
        free_phy_mem = available_memory(percent=35)
        maxsize = int(free_phy_mem / BLOCK_SIZE)
        self._input_buffer = self.manager.Queue(maxsize)
        self.output_buffer = self.manager.Queue(maxsize)

        BaseManager.register("ReoderBuffer", ReoderBuffer)
        bm = BaseManager()
        bm.start()
        self.reorder_buffer = bm.ReoderBuffer(self.output_buffer, 50)

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
 def StartManager(self, serveraddress="Localhost", port=80, key=None):
     if(self.queueManager == None):
         try:
             key = key.encode("utf-8")
         except:
             pass
         BaseManager.register('GetTaskQueue', callable=ProcessingQueueManager.ReturnTaskQueue)
         BaseManager.register('GetResultQueue', callable=ProcessingQueueManager.ReturnResultQueue)
         queueManager = BaseManager(address=(serveraddress, port), authkey=key)
         queueManager.start()
         self.queueManager = queueManager

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def run(self, end_node):
        if self._mode == EXEC_MODE_BATCH:
            BaseManager.register('FlowStates', FlowStates)
            BaseManager.register('Node', Node)
            manager = BaseManager()
            manager.start()
            stat = manager.FlowStates()

            p = Process(target=self._run_batch, args=(end_node, stat))
            p.start()
            return stat
        elif self._mode == EXEC_MODE_STREAMING:
            self._run_streaming(end_node)

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
	def __init__(self):
		# 发送任务的队列:
		task_queue = Queue.Queue()
		# 接收结果的队列:
		result_queue = Queue.Queue()
		class QueueManager(BaseManager):
		    pass
		# 把两个Queue都注册到网络上, callable参数关联了Queue对象:
		BaseManager.register('get_task_queue', callable=lambda: task_queue)
		BaseManager.register('get_result_queue', callable=lambda: result_queue)
		# 绑定端口5000, 设置验证码'abc':
		manager = BaseManager(address=('',8888), authkey='abc')
		# 启动Queue:
		manager.start()
		# 获得通过网络访问的Queue对象:
		self.task = manager.get_task_queue()
		self.result = manager.get_result_queue()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
class QueueServer(object):

    def __init__(self, queue_server_host, queue_server_port, queue_server_authkey=None):
        '''
            host,port defines where your queuing server should be running while *authkey* 
            is going to be used to authenticate any communication between this queue server
            and clients connected to it. Clients will need to send the *authkey* to connect
            this queue server.
        '''
        self.host = queue_server_host
        self.port = queue_server_port
        self.authkey = queue_server_authkey

        '''
        Lets just say, we have a client that wants to put some image realted data into database 
        and also want to generate thumbnails from it (You know, where it is going,
        I'll give you a hint, checkout my last post about multi-threading)
        '''
        database_queue = Queue()
        thumbnail_queue = Queue()
        
        '''now we have a queue, but if since we want our clients to use it
        we'll need to register this queue with BaseManager via some callable that our client 
        can use to generate the proxy object. Yes, clients will be actually
        able to get the (proxy) object of this Queue and for them, they can
        pretty much use it like a regular queue (however, internally, BaseManager
        will be proxying that data sharing between client and server (and thats the 
        fun, we don't have to worry about locking, shared memory handling etc as 
        BaseManager will handle that, and for us it will be like using Queue between
        threads'''
        BaseManager.register('database_queue', callable=lambda:database_queue)
        BaseManager.register('thumbnail_queue', callable=lambda:thumbnail_queue)

        '''Now that we have registered our queue with BaseManager, we can instantiate
        manager object and start the server. As mentioned, BaseManager will spawn a 
        server in a subprocess and will handle all the communcation and data synchronization'''
        self.manager = BaseManager(address=(self.host, self.port), 
                                   authkey=self.authkey)
        
    def start(self):
        print 'Starting Server Process...'
        self.manager.start()
        
    def stop(self):
        self.manager.shutdown()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
    def start(self):
        BaseManager.register('get_dispatched_job_queue', callable=self.get_dispatched_job_queue)
        BaseManager.register('get_finished_job_queue', callable=self.get_finished_job_queue)

        manager = BaseManager(address=('0.0.0.0', 8888), authkey='jobs')
        manager.start()

        dispatched_jobs = manager.get_dispatched_job_queue()
        finished_jobs = manager.get_finished_job_queue()

        job_id = 0
        while True:
            for i in range(0, 10):
                job_id = job_id + 1
                job = Job(job_id)
                print('Dispatch job: %s' % job.job_id)
                dispatched_jobs.put(job)

            while not dispatched_jobs.empty():
                job = finished_jobs.get(60)
                print('Finished Job: %s' % job.job_id)

        manager.shutdown()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
def manager():
    # load 'session.txt', or call login() to generate it
    try:
        with open('session.txt', 'rb') as f:
            headers = cPickle.load(f)
            cookies = cPickle.load(f)
    except:
        print '[-] 无session.txt文件, 调用login()...'
        session = DoubanLogin().login()
        headers = session.headers
        cookies = session.cookies

    # create task queue and result queue
    task_queue = Queue.Queue()
    result_queue = Queue.Queue()

    # register
    BaseManager.register('get_task_queue', callable=lambda: task_queue)
    BaseManager.register('get_result_queue', callable=lambda: result_queue)

    # bound port 5000, set authkey
    manager = BaseManager(address=('', PORT), authkey='douban')
    manager.start()
    task = manager.get_task_queue()
    result = manager.get_result_queue()

    # load task file
    done = task2file.load('done.txt')
    todo = task2file.load('todo.txt')

    # initial task(if no task file)
    new = set([INIT_ID])
    todo = (todo | (new - (new & done)))
    count = 1

    try:
        while len(todo) != 0:
            try:
                # select an id_ then send it to worker's task queue
                id_ = todo.pop()
                task.put(id_)
                print '\n[+] ========  No.%d  ID: %s  ========' % (count, id_)
                print '[~] Send to task queue...'
                time.sleep(DELAY_TIME)
                new = get_user_id.get_id(id_, headers=headers, cookies=cookies)

                # set() operation, add new IDs to todo
                add = (new - (new & done))
                todo = (todo | add)
                print '[+] 新发现用户ID: %d 个' % len(add)
                print '[~] Receiving User Information...'
                data = result.get()

                # save user information to 'info.txt'
                get_user_info.write2file('info.txt', data)
                print '[+] 已将用户信息保存至: info.txt'
                # add id_ to done
                done.add(id_)
                count += 1

                # to avoid task set expanding too fast, write them to file in time
                task2file.save('todo.txt', todo)
                task2file.save('done.txt', done)

            except Exception, e:
                print e
                exit()

    finally:
        manager.shutdown()
        print '\n[+] Manager exit.'
        exit()

# 需要导入模块: from multiprocessing.managers import BaseManager [as 别名]
# 或者: from multiprocessing.managers.BaseManager import start [as 别名]
 def start(self, initializer=ignore_interrupt, initargs=()):
     BaseManager.start(self, initializer=initializer, initargs=initargs)