Python SyncManager.list方法代码示例

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
    def __init__(self, servo_id):
        self.servo_id = servo_id
        self.angle = Value('f', 0.0)
        self.stop_signal = Value('b', False)

        # http://jtushman.github.io/blog/2014/01/14/python-|-multiprocessing-and-interrupts/
        manager = SyncManager() # instead of regular Manager because we want to ignore kb interrupt
        manager.start(Servo.init_mgr) # start the manager explicitly
        self.command_queue = manager.list([])
        self.current_command = manager.dict()

        self.finished = Value('b', False)

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
            sys.stderr.write(e.msg)
            failed = True

        scenarios_to_run = self.sort_scenarios(scenarios_to_run)

        # Explictly starting a sync manager to remain after keyboard interupt, so we can print out the errors
        manager = SyncManager()
        try:

            # shared state
            def mgr_init():
                signal.signal(signal.SIGINT, signal.SIG_IGN)

            manager.start(mgr_init)

            errors = manager.list()
            results = manager.list()
            scenario_queue = manager.list()
            interupted = multiprocessing.Event()

            for s in scenarios_to_run:
                scenario_queue.append(s)

            call_hook("before", "all")

            processes = []
            for i in xrange(self.workers):
                port_number = 8180 + i
                process = multiprocessing.Process(
                    target=process_scenarios, args=(scenario_queue, port_number, results, errors, interupted)
                )

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
def getSharedLists():
  manager = SyncManager()
  manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
  return (manager.list(), manager.list())

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
class getImgFile(PostGetter):
	def __init__(self, cookie_file, imgfile_base_path):
		PostGetter.__init__(self, cookie_file)
		self.imgfile_base_path = os.path.expanduser(imgfile_base_path)
		assert os.access(self.imgfile_base_path, os.F_OK) == True

		self.fileext = {'jpeg': 'jpg', 'png': 'png', 'gif': 'gif', 'jpg': 'jpg', 'bmp': 'bmp', 'tiff': 'tif'}
		self.pImg = re.compile(r'<img.*?\s+src\s*=\s*"\s*(?!")(.+?)\s*".*?/>', re.U | re.M | re.S | re.I)
		self.bad_domain = []
		self.use_proxy = []
		self.nms = None
		self.pDomainUseProxy = []
		self.multiprocess_manager = None


	def getImageUrl(self, forum_name, sector_name, id_from=None, nr_limit=None):
		u'''获取 reply里面的 img 链接，并入库
		'''
		cnt, tmpminid, step = 0, 0, 5000
##		forum=Forum.objects.get(name__iexact=forum_name)
		sector = Sector.objects.get(forum__name__iexact=forum_name, name__iexact=sector_name)
		if id_from:
			tmpminid = id_from
			maxid = Reply.objects.filter(post__sector=sector).aggregate(maxid=Max('id'))['maxid']
		elif self.processed_maxid:
			self.logger.info('max id processed last time is %d', self.processed_maxid)
			tmpminid = self.processed_maxid  # 上次处理过的reply的最大id
			maxid = Reply.objects.filter(post__sector=sector).aggregate(maxid=Max('id'))['maxid']
		else:
			tmp = Reply.objects.filter(post__sector=sector).aggregate(minid=Min('id'), maxid=Max('id'))
			tmpminid, maxid = tmp['minid'], tmp['maxid']

		self.logger.info('%s\n\tget img url from %s minid,maxid=%d,%d\n\n', '~~~~'*20, sector_name, tmpminid, maxid)
		itertimes = 0
		while (not nr_limit) or (nr_limit and cnt <= nr_limit):
			self.logger.info('%s%d) cnt=%d, tmpminid=%d %s', '-='*15, itertimes, cnt, tmpminid, '-=' * 15)
			rl = Reply.objects.defer('locate_id', 'author', 'crt_date').filter(
				post__sector = sector, id__gte = tmpminid, id__lt = tmpminid+step).order_by('id').iterator()

			for reply in rl:
				cnt += 1
##				self.logger.info('%d) reply %d',cnt,reply.id)

				if nr_limit and nr_limit <= cnt:
					self.logger.info('to nr_limit ! %d', reply.id)
					break

				self._getImg4Reply(reply)
				self.processed_maxid = reply.id

				if self.exitevent.is_set():
					self.logger.info('got exit signal!')
					break


			if self.exitevent.is_set():
				self.logger.info('got exit signal!')
				break

			if nr_limit and nr_limit <= cnt:
				self.logger.info('to nr_limit !')
				break

			if tmpminid > maxid:
				self.logger.info('to maxid! %d', tmpminid)
				break


			itertimes += 1
			tmpminid += step


	def _getImg4Reply(self, reply):
		u'''从 reply 中 获取 img 链接存入数据库'''
		imglist = self.pImg.findall(reply.content)
		dbg_title_prted = False
		premovetag = re.compile('(<.*?>)', re.M | re.S)
		if imglist:
			for i, x in enumerate(imglist):
##				self.logger.info('\t(%d) %d|%s',i+1,reply.id,x)
				# unquote chars like %E5%88%98%E5%BC%BA
##				x=re.sub('(?:(?:%[0-9A-Fa-f]{2})+)+?',lambda xx:self._tryDecode(urllib.unquote(xx.group().encode('gbk'))),x)
				try:
					x = re.sub('(?:(?:%[0-9A-Za-z]{2,3})+)+?', lambda xx: self._tryDecode(urllib.unquote(xx.group().encode('gbk'))), x)
				except UnicodeDecodeError as e:
					self.logger.info('reply %d except: %s\n\t%s', reply.id, e, x)
					continue

				# exceed db field length limit
				if len(x) > 255:
					self.logger.debug('got long url for reply %d, %d', reply.id, len(x))
					continue

				# remove tag
				x = premovetag.sub('', x)

				# convert related path to absolute path
				sr = urlparse.urlsplit(x)

				if sr.scheme == '':
#.........这里部分代码省略.........

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
#!/usr/bin/env python

from multiprocessing.managers import SyncManager
import sys

def f(d, l):
    d[1] = '1'
    d['2'] = 2
    d[0.25] = None
    l.reverse()

if __name__ == '__main__':
    manager = SyncManager(address=('127.0.0.1', int(sys.argv[1])), authkey='abc')

    manager.connect()

    print manager.list()

    print 'Done'

# 需要导入模块: from multiprocessing.managers import SyncManager [as 别名]
# 或者: from multiprocessing.managers.SyncManager import list [as 别名]
#!/usr/bin/env python

from multiprocessing.managers import SyncManager

def f(d, l):
    d[1] = '1'
    d['2'] = 2
    d[0.25] = None
    l.reverse()

if __name__ == '__main__':
    manager = SyncManager(address=('127.0.0.1', 0), authkey='abc')
    server = manager.get_server()

    manager.start()

    d = manager.dict()
    l = manager.list(range(10))

    print server.address
    server.serve_forever()