本文整理汇总了Python中multiprocessing.Queue.cancel_join_thread方法的典型用法代码示例。如果您正苦于以下问题:Python Queue.cancel_join_thread方法的具体用法?Python Queue.cancel_join_thread怎么用?Python Queue.cancel_join_thread使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类multiprocessing.Queue
的用法示例。
在下文中一共展示了Queue.cancel_join_thread方法的8个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: run
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
def run(self):
program_start = time.time()
wQ = Queue()
wQ.cancel_join_thread()
wP = []
for i in range(tp.threads):
p = Process(target=tp.process_file_write, args=(i,wQ,))
wP.append(p)
p.start()
# 10000 files enter here
sys.stdout.write(Fore.GREEN + "Reading from stdin...")
input_files = sys.stdin.readlines()
for input_file in input_files:
wQ.put(input_file.rstrip())
print Fore.GREEN + "\rDone reading from stdin. I found %d files." % (wQ.qsize())
for _ in wP:
wQ.put(None)
for p in wP:
p.join()
program_stop = time.time()
print Fore.GREEN + "Processed %d files in %f seconds (%fs avg)" % (len(input_files), program_stop-program_start, (program_stop-program_start)/len(input_files))
示例2: start
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
def start(self):
"""Run the load test."""
start_time = time.time()
job_queue = Queue()
stats_queue = Queue()
workers = []
delay = self.options.stagger_workers / 1000.0
for i in xrange(self.options.workers):
worker = WorkerBeeProcess(self.options, self.protocol, job_queue, stats_queue)
worker.start()
workers.append(worker)
time.sleep(delay)
# TODO: consider waiting until workers are ready
#if self.options.startup_wait:
# print "workers started; waiting %d seconds..." % self.options.startup_wait
# time.sleep(self.options.startup_wait)
stats_gatherer = StatsGatherer(self.options, stats_queue)
stats_gatherer.start()
queen = QueenBee(self.options, self.arguments, job_queue, stats_queue)
queen.start()
# Now wait while the queen does its thing.
try:
queen.join()
except KeyboardInterrupt:
print "Interrupted, shutting down..."
queen.terminate()
print "Waiting for workers to complete jobs and terminate (may take up to %d seconds)..." % self.options.max_ahead
try:
stop = Message(Message.STOP)
for worker in xrange(self.options.workers * self.options.threads):
job_queue.put(stop)
# Now wait for the workers to get the message. This may take a few
# minutes as the QueenBee likes to stay ahead by a bit.
for worker in workers:
worker.join()
# Tell the Stats Gatherer that it's done.
stats_queue.put(Message(Message.STOP))
# Wait for it to finish.
stats_gatherer.join()
print "Completed %d jobs in %0.2f seconds." % (queen.jobs_sent.value, time.time() - start_time)
except KeyboardInterrupt:
print "Interrupted before shutdown process completed."
job_queue.cancel_join_thread()
stats_queue.cancel_join_thread()
示例3: __init__
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
def __init__(self, rigol_backend="usbtmc", checkqueuedelay=.09, addqueuetime=.2):
"""
checkqueuedelay -> How quickly python will check the queues for new voltage
data to plot. This value must be less than addqueuetime to ensure that
every waveform is being plotted.
addqueuetime -> python waits <addqueuetime> seconds between each query about
the voltages from the oscilloscope.
q1 and q2 are the the queues which hold the voltage data for channel 1 and 2
on the oscilloscope.
"""
self.lock = RLock()
self.checkqueuedelay = int(checkqueuedelay * 1000) # in ms
self.addqueuetime = addqueuetime # in sec
self.dev = rigol.Rigol(rigol_backend)
self.dev.waveformPointsMode("NORM")
self.vpp = self.dev.askChannelScale(1) * 4
self.vpp2 = self.dev.askChannelScale(2) * 4
self.x = self.dev.getTimebase()
q1 = Queue()
q1.cancel_join_thread()
q2 = Queue()
q2.cancel_join_thread()
self.ch1 = False
self.ch2 = False
self.start(q1, q2)
示例4: DataBuffer
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class DataBuffer(object):
#========================
def __init__(self, binary=False):
#--------------------------------
self._binary = binary
self._queue = Queue()
self._data = [ ]
self._datalen = 0
self._pos = 0
def close(self):
#---------------
self._queue.cancel_join_thread()
def put(self, data):
#-------------------
self._queue.put(data)
def __iter__(self):
#------------------
while True:
while self._pos >= self._datalen:
data = self._queue.get()
if data is None: raise StopIteration
self._data = format_binary(data) if self._binary else format_text(data)
self._datalen = len(data)
self._pos = 0
self._pos += 1
d = self._data[self._pos - 1]
## Data could be a 2-D (or higher?) array.
if self._binary:
yield d if not isinstance(d, np.ndarray) else ''.join(d.flatten().tolist())
else:
yield d if isinstance(d, str) else ' '.join(d.flatten().tolist())
示例5: WorkerProcess
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class WorkerProcess(object):
def __init__(self, idnum, topic, collname, in_counter_value, out_counter_value,
drop_counter_value, queue_maxsize,
mongodb_host, mongodb_port, mongodb_name, nodename_prefix):
self.name = "WorkerProcess-%4d-%s" % (idnum, topic)
self.id = idnum
self.topic = topic
self.collname = collname
self.queue = Queue(queue_maxsize)
self.out_counter = Counter(out_counter_value)
self.in_counter = Counter(in_counter_value)
self.drop_counter = Counter(drop_counter_value)
self.worker_out_counter = Counter()
self.worker_in_counter = Counter()
self.worker_drop_counter = Counter()
self.mongodb_host = mongodb_host
self.mongodb_port = mongodb_port
self.mongodb_name = mongodb_name
self.nodename_prefix = nodename_prefix
self.quit = Value('i', 0)
self.process = Process(name=self.name, target=self.run)
self.process.start()
def init(self):
global use_setproctitle
if use_setproctitle:
setproctitle("mongodb_log %s" % self.topic)
self.mongoconn = Connection(self.mongodb_host, self.mongodb_port)
self.mongodb = self.mongoconn[self.mongodb_name]
self.mongodb.set_profiling_level = SLOW_ONLY
self.collection = self.mongodb[self.collname]
self.collection.count()
self.queue.cancel_join_thread()
rospy.init_node(WORKER_NODE_NAME % (self.nodename_prefix, self.id, self.collname),
anonymous=False)
self.subscriber = None
while not self.subscriber:
try:
msg_class, real_topic, msg_eval = rostopic.get_topic_class(self.topic, blocking=True)
self.subscriber = rospy.Subscriber(real_topic, msg_class, self.enqueue, self.topic)
except rostopic.ROSTopicIOException:
print("FAILED to subscribe, will keep trying %s" % self.name)
time.sleep(randint(1,10))
except rospy.ROSInitException:
print("FAILED to initialize, will keep trying %s" % self.name)
time.sleep(randint(1,10))
self.subscriber = None
def run(self):
self.init()
print("ACTIVE: %s" % self.name)
# run the thread
self.dequeue()
# free connection
# self.mongoconn.end_request()
def is_quit(self):
return self.quit.value == 1
def shutdown(self):
if not self.is_quit():
#print("SHUTDOWN %s qsize %d" % (self.name, self.queue.qsize()))
self.quit.value = 1
self.queue.put("shutdown")
while not self.queue.empty(): sleep(0.1)
#print("JOIN %s qsize %d" % (self.name, self.queue.qsize()))
self.process.join()
self.process.terminate()
def qsize(self):
return self.queue.qsize()
def enqueue(self, data, topic, current_time=None):
if not self.is_quit():
if self.queue.full():
try:
self.queue.get_nowait()
self.drop_counter.increment()
self.worker_drop_counter.increment()
except Empty:
pass
#self.queue.put((topic, data, current_time or datetime.now()))
self.queue.put((topic, data, rospy.get_time()))
self.in_counter.increment()
self.worker_in_counter.increment()
def dequeue(self):
while not self.is_quit():
#.........这里部分代码省略.........
示例6: mtype
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
self._log("sub: `proc_name` missing for mtype(%s)" % mtype)
return
if mtype is None:
self._log("sub: `mtype` missing for proc_name(%s)" % pname)
return
subs=self._map.get(mtype, [])
subs.append(pname)
self._map[mtype]=subs
## ================================================================
_centralInputQueue=Queue()
_centralInputQueue.cancel_join_thread()
_mswitch=MessageSwitch(_centralInputQueue)
Bus.subscribe("_sub", _mswitch._hsub)
Bus.subscribe("proc", _mswitch._hproc)
Bus.subscribe("start", _mswitch._hstart)
Bus.subscribe("mqueue?", _mswitch._qmqueue)
Bus.subscribe("mswitch_pump", _mswitch._hpump)
Bus.subscribe("mswitch_params", _mswitch._hparams)
Bus.subscribe("proc_starting", _mswitch._hproc_starting)
Bus.subscribe("xsub", _mswitch._hxsub)
Bus.subscribe("xbridge", _mswitch._hxbridge)
示例7: EventHistory
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class EventHistory():
def __init__(self):
self.queue = Queue()
self.events = []
self.running_events = {}
self.last_aggregation = datetime.datetime.now()
self.agg_template = {
'commit': 0,
'rollback': 0,
'max_latency': 0.0,
'running': 0,
'running_latency': 0.0
}
def register_start(self, name):
event_id = uuid.uuid4()
self.queue.put({
'name': name,
'event_id': event_id,
'time': datetime.datetime.now()
})
return event_id
def register_finish(self, event_id, status):
self.queue.put({
'event_id': event_id,
'status': status,
'time': datetime.datetime.now()
})
def load_queue(self):
while not self.queue.empty():
event = self.queue.get()
if 'name' in event:
# start mark
self.running_events[event['event_id']] = event
else:
# finish mark
if event['event_id'] in self.running_events:
start_ev = self.running_events[event['event_id']]
self.events.append({
'name': start_ev['name'],
'started_at': start_ev['time'],
'finished_at': event['time'],
'status': event['status']
})
self.running_events.pop(event['event_id'], None)
else:
# found finish event without corresponding start
raise
return
def aggregate(self):
self.load_queue()
agg = {}
for ev in self.events:
if ev['finished_at'] < self.last_aggregation:
continue
if ev['name'] not in agg:
agg[ev['name']] = self.agg_template.copy()
named_agg = agg[ev['name']]
latency = (ev['finished_at'] - ev['started_at']).total_seconds()
named_agg[ev['status']] += 1
if named_agg['max_latency'] < latency:
named_agg['max_latency'] = latency
for value in self.running_events.itervalues():
if value['name'] not in agg:
agg[value['name']] = self.agg_template.copy()
named_agg = agg[value['name']]
latency = (datetime.datetime.now() - value['time']).total_seconds()
if 'started' in named_agg:
named_agg['running'] += 1
if latency > named_agg['running_latency']:
named_agg['running_latency'] = latency
else:
named_agg['running'] = 1
named_agg['running_latency'] = latency
self.last_aggregation = datetime.datetime.now()
return agg
def aggregate_by(self, period):
return
def close(self):
print('closing queue')
self.queue.close()
print('clearing queue')
self.load_queue()
print('joining queue')
self.queue.cancel_join_thread()
示例8: TopicLogger
# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class TopicLogger(MongoDBLogger):
"""
This class implements a generic topic logger.
It simply dumps all messages received from the topic into the MongoDB.
"""
def __init__(self, name, topic, collname, mongodb_host, mongodb_port, mongodb_name, max_queuesize=QUEUE_MAXSIZE):
MongoDBLogger.__init__(self, name, topic, collname, mongodb_host, mongodb_port, mongodb_name)
self.worker_out_counter = Counter()
self.worker_in_counter = Counter()
self.worker_drop_counter = Counter()
self.queue = Queue(max_queuesize)
def _init(self):
"""
This method initializes this process.
It initializes the connection to the MongoDB and subscribes to the topic.
"""
self.mongoconn = Connection(self.mongodb_host, self.mongodb_port)
self.mongodb = self.mongoconn[self.mongodb_name]
self.mongodb.set_profiling_level = SLOW_ONLY
self.collection = self.mongodb[self.collname]
self.collection.count()
self.queue.cancel_join_thread()
self.subscriber = None
while not self.subscriber:
try:
msg_class, real_topic, msg_eval = rostopic.get_topic_class(self.topic, blocking=True)
self.subscriber = rospy.Subscriber(real_topic, msg_class, self._enqueue, self.topic)
except rostopic.ROSTopicIOException:
rospy.logwarn("FAILED to subscribe, will keep trying %s" % self.name)
time.sleep(randint(1, 10))
except rospy.ROSInitException:
rospy.logwarn("FAILED to initialize, will keep trying %s" % self.name)
time.sleep(randint(1, 10))
self.subscriber = None
def run(self):
"""
This method does the actual logging.
"""
self._init()
rospy.logdebug("ACTIVE: %s" % self.name)
# Process the messages
while not self.is_quit():
self._dequeue()
# we must make sure to clear the queue before exiting,
# or the parent thread might deadlock otherwise
self.subscriber.unregister()
self.subscriber = None
while not self.queue.empty():
self.queue.get_nowait()
rospy.logdebug("STOPPED: %s" % self.name)
def shutdown(self):
self.queue.put("shutdown")
super(TopicLogger, self).shutdown()
def _sanitize_value(self, v):
if isinstance(v, rospy.Message):
return self._message_to_dict(v)
elif isinstance(v, genpy.rostime.Time):
t = datetime.utcfromtimestamp(v.secs)
return t + timedelta(microseconds=v.nsecs / 1000.)
elif isinstance(v, genpy.rostime.Duration):
return v.secs + v.nsecs / 1000000000.
elif isinstance(v, list):
return [self._sanitize_value(t) for t in v]
else:
return v
def _message_to_dict(self, val):
d = {}
for f in val.__slots__:
d[f] = self._sanitize_value(getattr(val, f))
return d
def qsize(self):
return self.queue.qsize()
def _enqueue(self, data, topic, current_time=None):
if not self.is_quit():
if self.queue.full():
try:
self.queue.get_nowait()
self.worker_drop_counter.increment()
except Empty:
pass
self.queue.put((topic, data, rospy.get_time()))
self.worker_in_counter.increment()
def _dequeue(self):
try:
t = self.queue.get(True)
except IOError:
self.quit = True
return
#.........这里部分代码省略.........