Python Queue.cancel_join_thread方法代码示例

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
	def run(self):
                program_start = time.time()

		wQ = Queue()
                wQ.cancel_join_thread()
		wP = []
		
		for i in range(tp.threads):
			p = Process(target=tp.process_file_write, args=(i,wQ,))
			wP.append(p)
			p.start()

                # 10000 files enter here
                sys.stdout.write(Fore.GREEN + "Reading from stdin...")
                input_files = sys.stdin.readlines()
                for input_file in input_files:
                        wQ.put(input_file.rstrip())
                print Fore.GREEN + "\rDone reading from stdin. I found %d files." % (wQ.qsize())

		for _ in wP:
			wQ.put(None)

		for p in wP:
			p.join()

                program_stop = time.time()
		print Fore.GREEN + "Processed %d files in %f seconds (%fs avg)" % (len(input_files), program_stop-program_start, (program_stop-program_start)/len(input_files))

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
    def start(self):
        """Run the load test."""

        start_time = time.time()

        job_queue = Queue()
        stats_queue = Queue()

        workers = []

        delay = self.options.stagger_workers / 1000.0
        for i in xrange(self.options.workers):
            worker = WorkerBeeProcess(self.options, self.protocol, job_queue, stats_queue)
            worker.start()
            workers.append(worker)
            time.sleep(delay)

        # TODO: consider waiting until workers are ready
        #if self.options.startup_wait:
        #    print "workers started; waiting %d seconds..." % self.options.startup_wait
        #    time.sleep(self.options.startup_wait)

        stats_gatherer = StatsGatherer(self.options, stats_queue)
        stats_gatherer.start()

        queen = QueenBee(self.options, self.arguments, job_queue, stats_queue)
        queen.start()

        # Now wait while the queen does its thing.
        try:
            queen.join()
        except KeyboardInterrupt:
            print "Interrupted, shutting down..."
            queen.terminate()

        print "Waiting for workers to complete jobs and terminate (may take up to %d seconds)..." % self.options.max_ahead

        try:
            stop = Message(Message.STOP)
            for worker in xrange(self.options.workers * self.options.threads):
                job_queue.put(stop)

            # Now wait for the workers to get the message.  This may take a few
            # minutes as the QueenBee likes to stay ahead by a bit.

            for worker in workers:
                worker.join()

            # Tell the Stats Gatherer that it's done.
            stats_queue.put(Message(Message.STOP))

            # Wait for it to finish.
            stats_gatherer.join()

            print "Completed %d jobs in %0.2f seconds." % (queen.jobs_sent.value, time.time() - start_time)
        except KeyboardInterrupt:
            print "Interrupted before shutdown process completed."

            job_queue.cancel_join_thread()
            stats_queue.cancel_join_thread()

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
 def __init__(self, rigol_backend="usbtmc", checkqueuedelay=.09, addqueuetime=.2):
     """
     checkqueuedelay -> How quickly python will check the queues for new voltage
         data to plot.  This value must be less than addqueuetime to ensure that
         every waveform is being plotted.
     addqueuetime -> python waits <addqueuetime> seconds between each query about
         the voltages from the oscilloscope.
     q1 and q2 are the the queues which hold the voltage data for channel 1 and 2
         on the oscilloscope.
     """
     self.lock = RLock()
     self.checkqueuedelay = int(checkqueuedelay * 1000)  # in ms
     self.addqueuetime = addqueuetime  # in sec
     self.dev = rigol.Rigol(rigol_backend)
     self.dev.waveformPointsMode("NORM")
     self.vpp = self.dev.askChannelScale(1) * 4
     self.vpp2 = self.dev.askChannelScale(2) * 4
     self.x = self.dev.getTimebase()
     q1 = Queue()
     q1.cancel_join_thread()
     q2 = Queue()
     q2.cancel_join_thread()
     self.ch1 = False
     self.ch2 = False
     self.start(q1, q2)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class DataBuffer(object):
#========================

  def __init__(self, binary=False):
  #--------------------------------
    self._binary = binary
    self._queue = Queue()
    self._data = [ ]
    self._datalen = 0
    self._pos = 0

  def close(self):
  #---------------
    self._queue.cancel_join_thread()

  def put(self, data):
  #-------------------
    self._queue.put(data)

  def __iter__(self):
  #------------------
    while True:
      while self._pos >= self._datalen:
        data = self._queue.get()
        if data is None: raise StopIteration
        self._data = format_binary(data) if self._binary else format_text(data)
        self._datalen = len(data)
        self._pos = 0
      self._pos += 1
      d = self._data[self._pos - 1]
      ## Data could be a 2-D (or higher?) array.
      if self._binary:
        yield d if not isinstance(d, np.ndarray) else ''.join(d.flatten().tolist())
      else:
        yield d if isinstance(d, str) else ' '.join(d.flatten().tolist())

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class WorkerProcess(object):
    def __init__(self, idnum, topic, collname, in_counter_value, out_counter_value,
                 drop_counter_value, queue_maxsize,
                 mongodb_host, mongodb_port, mongodb_name, nodename_prefix):
        self.name = "WorkerProcess-%4d-%s" % (idnum, topic)
        self.id = idnum
        self.topic = topic
        self.collname = collname
        self.queue = Queue(queue_maxsize)
        self.out_counter = Counter(out_counter_value)
        self.in_counter  = Counter(in_counter_value)
        self.drop_counter = Counter(drop_counter_value)
        self.worker_out_counter = Counter()
        self.worker_in_counter  = Counter()
        self.worker_drop_counter = Counter()
        self.mongodb_host = mongodb_host
        self.mongodb_port = mongodb_port
        self.mongodb_name = mongodb_name
        self.nodename_prefix = nodename_prefix
        self.quit = Value('i', 0)

        self.process = Process(name=self.name, target=self.run)
        self.process.start()

    def init(self):
        global use_setproctitle
	if use_setproctitle:
            setproctitle("mongodb_log %s" % self.topic)

        self.mongoconn = Connection(self.mongodb_host, self.mongodb_port)
        self.mongodb = self.mongoconn[self.mongodb_name]
        self.mongodb.set_profiling_level = SLOW_ONLY

        self.collection = self.mongodb[self.collname]
        self.collection.count()

        self.queue.cancel_join_thread()

        rospy.init_node(WORKER_NODE_NAME % (self.nodename_prefix, self.id, self.collname),
                        anonymous=False)

        self.subscriber = None
        while not self.subscriber:
            try:
                msg_class, real_topic, msg_eval = rostopic.get_topic_class(self.topic, blocking=True)
                self.subscriber = rospy.Subscriber(real_topic, msg_class, self.enqueue, self.topic)
            except rostopic.ROSTopicIOException:
                print("FAILED to subscribe, will keep trying %s" % self.name)
                time.sleep(randint(1,10))
            except rospy.ROSInitException:
                print("FAILED to initialize, will keep trying %s" % self.name)
                time.sleep(randint(1,10))
                self.subscriber = None

    def run(self):
        self.init()

        print("ACTIVE: %s" % self.name)

        # run the thread
        self.dequeue()

        # free connection
        # self.mongoconn.end_request()

    def is_quit(self):
        return self.quit.value == 1

    def shutdown(self):
        if not self.is_quit():
            #print("SHUTDOWN %s qsize %d" % (self.name, self.queue.qsize()))
            self.quit.value = 1
            self.queue.put("shutdown")
            while not self.queue.empty(): sleep(0.1)
        #print("JOIN %s qsize %d" % (self.name, self.queue.qsize()))
        self.process.join()
        self.process.terminate()

 


    def qsize(self):
        return self.queue.qsize()

    def enqueue(self, data, topic, current_time=None):
        if not self.is_quit():
            if self.queue.full():
                try:
                    self.queue.get_nowait()
                    self.drop_counter.increment()
                    self.worker_drop_counter.increment()
                except Empty:
                    pass
            #self.queue.put((topic, data, current_time or datetime.now()))
            self.queue.put((topic, data, rospy.get_time()))
            self.in_counter.increment()
            self.worker_in_counter.increment()

    def dequeue(self):
        while not self.is_quit():
#.........这里部分代码省略.........

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
            self._log("sub: `proc_name` missing for mtype(%s)" % mtype)
            return
        
        if mtype is None:
            self._log("sub: `mtype` missing for proc_name(%s)" % pname)
            return
        
        subs=self._map.get(mtype, [])
        subs.append(pname)
        self._map[mtype]=subs
            

## ================================================================

_centralInputQueue=Queue()
_centralInputQueue.cancel_join_thread()
_mswitch=MessageSwitch(_centralInputQueue)
        
Bus.subscribe("_sub",           _mswitch._hsub)
Bus.subscribe("proc",           _mswitch._hproc)
Bus.subscribe("start",          _mswitch._hstart)

Bus.subscribe("mqueue?",        _mswitch._qmqueue)
Bus.subscribe("mswitch_pump",   _mswitch._hpump)
Bus.subscribe("mswitch_params", _mswitch._hparams)

Bus.subscribe("proc_starting",  _mswitch._hproc_starting)
Bus.subscribe("xsub",           _mswitch._hxsub)
Bus.subscribe("xbridge",        _mswitch._hxbridge)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class EventHistory():
    
    def __init__(self):
        self.queue = Queue()
        self.events = []
        self.running_events = {}
        self.last_aggregation = datetime.datetime.now()
        self.agg_template = {
            'commit': 0,
            'rollback': 0,
            'max_latency': 0.0,
            'running': 0,
            'running_latency': 0.0
        }

    def register_start(self, name):
        event_id = uuid.uuid4()
        self.queue.put({
            'name': name,
            'event_id': event_id,
            'time': datetime.datetime.now()
        })
        return event_id

    def register_finish(self, event_id, status):
        self.queue.put({
            'event_id': event_id,
            'status': status,
            'time': datetime.datetime.now()
        })

    def load_queue(self):
        while not self.queue.empty():
            event = self.queue.get()
            if 'name' in event:
                # start mark
                self.running_events[event['event_id']] = event
            else:
                # finish mark
                if event['event_id'] in self.running_events:
                    start_ev = self.running_events[event['event_id']]
                    self.events.append({
                        'name': start_ev['name'],
                        'started_at': start_ev['time'],
                        'finished_at': event['time'],
                        'status': event['status']
                    })
                    self.running_events.pop(event['event_id'], None)
                else:
                    # found finish event without corresponding start
                    raise
        return

    def aggregate(self):
        self.load_queue()

        agg = {}
        for ev in self.events:
            if ev['finished_at'] < self.last_aggregation:
                continue

            if ev['name'] not in agg:
                agg[ev['name']] = self.agg_template.copy()

            named_agg = agg[ev['name']]
            latency = (ev['finished_at'] - ev['started_at']).total_seconds()
            named_agg[ev['status']] += 1
            if named_agg['max_latency'] < latency:
                named_agg['max_latency'] = latency

        for value in self.running_events.itervalues():
            if value['name'] not in agg:
                agg[value['name']] = self.agg_template.copy()

            named_agg = agg[value['name']]
            latency = (datetime.datetime.now() - value['time']).total_seconds()
            if 'started' in named_agg:
                named_agg['running'] += 1
                if latency > named_agg['running_latency']:
                    named_agg['running_latency'] = latency
            else:
                named_agg['running'] = 1
                named_agg['running_latency'] = latency

        self.last_aggregation = datetime.datetime.now()
        return agg

    def aggregate_by(self, period):
        return

    def close(self):
        print('closing queue')
        self.queue.close()
        print('clearing queue')
        self.load_queue()
        print('joining queue')
        self.queue.cancel_join_thread()

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import cancel_join_thread [as 别名]
class TopicLogger(MongoDBLogger):
    """
    This class implements a generic topic logger.
    It simply dumps all messages received from the topic into the MongoDB.
    """

    def __init__(self, name, topic, collname, mongodb_host, mongodb_port, mongodb_name, max_queuesize=QUEUE_MAXSIZE):
        MongoDBLogger.__init__(self, name, topic, collname, mongodb_host, mongodb_port, mongodb_name)
        self.worker_out_counter = Counter()
        self.worker_in_counter = Counter()
        self.worker_drop_counter = Counter()
        self.queue = Queue(max_queuesize)

    def _init(self):
        """
        This method initializes this process.
        It initializes the connection to the MongoDB and subscribes to the topic.
        """
        self.mongoconn = Connection(self.mongodb_host, self.mongodb_port)
        self.mongodb = self.mongoconn[self.mongodb_name]
        self.mongodb.set_profiling_level = SLOW_ONLY

        self.collection = self.mongodb[self.collname]
        self.collection.count()

        self.queue.cancel_join_thread()
        self.subscriber = None
        while not self.subscriber:
            try:
                msg_class, real_topic, msg_eval = rostopic.get_topic_class(self.topic, blocking=True)
                self.subscriber = rospy.Subscriber(real_topic, msg_class, self._enqueue, self.topic)
            except rostopic.ROSTopicIOException:
                rospy.logwarn("FAILED to subscribe, will keep trying %s" % self.name)
                time.sleep(randint(1, 10))
            except rospy.ROSInitException:
                rospy.logwarn("FAILED to initialize, will keep trying %s" % self.name)
                time.sleep(randint(1, 10))
                self.subscriber = None

    def run(self):
        """
        This method does the actual logging.
        """
        self._init()
        rospy.logdebug("ACTIVE: %s" % self.name)
        # Process the messages
        while not self.is_quit():
            self._dequeue()

        # we must make sure to clear the queue before exiting,
        # or the parent thread might deadlock otherwise
        self.subscriber.unregister()
        self.subscriber = None
        while not self.queue.empty():
            self.queue.get_nowait()
        rospy.logdebug("STOPPED: %s" % self.name)

    def shutdown(self):
        self.queue.put("shutdown")
        super(TopicLogger, self).shutdown()

    def _sanitize_value(self, v):
        if isinstance(v, rospy.Message):
            return self._message_to_dict(v)
        elif isinstance(v, genpy.rostime.Time):
            t = datetime.utcfromtimestamp(v.secs)
            return t + timedelta(microseconds=v.nsecs / 1000.)
        elif isinstance(v, genpy.rostime.Duration):
            return v.secs + v.nsecs / 1000000000.
        elif isinstance(v, list):
            return [self._sanitize_value(t) for t in v]
        else:
            return v

    def _message_to_dict(self, val):
        d = {}
        for f in val.__slots__:
            d[f] = self._sanitize_value(getattr(val, f))
        return d

    def qsize(self):
        return self.queue.qsize()

    def _enqueue(self, data, topic, current_time=None):
        if not self.is_quit():
            if self.queue.full():
                try:
                    self.queue.get_nowait()
                    self.worker_drop_counter.increment()
                except Empty:
                    pass
            self.queue.put((topic, data, rospy.get_time()))
            self.worker_in_counter.increment()

    def _dequeue(self):
        try:
            t = self.queue.get(True)
        except IOError:
            self.quit = True
            return
#.........这里部分代码省略.........