Python LifoQueue.get方法代码示例

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class ThreadedNormalWorker(object):
    def __init__(self, print_errors=False):
        self.print_errors = print_errors
        self.queue = LifoQueue()

    def get_url_bulk(self):
        normals = Normals.objects(access_success=False)
        for i in normals:
            self.queue.put(item=i)

    def grab_from_queue(self):
        while not self.queue.empty():
            url = self.queue.get()
            normals_finder = NormalsSpider(url=url.url,
                                           print_errors=self.print_errors)
            normals_finder.update_normals_data()
            print(url.url)
            self.queue.task_done()

    def start(self, n_threads):
        self.get_url_bulk()
        for i in range(n_threads):
            thread = Thread(target=self.grab_from_queue())
            thread.start()
        self.queue.join()

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
    def match_query(self, query):
        '''Given a search query, return a tuple containing a regex match and
        trigger object that matches the given query.  If no match can be found,
        return a tuple of (None, None).'''

        sink = LifoQueue()

        while not self.triggers.empty():
            trigger = self.triggers.get()
            match = trigger.pattern.match(query)

            if match:
                break

            else:
                sink.put(trigger)
                trigger = None

        while not sink.empty():
            self.triggers.put(sink.get())

        if trigger:
            self.triggers.put(trigger)
            return (match, trigger)

        return (None, None)

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class QueryQueue:
    def __init__(self):
        self.queue = LifoQueue()
        self.comm_sender = CommSender()
        th = threading.Thread(target=self.send_require)
        th.start()

    def put(self, item):
        self.queue.put(item)

    def send_require(self):
        while True:
            time.sleep(1)
            c = ConnInfo.objects.all()[0]
            q = QueryInfo.objects.all()[0]
            r = RoomInfo.objects.all()[0]
            # if is logout or unconnected, only flush queue
            if c.is_log == "False" or c.is_conn == "False":
                while not self.queue.empty():
                    self.queue.get()
                continue

            # else get last item and flush queue
            if not self.queue.empty():
                query = self.queue.get()
                while not self.queue.empty():
                    self.queue.get()
                #
                m = ModeInfo.objects.all()[0]
                s = SensorInfo.objects.all()[0]
                ss = SettingInfo.objects.all()[0]
                if m.mode == 'cold' and ss.target_temp > s.current_temp:
                    query = 'standby'
                elif m.mode == 'hot' and ss.target_temp < s.current_temp:
                    query = 'standby'
                #
                q.query_speed = query
                q.save()
                r = self.comm_sender.send_msg(data={'type': 'require', 'source': r.room_number, 'speed': query})
                # if query is standby, we should change to standby immediately
                if query == 'standby' and r.json()['ack_nak'] == 'ACK':
                    q.current_speed = 'standby'
                    q.query_speed = 'None'
                    q.save()

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
def inorder_walk(a_root_node: BSTreeNode):
    node_stack = LifoQueue()
    current_item = a_root_node
    while True:
        while current_item:
            node_stack.put(current_item)
            current_item = current_item.left_child
        if node_stack.empty():
            break
        tmp_item = node_stack.get()
        yield tmp_item

        current_item = tmp_item.right_child

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
    def index(self, conf):

        session = requests.session()
        urls = LifoQueue()

        allowed_domains = conf['allowed_domains'].split(',')
        start = conf['url']
        ignore = re.compile(conf['ignore'])

        found = set([start])
        urls.put(start)

        while not urls.empty():
            url = urls.get()

            r = session.get(url)

            for link in BeautifulSoup(r.content, 'lxml').find_all('a'):
                link_href = link.get('href')

                if not link_href:
                    continue

                if link_href.startswith('/'):
                    link_href = urljoin(url, link_href)

                parsed = urlparse(link_href)

                if parsed.hostname not in allowed_domains:
                    continue

                if conf['ignore'] and ignore.match(link_href):
                    continue

                if link_href not in found:
                    found.add(link_href)
                    urls.put(link_href)

            file = MemoryFile(r.content)
            file.url = url
            file.mimetype = 'text/html'
            file.size = 0
            file.modified = None

            yield file

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class LifoExecutor:

    def __init__(self, executor, max_workers=None):
        self.executor = executor
        self.max_workers = max_workers or executor._max_workers
        self.queue = LifoQueue()
        self.loop = asyncio.get_event_loop()
        self.sem = asyncio.Semaphore(self.max_workers)

    def submit(self, f, *args):
        future = self.loop.create_future()
        self.queue.put((future, f, args))
        self.loop.create_task(self.run_task())
        return future

    async def run_task(self):
        await self.sem.acquire()
        future, f, args = self.queue.get()
        executor_future = self.loop.run_in_executor(self.executor, f, *args)
        executor_future.add_done_callback(lambda f, ff=future: self.done_callback(future, f))

    def done_callback(self, future, executor_future):
        self.sem.release()
        future.set_result(executor_future.result())

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class CrawlerType1(BaseCrawler):
    def __init__(self, name, start_url, list_of_url, number_of_threads,
                 delay_request=False, max_allowed_errors=3):
        """

        :param name: As usual
        :param start_url: As usual
        :param list_of_url: As usual
        :param number_of_threads: As usual
        :param delay_request: As usual
        :param max_allowed_errors: As usual
        """
        super().__init__(name, start_url, number_of_threads=number_of_threads,
                         delay_request=delay_request,
                         max_err=max_allowed_errors)
        self.url_list = list_of_url
        self.task_queue = LifoQueue()

    def run(self):
        """
        Method called from subclasses to start crawling process
        """
        while True:
            # Crawl cycle starts
            print_util.print_info(
                'Starting new crawl with {0}'.format(
                    self.name
                ),
                Colors.BLACK
            )
            # Add all URLs to task queue
            for url in self.url_list:
                self.task_queue.put(
                    {
                        'type': 0,
                        'url': url,
                        'n_errors': 0
                    }
                )
            # Start all threads
            threads = []
            for n in range(1, self.number_of_threads + 1):
                temp_thread = Thread(
                    target=self.threader,
                    args=(n,)
                )
                threads.append(temp_thread)
                temp_thread.start()
            for temp_thread in threads:
                temp_thread.join()
                # Crawl cycle ends

    def threader(self, thread_id):
        """
        Worker function
        :param thread_id: As usual
        """
        while not self.task_queue.empty():
            task = self.task_queue.get()

            if task['n_errors'] >= self.max_allowed_errors:
                print_util.print_warning(
                    '{0} --> Too many errors in task {1}. Skipping.'.format(
                        thread_id,
                        task
                    )
                )
                continue

            print_util.print_info(
                '{0} --> New task : {1}'.format(
                    thread_id,
                    task
                )
            )

            try:
                if task['type'] == 0:
                    self.get_artists(
                        thread_id,
                        task['url']
                    )
                elif task['type'] == 1:
                    self.get_artist_albums(
                        thread_id,
                        task['url'],
                        task['artist']
                    )
                elif task['type'] == 2:
                    self.get_song(
                        thread_id,
                        task['url'],
                        task['song'],
                        task['album'],
                        task['album_url'],
                        task['artist']
                    )

                print_util.print_info(
                    '{0} --> Task complete : {1}'.format(
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class Marshall(object):
    """ Serialises sending of packets and associates them with their
    corresponding responses.
    """
    _NOTAG = 0xffff

    def __init__(self, socket, maxrequests=1024):
        """
        :param socket:  connection to the server.  The socket will be closed by
            the marshall on shutdown.

        :param maxrequests: sets an upper limit on the number of requests that
            can be sent to the server without receiving a response.  Requests
            made beyond the limit will not be dropped but will instead wait for
            an earlier request to finish.  Maximum possible value is 65535.
        :type maxrequests: unsigned 16bit integer (0 <= maxtag <= 65535)
        """
        # the maximum length of a packet, including headers, that can be sent
        # by the marshall.  Should be set after receiving a version response
        self.max_message_size = 0xffffffff

        self._socket = socket

        # queue of ``(type, request, on_error, on_success, sequential)`` tuples
        # for passing requests to the send thread.  Adding false to the queue
        # will cause the send loop to exit
        self._send_queue = Queue()

        # queue of booleans used as a counter for the number of remaining
        # responses and stop the receive loop blocking on recv if no message is
        # expected
        # True indicates that a request has been sent and to expect a reply
        # False indicates that the receive loop should quit immediately
        self._recv_queue = Queue()

        # stack of available transaction tags that can be assigned to new
        # requests.
        # tags are used to identify the request that a response corresponds to.
        self._tags = LifoQueue(maxsize=maxrequests)
        for tag in range(1, maxrequests):
            self._tags.put(tag)

        # map from transaction tags (uint16) to completion callbacks
        self._callbacks = {}

        # responses to callbacks without tags are dispatched in the same order
        # the as the requests were submitted
        self._sequential_callbacks = Queue()

        self._send_thread = Thread(target=self._send_loop, daemon=True)
        self._send_thread.start()

        self._recv_thread = Thread(target=self._recv_loop, daemon=True)
        self._recv_thread.start()

    def _do_send(self, request_type, request,
                 on_success, on_error,
                 sequential):

        if not sequential:
            # bind the callback to a new tag.
            # self._tags.get() should block until a tag becomes available.
            tag = self._tags.get()
            assert tag not in self._callbacks
            self._callbacks[tag] = (on_success, on_error,)
        else:
            tag = self._NOTAG
            self._sequential_callbacks.put((on_success, on_error,))

        length = len(request) + _header.size
        if length > self.max_message_size:
            raise Exception("packet size exceeds maximum")

        header = _header.pack(length, request_type, tag)

        self._socket.sendall(header)
        self._socket.sendall(request)

    def _send_loop(self):
        """ loop for sending packets
        """
        while True:
            try:
                task = self._send_queue.get()

                # if task is False, shut down
                if not task:
                    log.info("quiting send loop")
                    return

                self._do_send(*task)

                # if nothing went wrong, notify the recv loop that another
                # response message is expected
                self._recv_queue.put(True)

                self._send_queue.task_done()

            except Exception as error:
                log.exception("error in send thread", stack_info=True)
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class UnhandledExceptionHandler(Singleton):
    """
    This class implements functionality to catch and log exceptions in a block of code, and also execute a set of
    teardown handlers intended to shut down the application gracefully and do any desired cleanup. It is implemented
    as a singleton because the teardown handlers can have global effects (e.g., stopping the event loop).

    This class is intended to be used as a context manager:
    >>> unhandled_exception_handler = UnhandledExceptionHandler.singleton()
    >>> with unhandled_exception_handler:
    >>>     # code which may throw an exception goes here!
    """

    HANDLED_EXCEPTION_EXIT_CODE = 1
    EXCEPTION_DURING_TEARDOWN_EXIT_CODE = 2

    def __init__(self):
        super().__init__()
        self._handling_lock = Lock()
        self._teardown_callback_stack = LifoQueue()  # we execute callbacks in the reverse order that they were added
        self._logger = log.get_logger(__name__)
        self._handled_exceptions = Queue()
        self._teardown_callback_raised_exception = False

        # Set up a handler to be called when process receives SIGTERM.
        # Note: this will raise if called on a non-main thread, but we should NOT work around that here. (That could
        # prevent the teardown handler from ever being registered!) Calling code should be organized so that this
        # singleton is only ever initialized on the main thread.
        signal.signal(signal.SIGTERM, self._application_teardown_signal_handler)
        signal.signal(signal.SIGINT, self._application_teardown_signal_handler)

    def add_teardown_callback(self, callback, *callback_args, **callback_kwargs):
        """
        Add a callback to be executed in the event of application teardown.

        :param callback: The method callback to execute
        :type callback: callable
        :param callback_args: args to be passed to the callback function
        :type callback_args: list
        :param callback_kwargs: kwargs to be passed to the callback function
        :type callback_kwargs: dict
        """
        self._teardown_callback_stack.put((callback, callback_args, callback_kwargs))

    def _application_teardown_signal_handler(self, sig, frame):
        """
        A signal handler that will trigger application teardown.

        :param sig: Signal number of the received signal
        :type sig: int
        :param frame: The interrupted stack frame
        :type frame: frame
        """
        signal_names = {
            signal.SIGTERM: 'SIGTERM',
            signal.SIGINT: 'SIGINT',
        }
        self._logger.info('{} signal received. Triggering teardown.', signal_names[sig])
        raise AppTeardown

    def __enter__(self):
        """
        Enables this to be used as a context manager. No special handling is needed on enter.
        """
        pass

    def __exit__(self, exc_type, exc_value, traceback):
        """
        Enables this to be used as a context manager. If an exception was raised during the execution block (inside the
        "with" statement) then exc_value will be set to the exception object.

        There are four situations in which we can go through this method:
        1. Exception, on main thread
            - The exception is logged and in some cases (e.g., SystemExit) may be immediately reraised.
            - Teardown callbacks are executed.
            - Example: A KeyboardInterrupt exception raised because user presses ctrl-c / sends SIGINT signal

        2. Exception, not on main thread
            - The exception is logged and in some cases may be passed to the main thread to be reraised.
            - Teardown callbacks are executed.
            - Example: Any unhandled exception that is raised on a SafeThread

        3. Normal exit, on main thread
            - We check to see if there was an exception that we need to reraise on the main thread. In almost all cases
              we will *not* reraise an exception on the main thread since it has already been logged and teardown
              callbacks have already been executed on the thread that raised the exception.
            - Teardown callbacks are *not* executed.
            - Example: A SystemExit exception raised by sys.exit() is passed from a SafeThread to the main thread to
                       make Python set the exit code.

        4. Normal exit, not on main thread
            - Do nothing! All is well.
        """
        if exc_value:
            # An exception occurred during execution, so run the teardown callbacks. We use a lock here since multiple
            # threads could raise exceptions at the same time and we only want to execute these once.
            with self._handling_lock:
                if not isinstance(exc_value, (SystemExit, AppTeardown, KeyboardInterrupt)):
                    # It is not very useful to log the SystemExit exception since it is raised by sys.exit(), and thus
                    # application exit is completely expected.
                    self._logger.exception('Unhandled exception handler caught exception.')
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class ConnectionPool(object):
    """
    Thread-safe connection pool.

    .. versionadded:: 0.5

    The `size` argument specifies how many connections this pool
    manages. Additional keyword arguments are passed unmodified to the
    :py:class:`happybase.Connection` constructor, with the exception of
    the `autoconnect` argument, since maintaining connections is the
    task of the pool.

    :param int size: the maximum number of concurrently open connections
    :param kwargs: keyword arguments passed to
                   :py:class:`happybase.Connection`
    """
    def __init__(self, size, **kwargs):
        if not isinstance(size, int):
            raise TypeError("Pool 'size' arg must be an integer")

        if not size > 0:
            raise ValueError("Pool 'size' arg must be greater than zero")

        logger.debug(
            "Initializing connection pool with %d connections", size)

        self._lock = threading.Lock()
        self._queue = LifoQueue(maxsize=size)
        self._thread_connections = threading.local()

        connection_kwargs = kwargs
        connection_kwargs['autoconnect'] = False

        for i in xrange(size):
            connection = Connection(**connection_kwargs)
            self._queue.put(connection)

        # The first connection is made immediately so that trivial
        # mistakes like unresolvable host names are raised immediately.
        # Subsequent connections are connected lazily.
        with self.connection():
            pass

    def _acquire_connection(self, timeout=None):
        """Acquire a connection from the pool."""
        try:
            return self._queue.get(True, timeout)
        except Empty:
            raise NoConnectionsAvailable(
                "No connection available from pool within specified "
                "timeout")

    def _return_connection(self, connection):
        """Return a connection to the pool."""
        self._queue.put(connection)

    @contextlib.contextmanager
    def connection(self, timeout=None):
        """
        Obtain a connection from the pool.

        This method *must* be used as a context manager, i.e. with
        Python's ``with`` block. Example::

            with pool.connection() as connection:
                pass  # do something with the connection

        If `timeout` is specified, this is the number of seconds to wait
        for a connection to become available before
        :py:exc:`NoConnectionsAvailable` is raised. If omitted, this
        method waits forever for a connection to become available.

        :param int timeout: number of seconds to wait (optional)
        :return: active connection from the pool
        :rtype: :py:class:`happybase.Connection`
        """

        connection = getattr(self._thread_connections, 'current', None)

        return_after_use = False
        if connection is None:
            # This is the outermost connection requests for this thread.
            # Obtain a new connection from the pool and keep a reference
            # in a thread local so that nested connection requests from
            # the same thread can return the same connection instance.
            #
            # Note: this code acquires a lock before assigning to the
            # thread local; see
            # http://emptysquare.net/blog/another-thing-about-pythons-
            # threadlocals/
            return_after_use = True
            connection = self._acquire_connection(timeout)
            with self._lock:
                self._thread_connections.current = connection

        try:
            # Open connection, because connections are opened lazily.
            # This is a no-op for connections that are already open.
            connection.open()

#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class Layer(QMainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        rec = QApplication.desktop().screenGeometry()
        self.w = rec.width()
        self.h = rec.height()
        self.kv = config.get('Settings', 'keys').split(' ')
        self.ki = 0
        self.keys = self.kv[self.ki]
        self.rect = QRectF(0, 0, self.w, self.h)
        self.shortcuts = []
        self.rects = LifoQueue()
        self.query = ''

        self.resize(self.w, self.h)
        self.setStyleSheet("background:rgba(0,0,0,%s)" % config.getfloat('Settings', 'background_opacity'))
        self.setAttribute(Qt.WA_TranslucentBackground);
        self.setWindowFlags(Qt.FramelessWindowHint);
        view = QGraphicsView(self)
        scene = QGraphicsScene()
        scene.setSceneRect(0, 0, self.w, self.h)
        view.setScene(scene)
        view.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff);
        view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff);

        self.setCentralWidget(view)
        self.scene = scene
        self.drawLines()
        self.esc = QShortcut(QKeySequence('Esc'), self)
        self.esc.activated.connect(lambda: move(*config.get('Settings', 'mouse_home_coords').split(' ')))
        self.back = QShortcut(QKeySequence('Backspace'), self)
        self.back.activated.connect(self.goBack)

    def goBack(self):
        if not self.query:
            return
        self.query = self.query[:-1]
        self.setArea(self.rects.get(), True)

    def setPointer(self, rect):
        move(rect.center().x()+self.pos().x()-10, rect.center().y()+self.pos().y()-10)
        # self.scene.addEllipse(rect.center().x()+self.pos().x()-2, rect.center().y()+self.pos().y()-2, 4, 4, QPen(QColor('blue')), QBrush(QColor('blue')))

    def clickHere(self, rect):
        self.showMinimized()
        time.sleep(0.3)
        click(rect.center().x()+self.pos().x()-10, rect.center().y()+self.pos().y()-10)
        time.sleep(0.3)
        move(*config.get('Settings', 'mouse_home_coords').split(' '))
        sys.exit()

    def getArea(self, k, direction):
        rect = self.rect
        i = self.keys.index(k)
        c = len(self.keys)
        y = rect.topLeft().y()
        x = rect.topLeft().x()

        if direction == 'hor':
            new_rect = QRectF(rect.width()/c*i+x, y, rect.width()/c, rect.height())
        else:
            new_rect = QRectF(x, rect.height()/c*i+y, rect.width(), rect.height()/c)
        return new_rect

    def selectArea(self, hint, rect):
        self.query += hint
        self.setArea(rect)

    def setArea(self, new_rect, back=False):
        if not new_rect:
            return
        if not back:
            self.rects.put(self.rect)
        self.rect = new_rect
        self.drawLines()

    def drawLines(self):
        for sh in self.shortcuts:
            sh.activated.disconnect()
            sh.setKey('')
            sh.setEnabled(False)
        self.shortcuts = []
        self.scene.clear()
        rect = self.rect
        c = len(self.keys)
        direction = 'hor' if rect.width() > rect.height() else 'vert'
        if (direction == 'hor' and rect.width()/c < 60) or (direction == 'vert' and rect.height()/c < 60):
            if c > 2:
                self.ki += 1
        else:
            if c < len(self.kv[0]):
                self.ki -= 1
        self.keys = self.kv[self.ki]
        c = len(self.keys)
        color = QColor(config.get('Settings', 'lines_color'))
        fontColor = QColor(config.get('Settings', 'text_color'))
        pen = QPen(color)
        if rect.width() > 100:
            fsize = rect.width()/c/2
        elif rect.width() > 50:
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class Sandbox(object):
    SUSPEND_TIME = object()
    SUSPEND_INST = object()
    PRIMITIVE = object()
    FUNCTION = object()
    startTime = None
    counter = None
    iterlimit = None
    timelimit = None
    sub = None

    def __init__(self, parent, function, arguments, *, globals_=None, interfaces=()):
        self.parent = parent
        if globals_ is None:
            globals_ = {}
        if parent:
            self.stack = parent.stack
            self.blocks = parent.blocks
            self.frames = parent.frames
            self.globals = parent.globals
            self.interfaces = parent.interfaces
        else:
            self.stack = LifoQueue()
            self.blocks = LifoQueue()
            self.frames = LifoQueue()
            self.globals = globals_
            self.interfaces = interfaces
        self.local_variables = {}
        self.function = function
        self.arguments = arguments
        self.index = 0

    def loadName(self, name):
        if name in self.local_variables:
            return self.local_variables[name]
        closure = self.function.closure
        while closure:
            if name in closure.local_variables:
                return closure.local_variables[name]
            closure = closure.function.closure
        if name in self.globals:
            return self.globals[name]
        raise NameError("%s is not defined" % name)

    def getAttr(self, obj, attr):
        try:
            verifyObject(IAbstractAccessible, obj)
        except:
            print(79, obj, type(obj), attr)
            raise
        for iface in obj.__implemented__.interfaces():
            if issubclass(iface, IAbstractAccessible) and attr in iface:
                if isinstance(iface[attr], (Method,)):
                    return AFunction(getattr(obj, attr))
                return getattr(obj, attr)
        print(82, obj, attr)
        raise AttributeError("%r attribute access denied" % type(obj))

    def storeName(self, name, value):
        self.local_variables[name] = value

    def callFunction(self, function, arguments, keywords=None):
        if keywords is None:
            keywords = {}
        if isinstance(function, AFunction):
            return [function.TYPE, function.function, arguments] + ([keywords] if keywords is not None else [])
        if isinstance(function, Function):
            return [self.FUNCTION, function, arguments]
        if Utils.issubclass(function, IAbstractAccessible):
            return [self.PRIMITIVE, function, arguments, keywords]
        raise TypeError("Function %r not allowed" % function)

    def execute(self, iterlimit, timelimit):
        self.startTime = time.time()
        self.counter = 0
        self.iterlimit = iterlimit
        self.timelimit = timelimit
        if hasattr(self.function, 'arguments') and len(self.arguments) != len(self.function.arguments):
            raise TypeError("%r expects %i arguments, got %i" % (
                self.function,
                len(self.function.arguments),
                len(self.arguments)))
        for idx, i in enumerate(self.function.arguments):
            v = self.arguments[idx]
            if isinstance(i, tuple):
                t = i[1]
                i = i[0]
                i = i.title().replace(' ', '')
                i = i[0].lower() + i[1:]
                if v == 'None':
                    self.local_variables[i] = None
                    continue
                t = self.loadName(t)
                v = self.callFunction(t, [v])
                v = v[1](*v[2], **v[3])

            self.local_variables[i] = v
        while True:
            self.counter += 1
            now = time.time()
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class CrawlerType0(BaseCrawler):
    def __init__(self, name, start_url, list_of_url, number_of_threads,
                 max_err=10, delay_request=False):

        # Constructor for BaseCrawler
        """
        Crawler for the websites of type 0.
        :param list_of_url: List of URLs to start with.
        """
        super().__init__(name, start_url, number_of_threads, max_err,
                         delay_request)

        # Initialize data members
        self.task_queue = LifoQueue()
        self.url_list = list_of_url

    def threader(self, thread_id):
        """
        Worker function.
        :return:
        :param thread_id: Assigned ID of thread.
        """
        while not self.task_queue.empty():  # While there are any tasks

            task = self.task_queue.get()  # Get one of them

            if task['n_errors'] >= self.max_allowed_errors:  # Too many errors
                print_util.print_warning(
                    '{0} --> Too many errors in task {1}. Skipping.'.format(
                        thread_id,
                        task
                    )
                )
                continue

            print_util.print_info(
                '{0} --> New task : {1}'.format(
                    thread_id,
                    task
                )
            )  # Log the task

            try:

                # Call corresponding function
                if task['type'] == 0:
                    self.get_movies(
                        thread_id,
                        task['url']
                    )
                elif task['type'] == 1:
                    self.download_movie(
                        thread_id,
                        task['url'],
                        task['movie']
                    )
                elif task['type'] == 2:
                    self.download_song(
                        thread_id,
                        task['url'],
                        task['song'],
                        task['movie'],
                        task['movie_url']
                    )

                print_util.print_info(
                    '{0} --> Task complete : {1}'.format(
                        thread_id,
                        task
                    ),
                    Colors.GREEN
                )  # Log success

            except Exception as e:  # Some error
                print_util.print_error(
                    '{0} --> Error : {1}'.format(
                        thread_id,
                        e
                    )
                )  # Log it
                task['n_errors'] += 1  # Increment number of errors
                self.task_queue.put(task)  # Put back in queue

    def run(self):
        """
        Function to be called by subclasses to start crawler.
        """
        while True:
            # Crawl cycle start
            print_util.print_info(
                'Starting new crawl with {0}.'.format(
                    self.name
                ),
                Colors.BLACK
            )
            # Add all URLs to task queue
            for url in self.url_list:
                self.task_queue.put(
                    {
                        'type': 0,
#.........这里部分代码省略.........

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class Stream():

    def __init__(self, com, port, maxlength = 100):

        self.maxlength = maxlength
        self._ring_buffer = deque([], maxlength)
        self._in_queue = LifoQueue()
        self._out_queue = Queue()

        self.com = com

        self.port = port
        # First, connect to the stream port
        self.com.connect(port,
                         self._in_queue,
                         self._out_queue,
                         self._ring_buffer,
                         self.on_data)

        self.lock = Lock() # lock for concurrent access to self.subscribers
        self.subscribers = []

    def last(self, n = None):
        """ Returns the latest or the n latest data records read
        from the data stream.

        If called without parameter, returns a single record, as a
        Python object. If no record has been received yet, returns None.

        If called with a parameter n, returns the n latest ones or less
        if less records have been received (in older to most recent order).

        Returns None only if no data has been ever published on the stream.
        """

        # No data yet?
        if not self._ring_buffer:
            return None

        if not n:
            res = self._ring_buffer[0]
        else:
            n = min(n, self.maxlength)
            res = list(self._ring_buffer)[:n] #TODO: optimize that! currently, the whole queue is copied :-/
            res.reverse()

        return res

    def get(self, timeout = None):
        """ Blocks until a record is available from the data stream, and returns it.

        If records are already available, returns the most recent one.

        :param timeout: (default: None) If timeout is a positive number, it
        blocks at most timeout seconds and raises the Empty exception if no
        item was available within that time. If timeout is None, block
        indefinitely.

        """
        # in_queue is a LIFO: get() returns the last inserted element,
        # ie the most recent record.
        res = self._in_queue.get(True, timeout)

        # Clear the queue: we do not need to stack older result.
        pymorselogger.debug("Clearing stream incoming queue")
        with self._in_queue.mutex:
            self._in_queue.queue = []

        # Note that we may loose one records between the get() and the clear()
        # if the other thread reacquires the mutex. It is however not a big
        # issue to loose a record here.

        return res

    def on_data(self, data):
        # Since this method can takes some time (invoking all callback
        # on the new incoming data), we copy the list of callbacks
        # to release quickly the lock on self.subscribers
        with self.lock:
            cbs = self.subscribers[:]

        for cb in cbs:
            cb(data)

    def subscribe(self, cb):
        with self.lock:
            self.subscribers.append(cb)

    def unsubscribe(self, cb):
        with self.lock:
            self.subscribers.remove(cb)

    def publish(self, msg):
        self._out_queue.put(msg)

# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import get [as 别名]
class CrawlerType2(BaseCrawler):
    def __init__(self, name, start_url, list_of_urls, number_of_threads,
                 delayed_request=False, max_allowed_error=10):
        super().__init__(name, start_url, number_of_threads,
                         delay_request=delayed_request,
                         max_err=max_allowed_error)
        self.url_list = list_of_urls
        self.task_queue = LifoQueue()

    def run(self):
        """
        Function to be called by subclasses to start crawler
        """
        while True:
            # Crawl cycle starts
            print_util.print_info(
                'Starting crawl with {0}'.format(
                    self.name
                ),
                Colors.BLACK
            )
            # Add URLs to task queue
            for url in self.url_list:
                self.task_queue.put(
                    {
                        'type': 0,
                        'url': url,
                        'n_errors': 0
                    }
                )
            # Start all threads
            threads = []
            for n in range(1, self.number_of_threads + 1):
                temp_thread = Thread(
                    target=self.threader,
                    args=(n,)
                )
                threads.append(temp_thread)
                temp_thread.start()
            # Wait for threads to finish
            for temp_thread in threads:
                temp_thread.join()
                # Crawl cycle ends

    def threader(self, thread_id):
        """
        Worker function
        :param thread_id: Ass usual
        """
        while not self.task_queue.empty():

            task = self.task_queue.get()
            if task['n_errors'] >= self.max_allowed_errors:
                print_util.print_warning(
                    '{0} --> Too many errors in task {1}. Skipping.'.format(
                        thread_id,
                        task
                    )
                )
                continue

            print_util.print_info(
                '{0} --> New task : {1}'.format(
                    thread_id,
                    task
                )
            )

            try:
                if task['type'] == 0:
                    self.get_artists(
                        thread_id,
                        task['url']
                    )
                elif task['type'] == 1:
                    self.get_artist(
                        thread_id,
                        task['url'],
                        task['artist']
                    )
                elif task['type'] == 2:
                    self.get_songs_from_page(
                        thread_id,
                        task['url'],
                        task['artist']
                    )
                elif task['type'] == 3:
                    self.get_song(
                        thread_id,
                        task['url'],
                        task['song'],
                        task['artist']
                    )
                print_util.print_info(
                    '{0} --> Task complete : {1}'.format(
                        thread_id,
                        task
                    ),
                    Colors.GREEN
                )
#.........这里部分代码省略.........