Python Pool.item方法代码示例

# 需要导入模块: from eventlet.pools import Pool [as 别名]
# 或者: from eventlet.pools.Pool import item [as 别名]
class StatsClient(StatsClientBase):
    """A client for statsd."""

    def __init__(self, host='localhost', port=8125, ipv6=False, prefix=None,
                 maxudpsize=512):
        """Create a new client."""
        fam = socket.AF_INET6 if ipv6 else socket.AF_INET
        family, _, _, _, addr = socket.getaddrinfo(host, port, fam, socket.SOCK_DGRAM)[0]
        self._addr = addr
        self._pool = Pool(max_size=100, min_size=0, create=lambda: socket.socket(family, socket.SOCK_DGRAM))
        self._prefix = prefix
        self._maxudpsize = maxudpsize

    def _send(self, data):
        """Send data to statsd."""
        try:
            with self._pool.item() as _sock:
                _sock.sendto(data.encode('ascii'), self._addr)
        except socket.error:
            # No time for love, Dr. Jones!
            pass

    def pipeline(self):
        return Pipeline(self)

# 需要导入模块: from eventlet.pools import Pool [as 别名]
# 或者: from eventlet.pools.Pool import item [as 别名]
class RemoteProcedureCall(object):
    def __init__(self):
        alarm_rpc_connection = lambda : RpcConnection(True)
        self.metric_pool = Pool(create=RpcConnection, max_size=500)
        self.alarm_pool = Pool(create=alarm_rpc_connection, max_size=20)
        
        
    def read_msg(self):
        msg = 'AMQP Connection is closed %d time(s)... retrying.'
        max_retries = 5
        with self.metric_pool.item() as conn:
            for i in range(max_retries + 1):
                try:
                    frame, header, body = conn.channel.basic_get(
                                                    queue='metric_queue')
                    if frame:
                        conn.channel.basic_ack(delivery_tag=frame.delivery_tag)
                    return frame, body
                
                except (AMQPConnectionError, AMQPChannelError, 
                        ConnectionClosed):
                    if i < max_retries:
                        conn.connect()
                        LOG.warn(_(msg) % i)
                        time.sleep(2 * i)
                    else:
                        raise


    def send_msg(self, message_id, body):
        """
        Args:
            message_id: int
                ex) PUT_METRIC_DATA_MSG_ID (0x0001)
                    PUT_METRIC_ALARM_MSG_ID (0x0002)
                    ...
            body: dict object (will be converted into json format)
            
        """

        def publish(body, properties, use_metric_pool=True):
            msg = 'AMQP Connection is closed %d time(s)... retrying.'
            max_retries = 5
            mq_pool = self.metric_pool if use_metric_pool else self.alarm_pool
            
            with mq_pool.item() as conn:
                for i in range(max_retries + 1):
                    try:
                        return conn.channel.basic_publish(exchange='',
                                           routing_key='metric_queue',
                                           body=body, properties=properties)
                    except ConnectionClosed:
                        if i < max_retries:
                            conn.connect()
                            LOG.warn(_(msg) % i)
                            time.sleep(2 * i)
                        else:
                            raise


        if type(message_id) is not int:
            raise RpcInvokeException()
        
        message_uuid = str(uuid.uuid4()) 
        body.setdefault('message_id', message_id)
        body.setdefault('message_uuid', message_uuid)
        
        properties=pika.BasicProperties(delivery_mode=2)
        use_metric_pool = (message_id == PUT_METRIC_DATA_MSG_ID)
        publish(json.dumps(body), properties, use_metric_pool)
            
        LOG.info(_("send_msg - id(%03d), %s"), message_id, message_uuid)
        LOG.debug(_("send_msg - body(%s)"), str(body))

# 需要导入模块: from eventlet.pools import Pool [as 别名]
# 或者: from eventlet.pools.Pool import item [as 别名]
class ZAuthServer(CmdBase):

    _dbs = {}
    _pool = None

    def _create(self):
        _tm = TransactionManager()
        return DBConnectionContainer(self._dbs['zodb'], 
                self._dbs['zodb_session'],_tm)

    def _setup_dbs(self):
        for db in 'zodb', 'zodb_session':
            options = copy.deepcopy(self.options.__dict__)
            options['zodb_db'] = db
            connectionFactory = getUtility(IZodbFactoryLookup).get()
            self._dbs[db], _ = connectionFactory.getConnection(**options)
        self._pool = Pool(create=self._create, max_size=20)

    @property
    @contextmanager
    def db(self):
        """
        Use this context manager exclusively for database access. It manages
        checking DBConnectionContainers out from the pool, wrapping them in
        a tpool Proxy, and cleaning up transactions afterward.
        """
        with self._pool.item() as conns:
            proxied = tpool.Proxy(conns)
            yield proxied
            proxied.abort()

    def run(self, host, port):
        self._setup_dbs()
        wsgi.server(eventlet.listen((host, port)), self.route)

    def _unauthorized(self, msg, start_response):
        start_response('401 Unauthorized', [('Content-Type', 'text/html')])
        return msg

    def _challenge(self, start_response):
        body = 'Please authenticate'
        headers = [
            ('content-type', 'text/plain'),
            ('content-length', str(len(body))),
            ('WWW-Authenticate', 'Basic realm="%s"' % "ZAuthRealm")]
        start_response('401 Unauthorized', headers)
        return [body]

    def handleLogin(self, env, start_response):
        basic = env.get('HTTP_AUTHORIZATION', None)
        if basic is None:
            return self._challenge(start_response)
        response = WSGIResponse()
        request = HTTPRequest(env['wsgi.input'], env, response)
        with self.db as db:
            db.sync()
            authorization = IAuthorizationTool(db.dmd())
            credentials = authorization.extractCredentials(request)

        login = credentials.get('login', None)
        password = credentials.get('password', None)
        # no credentials to test authentication
        if login is None or password is None:
            return self._unauthorized("Missing Authentication Credentials", start_response)

        # test authentication
        if not authorization.authenticateCredentials(login, password):
            return self._unauthorized( "Failed Authentication", start_response)

        # create the session data
        with self.db as db:
            db.sync()
            db.browser_id_manager().REQUEST = request
            tokenId = db.browser_id_manager().getBrowserId(create=1)
        expires = time.time() + 60 * 20
        token = dict(id=tokenId, expires=expires)
        with self.db as db:
            db.sync()
            session = db.session_data()
            if session.get(tokenId) is None:
                session[tokenId] = token
                db.commit()
        start_response('200 OK', [('Content-Type', 'text/html')])
        return json.dumps(token)

    
    def handleValidate(self, env, start_response):
        queryString = env.get('QUERY_STRING')
        if not queryString:
            return self._unauthorized("Missing Token Id", start_response)

        tokenId = queryString.replace('id=', '')
        if tokenId is None:
            return self._unauthorized("Missing Token Id", start_response)
        token = None
        expired = False
        with self.db as db:
            db.sync()
            token = db.session_data().get(tokenId)
        if token is None:
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.pools import Pool [as 别名]
# 或者: from eventlet.pools.Pool import item [as 别名]
class Service(ConsumerMixin):
    def __init__(
            self, controllercls, connection_factory, exchange, topic,
            pool=None, poolsize=1000):
        self.nodeid = UIDGEN()

        self.max_workers = poolsize
        if pool is None:
            self.procpool = GreenPool(size=poolsize)
        else:
            self.procpool = pool

        self.controller = controllercls()
        self.service = self.controller
        self.topic = topic
        self.greenlet = None
        self.consume_ready = Event()

        node_topic = "{}.{}".format(self.topic, self.nodeid)
        self.nova_queues = [
            entities.get_topic_queue(exchange, topic),
            entities.get_topic_queue(exchange, node_topic),
            entities.get_fanout_queue(topic), ]

        self._channel = None
        self._consumers = None

        self.connection = connection_factory()
        self.connection_factory = connection_factory

        inject_dependencies(self.controller, self)

        self._connection_pool = Pool(
            max_size=self.procpool.size,
            create=connection_factory
        )

        self.workers = set()
        self._pending_ack_messages = []
        self._pending_requeue_messages = []
        self._do_cancel_consumers = False
        self._consumers_cancelled = Event()

        self._timers = list(get_timers(self.controller))

    def start(self):
        self.start_timers()
        # greenlet has a magic attribute ``dead`` - pylint: disable=E1101
        if self.greenlet is not None and not self.greenlet.dead:
            raise RuntimeError()
        self.greenlet = eventlet.spawn(self.run)

    def start_timers(self):
        for timer in self._timers:
            timer.start()

    def get_consumers(self, Consumer, channel):
        nova_consumer = Consumer(
            self.nova_queues, callbacks=[self.on_nova_message, ])

        consume_consumers = get_consumers(
            Consumer, self, self.on_consume_message)

        consumers = [nova_consumer] + list(consume_consumers)

        prefetch_count = self.procpool.size
        for consumer in consumers:
            consumer.qos(prefetch_count=prefetch_count)

        return consumers

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        self._consumers = consumers
        self._channel = channel
        self.consume_ready.send(None)

    def on_consume_end(self, connection, channel):
        self.consume_ready.reset()

    def on_nova_message(self, body, message):
        _log.debug('spawning RPC worker (%d free)', self.procpool.free())

        gt = self.procpool.spawn(self.handle_rpc_message, body)

        gt.link(self.handle_rpc_message_processed, message)
        self.workers.add(gt)

    def on_consume_message(self, consumer_method_config, body, message):
        _log.debug('spawning consume worker (%d free)', self.procpool.free())

        gt = self.procpool.spawn(
            self.handle_consume_message, consumer_method_config, body, message)

        gt.link(self.handle_consume_message_processed)
        self.workers.add(gt)

    def handle_rpc_message(self, body):
        # item is patched on for python with ``with``, pylint can't find it
        # pylint: disable=E1102
        with self._connection_pool.item() as connection:
#.........这里部分代码省略.........