Python pools.Pool类代码示例

 def __init__(self, server, size, connect_timeout):
     Pool.__init__(self, max_size=size)
     self.server = server
     self._connect_timeout = connect_timeout
     self._parent_class_getter = super(MemcacheConnPool, self).get
     try:
         # call the patched .get() if eventlet is older than 0.9.17
         if StrictVersion(eventlet_version) < StrictVersion('0.9.17'):
             self._parent_class_getter = self._upstream_fixed_get
     except ValueError:
         # "invalid" version number or otherwise error parsing version
         pass

 def _setup_dbs(self):
     for db in 'zodb', 'zodb_session':
         options = copy.deepcopy(self.options.__dict__)
         options['zodb_db'] = db
         connectionFactory = getUtility(IZodbFactoryLookup).get()
         self._dbs[db], _ = connectionFactory.getConnection(**options)
     self._pool = Pool(create=self._create, max_size=20)

 def __init__(self, host='localhost', port=8125, ipv6=False, prefix=None,
              maxudpsize=512):
     """Create a new client."""
     fam = socket.AF_INET6 if ipv6 else socket.AF_INET
     family, _, _, _, addr = socket.getaddrinfo(host, port, fam, socket.SOCK_DGRAM)[0]
     self._addr = addr
     self._pool = Pool(max_size=100, min_size=0, create=lambda: socket.socket(family, socket.SOCK_DGRAM))
     self._prefix = prefix
     self._maxudpsize = maxudpsize

    def put(self, connection):
        if self.current_size > self.max_size:
            self.current_size -= 1
            # close the connection if needed
            if connection.sock is not None:
                connection.close()
            return

        try:
            response = connection.getresponse()
            response.close()
        except httplib.ResponseNotReady:
            pass
        except:
            connection.close()
            connection = self.create()
            
        if connection.sock is None:
            connection = self.create()
            
        Pool.put(self, connection)

    def __init__(
            self, controllercls, connection_factory, exchange, topic,
            pool=None, poolsize=1000):
        self.nodeid = UIDGEN()

        self.max_workers = poolsize
        if pool is None:
            self.procpool = GreenPool(size=poolsize)
        else:
            self.procpool = pool

        self.controller = controllercls()
        self.service = self.controller
        self.topic = topic
        self.greenlet = None
        self.consume_ready = Event()

        node_topic = "{}.{}".format(self.topic, self.nodeid)
        self.nova_queues = [
            entities.get_topic_queue(exchange, topic),
            entities.get_topic_queue(exchange, node_topic),
            entities.get_fanout_queue(topic), ]

        self._channel = None
        self._consumers = None

        self.connection = connection_factory()
        self.connection_factory = connection_factory

        inject_dependencies(self.controller, self)

        self._connection_pool = Pool(
            max_size=self.procpool.size,
            create=connection_factory
        )

        self.workers = set()
        self._pending_ack_messages = []
        self._pending_requeue_messages = []
        self._do_cancel_consumers = False
        self._consumers_cancelled = Event()

        self._timers = list(get_timers(self.controller))

class StatsClient(StatsClientBase):
    """A client for statsd."""

    def __init__(self, host='localhost', port=8125, ipv6=False, prefix=None,
                 maxudpsize=512):
        """Create a new client."""
        fam = socket.AF_INET6 if ipv6 else socket.AF_INET
        family, _, _, _, addr = socket.getaddrinfo(host, port, fam, socket.SOCK_DGRAM)[0]
        self._addr = addr
        self._pool = Pool(max_size=100, min_size=0, create=lambda: socket.socket(family, socket.SOCK_DGRAM))
        self._prefix = prefix
        self._maxudpsize = maxudpsize

    def _send(self, data):
        """Send data to statsd."""
        try:
            with self._pool.item() as _sock:
                _sock.sendto(data.encode('ascii'), self._addr)
        except socket.error:
            # No time for love, Dr. Jones!
            pass

    def pipeline(self):
        return Pipeline(self)

def generate_report(conf, policy_name=None):
    global json_output
    json_output = config_true_value(conf.get('dump_json', 'no'))
    if policy_name is None:
        policy = POLICIES.default
    else:
        policy = POLICIES.get_by_name(policy_name)
        if policy is None:
            exit('Unable to find policy: %s' % policy_name)
    if not json_output:
        print('Using storage policy: %s ' % policy.name)

    swift_dir = conf.get('swift_dir', '/etc/swift')
    retries = int(conf.get('retries', 5))
    concurrency = int(conf.get('concurrency', 25))
    endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
    region_name = str(conf.get('region_name', ''))
    container_report = config_true_value(conf.get('container_report', 'yes'))
    object_report = config_true_value(conf.get('object_report', 'yes'))
    if not (object_report or container_report):
        exit("Neither container or object report is set to run")
    user_domain_name = str(conf.get('user_domain_name', ''))
    project_domain_name = str(conf.get('project_domain_name', ''))
    project_name = str(conf.get('project_name', ''))
    insecure = config_true_value(conf.get('keystone_api_insecure', 'no'))

    coropool = GreenPool(size=concurrency)

    os_options = {'endpoint_type': endpoint_type}
    if user_domain_name:
        os_options['user_domain_name'] = user_domain_name
    if project_domain_name:
        os_options['project_domain_name'] = project_domain_name
    if project_name:
        os_options['project_name'] = project_name
    if region_name:
        os_options['region_name'] = region_name

    url, token = get_auth(conf['auth_url'], conf['auth_user'],
                          conf['auth_key'],
                          auth_version=conf.get('auth_version', '1.0'),
                          os_options=os_options,
                          insecure=insecure)
    account = url.rsplit('/', 1)[1]
    connpool = Pool(max_size=concurrency)
    connpool.create = lambda: SimpleClient(
        url=url, token=token, retries=retries)

    container_ring = Ring(swift_dir, ring_name='container')
    object_ring = Ring(swift_dir, ring_name=policy.ring_name)

    output = {}
    if container_report:
        output['container'] = container_dispersion_report(
            coropool, connpool, account, container_ring, retries,
            conf.get('partitions'), policy)
    if object_report:
        output['object'] = object_dispersion_report(
            coropool, connpool, account, object_ring, retries,
            conf.get('partitions'), policy)

    return output

 def __init__(self, mocks):
     Pool.__init__(self, max_size=2)
     self.mocks = mocks
     # setting this for the eventlet workaround in the MemcacheConnPool
     self._parent_class_getter = super(memcached.MemcacheConnPool, self).get

 def __init__(self, mocks):
     Pool.__init__(self, max_size=2)
     self.mocks = mocks

 def __init__(self, uri, use_proxy=False, min_size=0, max_size=4):
     self.uri = uri
     self.use_proxy = use_proxy
     Pool.__init__(self, min_size, max_size)

try:
    import yajl as json
except ImportError:
    import json

import socket
from urllib import urlencode

from heroshi import get_logger
log = get_logger("api")
from heroshi.conf import settings
from heroshi.error import ApiError


manager_connections = Pool(max_size=2)
manager_connections.create = lambda: httplib2.Http(timeout=20)


def request_manager(resource, method, data=None, headers=None):
    use_headers = {
        'User-Agent': settings.identity['user_agent'],
        'X-Heroshi-Auth': settings.api_key,
        'Expect': '', # a try to fix result: 100 not-ok problem
    }
    if headers is not None:
        use_headers.update(headers)

    # concat URIs carefully
    base_url = settings.manager_url
    url = base_url.strip('/') + resource

 def __init__(self):
     alarm_rpc_connection = lambda : RpcConnection(True)
     self.metric_pool = Pool(create=RpcConnection, max_size=500)
     self.alarm_pool = Pool(create=alarm_rpc_connection, max_size=20)

class RemoteProcedureCall(object):
    def __init__(self):
        alarm_rpc_connection = lambda : RpcConnection(True)
        self.metric_pool = Pool(create=RpcConnection, max_size=500)
        self.alarm_pool = Pool(create=alarm_rpc_connection, max_size=20)
        
        
    def read_msg(self):
        msg = 'AMQP Connection is closed %d time(s)... retrying.'
        max_retries = 5
        with self.metric_pool.item() as conn:
            for i in range(max_retries + 1):
                try:
                    frame, header, body = conn.channel.basic_get(
                                                    queue='metric_queue')
                    if frame:
                        conn.channel.basic_ack(delivery_tag=frame.delivery_tag)
                    return frame, body
                
                except (AMQPConnectionError, AMQPChannelError, 
                        ConnectionClosed):
                    if i < max_retries:
                        conn.connect()
                        LOG.warn(_(msg) % i)
                        time.sleep(2 * i)
                    else:
                        raise


    def send_msg(self, message_id, body):
        """
        Args:
            message_id: int
                ex) PUT_METRIC_DATA_MSG_ID (0x0001)
                    PUT_METRIC_ALARM_MSG_ID (0x0002)
                    ...
            body: dict object (will be converted into json format)
            
        """

        def publish(body, properties, use_metric_pool=True):
            msg = 'AMQP Connection is closed %d time(s)... retrying.'
            max_retries = 5
            mq_pool = self.metric_pool if use_metric_pool else self.alarm_pool
            
            with mq_pool.item() as conn:
                for i in range(max_retries + 1):
                    try:
                        return conn.channel.basic_publish(exchange='',
                                           routing_key='metric_queue',
                                           body=body, properties=properties)
                    except ConnectionClosed:
                        if i < max_retries:
                            conn.connect()
                            LOG.warn(_(msg) % i)
                            time.sleep(2 * i)
                        else:
                            raise


        if type(message_id) is not int:
            raise RpcInvokeException()
        
        message_uuid = str(uuid.uuid4()) 
        body.setdefault('message_id', message_id)
        body.setdefault('message_uuid', message_uuid)
        
        properties=pika.BasicProperties(delivery_mode=2)
        use_metric_pool = (message_id == PUT_METRIC_DATA_MSG_ID)
        publish(json.dumps(body), properties, use_metric_pool)
            
        LOG.info(_("send_msg - id(%03d), %s"), message_id, message_uuid)
        LOG.debug(_("send_msg - body(%s)"), str(body))

 def __init__(self, server, size, connect_timeout):
     Pool.__init__(self, max_size=size)
     self.server = server
     self._connect_timeout = connect_timeout

 def __init__(self, min_size = 0, max_size = 4, *args, **kwargs):
     self._args = args
     self._kwargs = kwargs
     Pool.__init__(self, min_size, max_size)