Python eventlet.GreenPile类代码示例

    def make_requests(self, req, ring, part, method, path, headers,
                      query_string=''):
        """
        Sends an HTTP request to multiple nodes and aggregates the results.
        It attempts the primary nodes concurrently, then iterates over the
        handoff nodes as needed.

        :param req: a request sent by the client
        :param ring: the ring used for finding backend servers
        :param part: the partition number
        :param method: the method to send to the backend
        :param path: the path to send to the backend
        :param headers: a list of dicts, where each dict represents one
                        backend request that should be made.
        :param query_string: optional query string to send to the backend
        :returns: a swob.Response object
        """
        start_nodes = ring.get_part_nodes(part)
        nodes = GreenthreadSafeIterator(self.iter_nodes(ring, part))
        pile = GreenPile(len(start_nodes))
        for head in headers:
            pile.spawn(self._make_request, nodes, part, method, path,
                       head, query_string, self.app.logger.thread_locals)
        response = [resp for resp in pile if resp]
        while len(response) < len(start_nodes):
            response.append((HTTP_SERVICE_UNAVAILABLE, '', '', ''))
        statuses, reasons, resp_headers, bodies = zip(*response)
        return self.best_response(req, statuses, reasons, bodies,
                                  '%s %s' % (self.server_type, req.method),
                                  headers=resp_headers)

    def _get_results(self, writers):
        # get the results from writers
        success_chunks = []
        failed_chunks = []

        # we use eventlet GreenPile to read the responses from the writers
        pile = GreenPile(len(writers))

        for writer in writers:
            if writer.failed:
                failed_chunks.append(writer.chunk)
                continue
            pile.spawn(self._get_response, writer)

        def _handle_resp(writer, resp):
            if resp:
                if resp.status == 201:
                    # TODO check checksum in response
                    success_chunks.append(writer.chunk)
                else:
                    writer.failed = True
                    logger.error("Wrong status code from %s (%s)",
                                 writer.chunk, resp.status)
                    writer.chunk['error'] = 'HTTP %s' % resp.status
                    failed_chunks.append(writer.chunk)

        for (writer, resp) in pile:
            _handle_resp(writer, resp)

        quorum = self._check_quorum(success_chunks)

        return success_chunks + failed_chunks, quorum

def direct_get_container_policy_index(container_ring, account_name,
                                      container_name):
    """
    Talk directly to the primary container servers to figure out the storage
    policy index for a given container.

    :param container_ring: ring in which to look up the container locations
    :param account_name: name of the container's account
    :param container_name: name of the container
    :returns: storage policy index, or None if it couldn't get a quorum
    """
    def _eat_client_exception(*args):
        try:
            return direct_head_container(*args)
        except ClientException as err:
            if err.http_status == 404:
                return err.http_headers
        except (Timeout, socket.error):
            pass

    pile = GreenPile()
    part, nodes = container_ring.get_nodes(account_name, container_name)
    for node in nodes:
        pile.spawn(_eat_client_exception, node, part, account_name,
                   container_name)

    headers = [x for x in pile if x is not None]
    if len(headers) < quorum_size(len(nodes)):
        return
    return best_policy_index(headers)

    def get_stream(self):
        range_infos = self._get_range_infos()
        chunk_iter = iter(self.chunks)

        # we use eventlet GreenPool to manage readers
        with utils.ContextPool(self.storage_method.ec_nb_data) as pool:
            pile = GreenPile(pool)
            # we use eventlet GreenPile to spawn readers
            for _j in range(self.storage_method.ec_nb_data):
                pile.spawn(self._get_fragment, chunk_iter, self.storage_method)

            readers = []
            for reader, parts_iter in pile:
                if reader.status in (200, 206):
                    readers.append((reader, parts_iter))
                # TODO log failures?

        # with EC we need at least ec_nb_data valid readers
        if len(readers) >= self.storage_method.ec_nb_data:
            # all readers should return the same Content-Length
            # so just take the headers from one of them
            resp_headers = HeadersDict(readers[0][0].headers)
            fragment_length = int(resp_headers.get('Content-Length'))
            r = [it for reader, it in readers]
            stream = ECStream(self.storage_method, r, range_infos,
                              self.meta_length, fragment_length)
            # start the stream
            stream.start()
            return stream
        else:
            raise exc.OioException("Not enough valid sources to read")

    def _get_writers(self):
        # init writers to the chunks
        pile = GreenPile(len(self.meta_chunk))

        # we use eventlet GreenPile to spawn the writers
        for pos, chunk in enumerate(self.meta_chunk):
            pile.spawn(self._get_writer, chunk)

        writers = [w for w in pile]
        return writers

 def _handle_rawx(self, url, chunks, headers, storage_method, reqid):
     pile = GreenPile(PARALLEL_CHUNKS_DELETE)
     cid = url.get('id')
     for chunk in chunks:
         pile.spawn(self.delete_chunk, chunk, cid, reqid)
     resps = [resp for resp in pile if resp]
     for resp in resps:
         if resp.status != 204:
             self.logger.warn(
                 'failed to delete chunk %s (HTTP %s)',
                 resp.chunk['id'], resp.status)

 def send_to_peers(self, peers, key):
     pile = GreenPile(len(peers))
     # Have the first peer to sync to the local cluster
     sync_to_peer = self.my_cluster
     for peer in peers:
         # create thread per peer and send a request
         pile.spawn(self.send_to_peer, peer, sync_to_peer, key)
         # Have the next peer to sync to the present peer
         sync_to_peer = peer
     # collect the results, if anyone failed....
     response = [resp for resp in pile if resp]
     while len(response) < len(peers):
         response.append((HTTP_SERVICE_UNAVAILABLE, None, None))
     return response

 def frag_iter():
     pile = GreenPile(len(resps))
     while True:
         for resp in resps:
             pile.spawn(_get_frag, resp)
         try:
             with Timeout(self.read_timeout):
                 frag = [frag for frag in pile]
         except (Exception, Timeout):
             # TODO complete error message
             self.logger.exception('ERROR rebuilding')
             break
         if not all(frag):
             break
         rebuilt_frag = self._reconstruct(frag)
         yield rebuilt_frag

 def __init__(self, host='127.0.0.1', port=9410):
     """
     :param host: zipkin collector IP addoress (default '127.0.0.1')
     :param port: zipkin collector port (default 9410)
     """
     self.host = host
     self.port = port
     self.pile = GreenPile(1)
     self._connect()

class ZipkinClient(object):

    def __init__(self, host='127.0.0.1', port=9410):
        """
        :param host: zipkin collector IP addoress (default '127.0.0.1')
        :param port: zipkin collector port (default 9410)
        """
        self.host = host
        self.port = port
        self.pile = GreenPile(1)
        self._connect()

    def _connect(self):
        socket = TSocket.TSocket(self.host, self.port)
        self.transport = TTransport.TFramedTransport(socket)
        protocol = TBinaryProtocol.TBinaryProtocol(self.transport,
                                                   False, False)
        self.scribe_client = scribe.Client(protocol)
        try:
            self.transport.open()
        except TTransport.TTransportException as e:
            warnings.warn(e.message)

    def _build_message(self, thrift_obj):
        trans = TTransport.TMemoryBuffer()
        protocol = TBinaryProtocol.TBinaryProtocolAccelerated(trans=trans)
        thrift_obj.write(protocol)
        return base64.b64encode(trans.getvalue())

    def send_to_collector(self, span):
        self.pile.spawn(self._send, span)

    def _send(self, span):
        log_entry = scribe.LogEntry(CATEGORY, self._build_message(span))
        try:
            self.scribe_client.Log([log_entry])
        except Exception as e:
            msg = 'ZipkinClient send error %s' % str(e)
            warnings.warn(msg)
            self._connect()

    def close(self):
        self.transport.close()

    def _get_put_connections(self, req, nodes, partition, outgoing_headers,
                             policy, expect):
        """
        Establish connections to storage nodes for PUT request
        """
        obj_ring = policy.object_ring
        node_iter = GreenthreadSafeIterator(
            self.iter_nodes_local_first(obj_ring, partition))
        pile = GreenPile(len(nodes))

        for nheaders in outgoing_headers:
            if expect:
                nheaders['Expect'] = '100-continue'
            pile.spawn(self._connect_put_node, node_iter, partition,
                       req.swift_entity_path, nheaders,
                       self.app.logger.thread_locals)

        conns = [conn for conn in pile if conn]

        return conns

    def make_requests(self, req, ring, part, method, path, headers, query_string=""):
        """
        Sends an HTTP request to multiple nodes and aggregates the results.
        It attempts the primary nodes concurrently, then iterates over the
        handoff nodes as needed.

        :param headers: a list of dicts, where each dict represents one
                        backend request that should be made.
        :returns: a swob.Response object
        """
        start_nodes = ring.get_part_nodes(part)
        nodes = self.iter_nodes(part, start_nodes, ring)
        pile = GreenPile(len(start_nodes))
        for head in headers:
            pile.spawn(self._make_request, nodes, part, method, path, head, query_string, self.app.logger.thread_locals)
        response = [resp for resp in pile if resp]
        while len(response) < len(start_nodes):
            response.append((HTTP_SERVICE_UNAVAILABLE, "", ""))
        statuses, reasons, bodies = zip(*response)
        return self.best_response(req, statuses, reasons, bodies, "%s %s" % (self.server_type, req.method))

 def fragment_payload_iter():
     # We need a fragment from each connections, so best to
     # use a GreenPile to keep them ordered and in sync
     pile = GreenPile(len(responses))
     while True:
         for resp in responses:
             pile.spawn(_get_one_fragment, resp)
         try:
             with Timeout(self.node_timeout):
                 fragment_payload = [fragment for fragment in pile]
         except (Exception, Timeout):
             self.logger.exception(
                 _("Error trying to rebuild %(path)s " "policy#%(policy)d frag#%(frag_index)s"),
                 {"path": path, "policy": policy, "frag_index": frag_index},
             )
             break
         if not all(fragment_payload):
             break
         rebuilt_fragment = self._reconstruct(policy, fragment_payload, frag_index)
         yield rebuilt_fragment

    def rebuild(self):
        pile = GreenPile(len(self.meta_chunk))

        nb_data = self.storage_method.ec_nb_data

        headers = {}
        for chunk in self.meta_chunk:
            pile.spawn(self._get_response, chunk, headers)

        resps = []
        for resp in pile:
            if not resp:
                continue
            resps.append(resp)
            if len(resps) >= self.storage_method.ec_nb_data:
                break
        else:
            logger.error('Unable to read enough valid sources to rebuild')
            raise exc.UnrecoverableContent('Unable to rebuild chunk')

        rebuild_iter = self._make_rebuild_iter(resps[:nb_data])
        return rebuild_iter

    def handle_object_delete(self, event):
        """
        Handle object deletion.
        Delete the chunks of the object.
        :param event:
        """
        self.logger.debug('worker "%s" handle object delete', self.name)
        pile = GreenPile(PARALLEL_CHUNKS_DELETE)

        chunks = []

        for item in event.get('data'):
            if item.get('type') == 'chunks':
                chunks.append(item)
        if not len(chunks):
            self.logger.warn('No chunks found in event data')
            return

        def delete_chunk(chunk):
            resp = None
            try:
                with Timeout(CHUNK_TIMEOUT):
                    resp = self.session.delete(chunk['id'])
            except (Exception, Timeout) as e:
                self.logger.warn('error while deleting chunk %s "%s"',
                                 chunk['id'], str(e.message))
            return resp

        for chunk in chunks:
            pile.spawn(delete_chunk, chunk)

        resps = [resp for resp in pile if resp]

        for resp in resps:
            if resp.status_code == 204:
                self.logger.info('deleted chunk %s' % resp.url)
            else:
                self.logger.warn('failed to delete chunk %s' % resp.url)