Python AIOKafkaProducer.send方法代码示例

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        yield from self.wait_topic(producer.client, self.topic)
        with self.assertRaisesRegexp(AssertionError, 'value must be bytes'):
            yield from producer.send(self.topic, 'hello, Kafka!')
        future = yield from producer.send(self.topic, b'hello, Kafka!')
        resp = yield from future
        self.assertEqual(resp.topic, self.topic)
        self.assertTrue(resp.partition in (0, 1))
        self.assertEqual(resp.offset, 0)

        fut = yield from producer.send(self.topic, b'second msg', partition=1)
        resp = yield from fut
        self.assertEqual(resp.partition, 1)

        future = yield from producer.send(self.topic, b'value', key=b'KEY')
        resp = yield from future
        self.assertTrue(resp.partition in (0, 1))

        resp = yield from producer.send_and_wait(self.topic, b'value')
        self.assertTrue(resp.partition in (0, 1))

        yield from producer.stop()
        with self.assertRaises(ProducerClosed):
            yield from producer.send(self.topic, b'value', key=b'KEY')

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        self.add_cleanup(producer.stop)
        with self.assertRaises(TypeError):
            yield from producer.send(self.topic, 'hello, Kafka!', partition=0)
        future = yield from producer.send(
            self.topic, b'hello, Kafka!', partition=0)
        resp = yield from future
        self.assertEqual(resp.topic, self.topic)
        self.assertTrue(resp.partition in (0, 1))
        self.assertEqual(resp.offset, 0)

        fut = yield from producer.send(self.topic, b'second msg', partition=1)
        resp = yield from fut
        self.assertEqual(resp.partition, 1)

        future = yield from producer.send(self.topic, b'value', key=b'KEY')
        resp = yield from future
        self.assertTrue(resp.partition in (0, 1))

        resp = yield from producer.send_and_wait(self.topic, b'value')
        self.assertTrue(resp.partition in (0, 1))

        yield from producer.stop()
        with self.assertRaises(ProducerClosed):
            yield from producer.send(self.topic, b'value', key=b'KEY')

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_with_serializer(self):
        def key_serializer(val):
            return val.upper().encode()

        def serializer(val):
            return json.dumps(val).encode()

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            value_serializer=serializer,
            key_serializer=key_serializer, acks='all',
            max_request_size=1000)
        yield from producer.start()
        key = 'some key'
        value = {'strKey': 23523.443, 23: 'STRval'}
        future = yield from producer.send(self.topic, value, key=key)
        resp = yield from future
        partition = resp.partition
        offset = resp.offset
        self.assertTrue(partition in (0, 1))  # partition

        future = yield from producer.send(self.topic, 'some str', key=key)
        resp = yield from future
        # expect the same partition bcs the same key
        self.assertEqual(resp.partition, partition)
        # expect offset +1
        self.assertEqual(resp.offset, offset + 1)

        value[23] = '*VALUE' * 800
        with self.assertRaises(MessageSizeTooLargeError):
            yield from producer.send(self.topic, value, key=key)

        yield from producer.stop()
        yield from producer.stop()  # shold be Ok

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_error(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            retry_backoff_ms=100,
            linger_ms=5, request_timeout_ms=400)
        yield from producer.start()

        @asyncio.coroutine
        def mocked_send(nodeid, req):
            # RequestTimedOutCode error for partition=0
            return ProduceResponse[0]([(self.topic, [(0, 7, 0), (1, 0, 111)])])

        with mock.patch.object(producer.client, 'send') as mocked:
            mocked.side_effect = mocked_send
            fut1 = yield from producer.send(self.topic, b'text1', partition=0)
            fut2 = yield from producer.send(self.topic, b'text2', partition=1)
            with self.assertRaises(RequestTimedOutError):
                yield from fut1
            resp = yield from fut2
            self.assertEqual(resp.offset, 111)

        @asyncio.coroutine
        def mocked_send_with_sleep(nodeid, req):
            # RequestTimedOutCode error for partition=0
            yield from asyncio.sleep(0.1, loop=self.loop)
            return ProduceResponse[0]([(self.topic, [(0, 7, 0)])])

        with mock.patch.object(producer.client, 'send') as mocked:
            mocked.side_effect = mocked_send_with_sleep
            with self.assertRaises(RequestTimedOutError):
                future = yield from producer.send(
                    self.topic, b'text1', partition=0)
                yield from future
        yield from producer.stop()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_get_offsets(self):
        client = AIOKafkaClient(loop=self.loop, bootstrap_servers=self.hosts)
        yield from client.bootstrap()

        subscription = SubscriptionState("earliest")
        subscription.subscribe(topics=("topic1",))
        coordinator = GroupCoordinator(client, subscription, loop=self.loop, group_id="getoffsets-group")

        yield from self.wait_topic(client, "topic1")
        producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        yield from producer.send("topic1", b"first msg", partition=0)
        yield from producer.send("topic1", b"second msg", partition=1)
        yield from producer.send("topic1", b"third msg", partition=1)
        yield from producer.stop()

        yield from coordinator.ensure_active_group()

        offsets = {
            TopicPartition("topic1", 0): OffsetAndMetadata(1, ""),
            TopicPartition("topic1", 1): OffsetAndMetadata(2, ""),
        }
        yield from coordinator.commit_offsets(offsets)

        self.assertEqual(subscription.all_consumed_offsets(), {})
        subscription.seek(("topic1", 0), 0)
        subscription.seek(("topic1", 1), 0)
        yield from coordinator.refresh_committed_offsets()
        self.assertEqual(subscription.assignment[("topic1", 0)].committed, 1)
        self.assertEqual(subscription.assignment[("topic1", 1)].committed, 2)

        yield from coordinator.close()
        yield from client.close()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
 def test_producer_send_noack(self):
     producer = AIOKafkaProducer(
         loop=self.loop, bootstrap_servers=self.hosts, acks=0)
     yield from producer.start()
     fut1 = yield from producer.send(
         self.topic, b'hello, Kafka!', partition=0)
     fut2 = yield from producer.send(
         self.topic, b'hello, Kafka!', partition=1)
     done, _ = yield from asyncio.wait([fut1, fut2], loop=self.loop)
     for item in done:
         self.assertEqual(item.result(), None)
     yield from producer.stop()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_with_compression(self):
        with self.assertRaises(ValueError):
            producer = AIOKafkaProducer(
                loop=self.loop, compression_type='my_custom')

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts,
            compression_type='gzip')

        yield from producer.start()
        yield from self.wait_topic(producer.client, self.topic)

        # short message will not be compressed
        future = yield from producer.send(
            self.topic, b'this msg is too short for compress')
        resp = yield from future
        self.assertEqual(resp.topic, self.topic)
        self.assertTrue(resp.partition in (0, 1))

        # now message will be compressed
        resp = yield from producer.send_and_wait(
            self.topic, b'large_message-'*100)
        self.assertEqual(resp.topic, self.topic)
        self.assertTrue(resp.partition in (0, 1))
        yield from producer.stop()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def send_messages(self, partition, messages, *, topic=None,
                      timestamp_ms=None, return_inst=False, headers=None):
        topic = topic or self.topic
        ret = []
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        try:
            yield from self.wait_topic(producer.client, topic)

            for msg in messages:
                if isinstance(msg, str):
                    msg = msg.encode()
                elif isinstance(msg, int):
                    msg = str(msg).encode()
                future = yield from producer.send(
                    topic, msg, partition=partition,
                    timestamp_ms=timestamp_ms, headers=headers)
                resp = yield from future
                self.assertEqual(resp.topic, topic)
                self.assertEqual(resp.partition, partition)
                if return_inst:
                    ret.append(resp)
                else:
                    ret.append(msg)
        finally:
            yield from producer.stop()
        return ret

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_leader_notfound(self):
        producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.hosts, request_timeout_ms=200)
        yield from producer.start()
        yield from self.wait_topic(producer.client, self.topic)

        with mock.patch.object(ClusterMetadata, "leader_for_partition") as mocked:
            mocked.return_value = -1
            future = yield from producer.send(self.topic, b"text")
            with self.assertRaises(LeaderNotAvailableError):
                yield from future

        with mock.patch.object(ClusterMetadata, "leader_for_partition") as mocked:
            mocked.return_value = None
            future = yield from producer.send(self.topic, b"text")
            with self.assertRaises(NotLeaderForPartitionError):
                yield from future

        yield from producer.stop()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_with_headers_raise_error(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        self.add_cleanup(producer.stop)

        with self.assertRaises(UnsupportedVersionError):
            yield from producer.send(
                self.topic, b'msg', partition=0,
                headers=[("type", b"Normal")])

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_with_headers(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        self.add_cleanup(producer.stop)

        fut = yield from producer.send(
            self.topic, b'msg', partition=0, headers=[("type", b"Normal")])
        resp = yield from fut
        self.assertEqual(resp.partition, 0)

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_reenque_resets_waiters(self):
        # See issue #409. If reenqueue method does not reset the waiter
        # properly new batches will raise RecursionError.

        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts, linger_ms=1000)
        yield from producer.start()
        self.add_cleanup(producer.stop)

        # 1st step is to force an error in produce sequense and force a
        # reenqueue on 1 batch.
        with mock.patch.object(producer.client, 'send') as mocked:
            send_fut = create_future(self.loop)

            @asyncio.coroutine
            def mocked_func(node_id, request):
                if not send_fut.done():
                    send_fut.set_result(None)
                raise UnknownTopicOrPartitionError()
            mocked.side_effect = mocked_func

            fut = yield from producer.send(
                self.topic, b'Some MSG', partition=0)
            yield from send_fut
            # 100ms backoff time
            yield from asyncio.sleep(0.11, loop=self.loop)
        self.assertFalse(fut.done())
        self.assertTrue(producer._message_accumulator._batches)

        # Then we add another msg right after the reenqueue. As we use
        # linger_ms `_sender_routine` will be locked for some time after we
        # reenqueue batch, so this add will be forced to wait a longer time.
        # If drain_waiter is broken it will end up with a RecursionError.
        fut2 = yield from producer.send(self.topic, b'Some MSG 2', partition=0)

        yield from fut2
        self.assertTrue(fut.done())
        self.assertTrue(fut2.done())
        msg1 = yield from fut
        msg2 = yield from fut2

        # The order should be preserved
        self.assertLess(msg1.offset, msg2.offset)

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_with_compression(self):
        with self.assertRaises(ValueError):
            producer = AIOKafkaProducer(loop=self.loop, compression_type="my_custom")

        producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.hosts, compression_type="gzip")

        yield from producer.start()
        yield from self.wait_topic(producer.client, self.topic)

        future = yield from producer.send(self.topic, b"this msg is compressed by client")
        resp = yield from future
        self.assertEqual(resp.topic, self.topic)
        self.assertTrue(resp.partition in (0, 1))
        yield from producer.stop()

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_flush_test(self):
        producer = AIOKafkaProducer(
            loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()
        self.add_cleanup(producer.stop)

        fut1 = yield from producer.send("producer_flush_test", b'text1')
        fut2 = yield from producer.send("producer_flush_test", b'text2')
        self.assertFalse(fut1.done())
        self.assertFalse(fut2.done())

        yield from producer.flush()
        self.assertTrue(fut1.done())
        self.assertTrue(fut2.done())

# 需要导入模块: from aiokafka.producer import AIOKafkaProducer [as 别名]
# 或者: from aiokafka.producer.AIOKafkaProducer import send [as 别名]
    def test_producer_send_timeout(self):
        producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.hosts)
        yield from producer.start()

        @asyncio.coroutine
        def mocked_send(nodeid, req):
            raise KafkaTimeoutError()

        with mock.patch.object(producer.client, "send") as mocked:
            mocked.side_effect = mocked_send

            fut1 = yield from producer.send(self.topic, b"text1")
            fut2 = yield from producer.send(self.topic, b"text2")
            done, _ = yield from asyncio.wait([fut1, fut2], loop=self.loop)
            for item in done:
                with self.assertRaises(KafkaTimeoutError):
                    item.result()