TypeScript kafka-node.Consumer类代码示例

  kit('publishes to a kafka topic', (done) => {
    let topic = 'gustavTest-publish';

    let consumer = new kafka.Consumer(client, [{
      topic
    }]);

    let obs = new Observable(o => {
      setTimeout(() => o.next('hello'), 15);
    });

    gr.to(topic, obs);

    consumer.on('message', (message) => {
      expect(message.value).to.equal('hello');
      done();
    });
  });

  // two methods, one called on from, other on to
  from(topic: string, offset?: number): Observable<any> {
    let client = this.getClient();

    let consumer = new kafka.Consumer(client, [{
      topic
    }]/* TODO */);

    return new Observable(o => {
      consumer.on('message', m => {
        if (m.value === '__done') {
          return o.complete();
        }
        o.next(m.value)
      });
      consumer.on('error', err => o.error(err));

      return () => consumer.close(() => {});
    });
  }

//.........这里部分代码省略.........
        }
      }

      if (_.isEmpty(restoreEventSetup)) {
        this.logger.warn('No data was setup for the restore process.');
      } else {
        const that = this;
        // Start the restore process
        this.logger.warn('restoring data');

        for (let topicName in restoreEventSetup) {
          const topicSetup: any = restoreEventSetup[topicName];
          const restoreTopic: Topic = topicSetup.topic;
          const topicEvents: any = topicSetup.events;

          // saving listeners for potentially subscribed events on this topic,
          // so they don't get called during the restore process
          const previousEvents: string[] = _.cloneDeep(restoreTopic.subscribed);
          const listenersBackup = new Map<string, Function[]>();
          for (let event of previousEvents) {
            listenersBackup.set(event, (restoreTopic.emitter as EventEmitter).listeners(event));
            await restoreTopic.removeAllListeners(event);
          }

          // const eventNames = _.keys(restoreTopic.events);
          const baseOffset: number = topicSetup.baseOffset;
          const targetOffset: number = (await restoreTopic.$offset(-1)) - 1;
          const ignoreOffsets: number[] = topicSetup.ignoreOffset;
          const eventNames = _.keys(topicEvents);

          this.logger.debug(`topic ${topicName} has current offset ${targetOffset}`);

          const consumerClient = new kafka.KafkaClient({ kafkaHost: kafkaEventsCfg.kafkaHost });
          const consumer = new kafka.Consumer(
            consumerClient,
            [
              { topic: restoreTopic.name, offset: baseOffset }
            ],
            {
              autoCommit: true,
              encoding: 'buffer',
              fromOffset: true
            }
          );

          const drainEvent = (message, done) => {
            const msg = message.value;
            const eventName = message.key.toString();
            const context = _.pick(message, ['offset', 'partition', 'topic']);
            const eventListener = topicEvents[message.key];
            // decode protobuf
            let decodedMsg = that.kafkaEvents.provider.decodeObject(kafkaEventsCfg, eventName, msg);
            decodedMsg = _.pick(decodedMsg, _.keys(decodedMsg)); // preventing protobuf.js special fields
            eventListener(decodedMsg, context, that.config, eventName).then(() => {
              done();
            }).catch((err) => {
              that.logger.error(`Exception caught invoking restore listener for event ${eventName}:`, err);
              done(err);
            });

            if (message.offset >= targetOffset) {
              for (let event of eventNames) {
                restoreTopic.removeAllListeners(event).then(() => { }).catch((err) => {
                  that.logger.error('Error removing listeners after restore', err);
                });
              }

    return new Observable(o => {
      consumer.on('message', m => {
        if (m.value === '__done') {
          return o.complete();
        }
        o.next(m.value)
      });
      consumer.on('error', err => o.error(err));

      return () => consumer.close(() => {});
    });

 return () => consumer.close(() => {});