Java Client类代码示例

import com.twitter.hbc.core.Client; //导入依赖的package包/类
private  Client getTwitterClient(Properties props, BlockingQueue<String> messageQueue) {

        String clientName = props.getProperty("clientName");
        String consumerKey = props.getProperty("consumerKey");
        String consumerSecret = props.getProperty("consumerSecret");
        String token = props.getProperty("token");
        String tokenSecret = props.getProperty("tokenSecret");
        List<String> searchTerms = Arrays.asList(props.getProperty("searchTerms").split(","));

        Authentication authentication = new OAuth1(consumerKey,consumerSecret,token,tokenSecret);
        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        hosebirdEndpoint.trackTerms(searchTerms);

        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name(clientName)
                .hosts(hosebirdHosts)
                .authentication(authentication)
                .endpoint(hosebirdEndpoint)
                .processor(new StringDelimitedProcessor(messageQueue));

          return clientBuilder.build();

    }
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void run_should_create_and_start_metrics_reporter() {
    HosebirdReader tr = mock(HosebirdReader.class);
    when(tr.parseConfigFile(anyString())).thenReturn(this.config);
    Metrics metrics = mock(Metrics.class);
    when(tr.getMetrics(any(StatsReporter.StatsTracker.class))).thenReturn(metrics);
    Client client = mock(Client.class);
    when(tr.getHosebirdClient(any(LinkedBlockingQueue.class), any(Config.class))).thenReturn(client);
    tr.setLogger(this.logger);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    String[] args = {"FILE"};
    tr.run(args);

    verify(metrics).createAndStartStatsDReporter(any(MetricRegistry.class), eq("G_HOST"), eq(1111), eq("G_PREFIX"), eq(2));
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void run_should_retry_if_readandsend_ends() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    when(tr.getMetrics(any(StatsReporter.StatsTracker.class))).thenCallRealMethod();
    when(tr.parseConfigFile(anyString())).thenReturn(config);
    Client client = mock(Client.class);
    when(tr.getHosebirdClient(any(LinkedBlockingQueue.class), any(Config.class))).thenReturn(client);
    Producer<String, String> producer = mock(Producer.class);
    when(tr.getKafkaProducer(any(KafkaConfig.class))).thenReturn(producer);
    when(tr.getRetry()).thenReturn(true).thenReturn(true).thenReturn(false);

    String[] args = {"1"};
    tr.run(args);

    verify(tr, times(2)).readAndSend(any(LinkedBlockingQueue.class), anyString(), any(Producer.class), any(ReadAndSendPredicate.class));
    verify(tr, times(2)).logClientExitReason(any(Client.class));
    verify(client, times(2)).connect();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void run_should_mark_metric_if_readandsend_ends() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    when(tr.parseConfigFile(anyString())).thenReturn(config);
    when(tr.getMetrics(any(StatsReporter.StatsTracker.class))).thenCallRealMethod();
    Client client = mock(Client.class);
    when(tr.getHosebirdClient(any(LinkedBlockingQueue.class), any(Config.class))).thenReturn(client);
    Producer<String, String> producer = mock(Producer.class);
    when(tr.getKafkaProducer(any(KafkaConfig.class))).thenReturn(producer);
    when(tr.getRetry()).thenReturn(true).thenReturn(true).thenReturn(false);

    String[] args = {"1"};
    tr.run(args);

    assertEquals(2, tr.metrics.disconnected.getCount());
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
/**
 * Logs the reason that the Hosebird client said it was done
 * by examining the exit event.
 * @param client the Hosebird client
 */
@VisibleForTesting
@SuppressWarnings("checkstyle:designforextension")
protected void logClientExitReason(final Client client) {
    BasicClient bc = (BasicClient) client;
    Event e = bc.getExitEvent();

    log.error(
            "Hosebird client stopped: {} {}",
            new Object[]{
            e.getEventType().name(),
            e.getMessage()});
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void shutdownhook_should_send_all_messages_in_buffer_to_kafka() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    tr.metrics.shutdown = mock(Meter.class);
    tr.metrics.read = mock(Meter.class);
    doCallRealMethod().when(tr).addShutdownHook(any(Client.class), any(LinkedBlockingQueue.class), anyString(), any(Producer.class));
    doCallRealMethod().when(tr).readAndSend(any(LinkedBlockingQueue.class), anyString(), any(Producer.class), any(ReadAndSendPredicate.class));

    LinkedBlockingQueue buffer = new LinkedBlockingQueue(5);
    buffer.add("1");
    buffer.add("2");
    buffer.add("3");
    buffer.add("4");
    buffer.add("5");
    KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    tr.addShutdownHook(client, buffer, "TOPIC", producer);

    ArgumentCaptor<Thread> threadCaptor = ArgumentCaptor.forClass(Thread.class);
    verify(tr).runtimeAddShutdownHook(threadCaptor.capture());
    Thread thread = threadCaptor.getValue();
    thread.run();

    try {
        thread.join();
    } catch(Exception e) {
        assertNull(e);
    }

    assertEquals(0, buffer.size());
    verify(producer, times(5)).send(any(ProducerRecord.class), any(Callback.class));
    verify(client).stop();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void shutdownhook_should_end_if_exception_thrown() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    tr.metrics.shutdown = mock(Meter.class);
    doCallRealMethod().when(tr).addShutdownHook(any(Client.class), any(LinkedBlockingQueue.class), anyString(), any(Producer.class));
    doThrow(new RuntimeException("ERROR")).when(tr).readAndSend(any(LinkedBlockingQueue.class), anyString(), any(Producer.class), any(ReadAndSendPredicate.class));

    LinkedBlockingQueue buffer = new LinkedBlockingQueue(1);
    buffer.add("1");
    KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    tr.addShutdownHook(client, buffer, "TOPIC", producer);

    ArgumentCaptor<Thread> threadCaptor = ArgumentCaptor.forClass(Thread.class);
    verify(tr).runtimeAddShutdownHook(threadCaptor.capture());
    Thread thread = threadCaptor.getValue();

    try {
        thread.run();
    } catch(Exception e) {
        assertEquals("ERROR", e.getMessage());
    }

    verify(producer, times(0)).send(any(ProducerRecord.class), any(Callback.class));
    assertEquals(1, buffer.size());
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void shutdownhook_should_set_retry_to_false() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    tr.metrics.shutdown = mock(Meter.class);
    doCallRealMethod().when(tr).addShutdownHook(any(Client.class), any(LinkedBlockingQueue.class), anyString(), any(Producer.class));

    LinkedBlockingQueue buffer = new LinkedBlockingQueue(1);
    buffer.add("1");
    KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    tr.addShutdownHook(client, buffer, "TOPIC", producer);

    ArgumentCaptor<Thread> threadCaptor = ArgumentCaptor.forClass(Thread.class);
    verify(tr).runtimeAddShutdownHook(threadCaptor.capture());
    Thread thread = threadCaptor.getValue();

    thread.run();

    try {
        thread.join();
    } catch(Exception e) {
        assertNull(e);
    }

    verify(tr).setRetry(false);
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void shutdownhook_should_close_kafka_producer() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    tr.metrics.shutdown = mock(Meter.class);
    doCallRealMethod().when(tr).addShutdownHook(any(Client.class), any(LinkedBlockingQueue.class), anyString(), any(Producer.class));

    LinkedBlockingQueue buffer = new LinkedBlockingQueue(1);
    buffer.add("1");
    KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    tr.addShutdownHook(client, buffer, "TOPIC", producer);

    ArgumentCaptor<Thread> threadCaptor = ArgumentCaptor.forClass(Thread.class);
    verify(tr).runtimeAddShutdownHook(threadCaptor.capture());
    Thread thread = threadCaptor.getValue();

    thread.run();

    try {
        thread.join();
    } catch(Exception e) {
        assertNull(e);
    }

    verify(producer).close();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void shutdownhook_should_mark_metric() {
    HosebirdReader tr = mock(HosebirdReader.class);
    tr.setLogger(this.logger);
    Meter meter = mock(Meter.class);
    StatsReporter stats = new StatsReporter();
    tr.metrics = new Metrics(stats.getStatsTracker());
    tr.metrics.shutdown = meter;
    doCallRealMethod().when(tr).addShutdownHook(any(Client.class), any(LinkedBlockingQueue.class), anyString(), any(Producer.class));

    LinkedBlockingQueue buffer = new LinkedBlockingQueue(1);
    buffer.add("1");
    KafkaProducer<String, String> producer = mock(KafkaProducer.class);
    tr.addShutdownHook(client, buffer, "TOPIC", producer);

    ArgumentCaptor<Thread> threadCaptor = ArgumentCaptor.forClass(Thread.class);
    verify(tr).runtimeAddShutdownHook(threadCaptor.capture());
    Thread thread = threadCaptor.getValue();

    thread.run();

    try {
        thread.join();
    } catch(Exception e) {
        assertNull(e);
    }

    verify(meter).mark();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
/**
 * Gets the Gnip client.
 * @param buffer the queue which the client reads into
 * @param readerConfig the Gnip Reader configuration
 * @return the built Gnip client
 */
@SuppressWarnings("checkstyle:designforextension")
@Override
protected Client getHosebirdClient(
          final LinkedBlockingQueue<String> buffer,
          final Config readerConfig) {

    getLogger().info("Building Gnip client");

    GnipConfig config = ((GnipReaderConfig) readerConfig).gnip;

    RealTimeEnterpriseStreamingEndpoint_v2 endpoint =
            new RealTimeEnterpriseStreamingEndpoint_v2(
                    config.account,
                    config.product,
                    config.label);
    BasicAuth auth = new BasicAuth(config.username, config.password);
    LineStringProcessor processor =
            new LineStringProcessor(buffer, readerConfig.hosebird.bufferTimeout);

    return this.getClientBuilder()
            .name("Gnip Reader")
            .hosts(config.host)
            .endpoint(endpoint)
            .authentication(auth)
            .processor(processor)
            .retries(readerConfig.hosebird.retries)
            .build();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
/**
 * @param tweetQueue tweet Queue.
 * @param hosts Hostes.
 * @param endpoint Endpoint.
 * @return Client.
 */
protected Client buildClient(BlockingQueue<String> tweetQueue, HttpHosts hosts, StreamingEndpoint endpoint) {
    Authentication authentication = new OAuth1(oAuthSettings.getConsumerKey(), oAuthSettings.getConsumerSecret(),
        oAuthSettings.getAccessToken(), oAuthSettings.getAccessTokenSecret());

    ClientBuilder builder = new ClientBuilder()
        .name("Ignite-Twitter-Client")
        .hosts(hosts)
        .authentication(authentication)
        .endpoint(endpoint)
        .processor(new StringDelimitedProcessor(tweetQueue));

    return builder.build();
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
public void subscribe(final StatusStreamHandler listener, String... terms) {
	/**
	 * Set up your blocking queues: Be sure to size these properly based on
	 * expected TPS of your stream
	 */
	BlockingQueue<String> msgQueue = new LinkedBlockingQueue<String>(100000);
	BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>(1000);

	/**
	 * Declare the host you want to connect to, the endpoint, and
	 * authentication (basic auth or oauth)
	 */
	Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
	StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
	hosebirdEndpoint.trackTerms(Lists.newArrayList(terms));

	Authentication hosebirdAuth = oAuth();

	ClientBuilder builder = new ClientBuilder().name("Hosebird-Client-01")
			// optional: mainly for the logs
			.hosts(hosebirdHosts).authentication(hosebirdAuth).endpoint(hosebirdEndpoint)
			.processor(new StringDelimitedProcessor(msgQueue)).eventMessageQueue(eventQueue);
	Client client = builder.build();

	final ExecutorService executorService = Executors.newFixedThreadPool(1);
	final Twitter4jStatusClient t4jClient = new Twitter4jStatusClient(client, msgQueue,
			Lists.newArrayList(listener), executorService);
	t4jClient.connect();

	// Call this once for every thread you want to spin off for processing
	// the raw messages.
	// This should be called at least once.
	t4jClient.process(); // required to start processing the messages
}
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
public void collect() throws IOException {

        final BlockingQueue<String> queue = new LinkedBlockingQueue<>(QUEUE_SIZE);
        final StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

        // Set location to collect on
        endpoint.locations(Lists.newArrayList(COLLECTION_LOCATION));

        // add some track terms with this code:
        // endpoint.trackTerms(Lists.newArrayList("twitterapi", "#yolo"));

        final Authentication auth = new OAuth1(consumerKey, consumerSecret, token, secret);

        // Create a new BasicClient. By default gzip is enabled.
        final Client client = new ClientBuilder()
                .hosts(Constants.STREAM_HOST)
                .endpoint(endpoint)
                .authentication(auth)
                .processor(new StringDelimitedProcessor(queue))
                .build();

        // Establish a connection
        client.connect();

        while (!client.isDone()) {
            writeTweetsToFile(getFileToWrite(), queue);
        }

        client.stop();
        log.info("Client stopped, restart needed");
    }
 

import com.twitter.hbc.core.Client; //导入依赖的package包/类
public static void run(String username,
                       String password,
                       String account,
                       String label,
                       String product) throws InterruptedException {
  BlockingQueue<String> queue = new LinkedBlockingQueue<String>(10000);

  BasicAuth auth = new BasicAuth(username, password);

  RealTimeEnterpriseStreamingEndpoint endpoint = new RealTimeEnterpriseStreamingEndpoint(account, product, label);

  // Create a new BasicClient. By default gzip is enabled.
  Client client = new ClientBuilder()
          .name("PowerTrackClient-01")
          .hosts(Constants.ENTERPRISE_STREAM_HOST)
          .endpoint(endpoint)
          .authentication(auth)
          .processor(new LineStringProcessor(queue))
          .build();

  // Establish a connection
  client.connect();

  // Do whatever needs to be done with messages
  for (int msgRead = 0; msgRead < 1000; msgRead++) {
    String msg = queue.take();
    System.out.println(msg);
  }

  client.stop();
}