Java RpcClient类代码示例

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@Override
protected RpcClient initializeRpcClient(Properties props) {
  // Only one thread is enough, since only one sink thread processes
  // transactions at any given time. Each sink owns its own Rpc client.
  props.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECTION_POOL_SIZE,
                    String.valueOf(1));
  boolean enableKerberos = Boolean.parseBoolean(
      props.getProperty(RpcClientConfigurationConstants.KERBEROS_KEY, "false"));
  if (enableKerberos) {
    return SecureRpcClientFactory.getThriftInstance(props);
  } else {
    props.setProperty(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
                      RpcClientFactory.ClientType.THRIFT.name());
    return RpcClientFactory.getInstance(props);
  }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@Test
public void testFailure() throws Exception {
  try {

    StagedInstall.getInstance().startAgent(
        "rpccagent", CONFIG_FILE_PRCCLIENT_TEST);
    StagedInstall.waitUntilPortOpens("localhost", 12121, 20000);
    RpcClient client = RpcClientFactory.getDefaultInstance(
        "localhost", 12121);
    String[] text = {"foo", "bar", "xyz", "abc"};
    for (String str : text) {
      client.append(EventBuilder.withBody(str.getBytes()));
    }

    // Stop the agent
    StagedInstall.getInstance().stopAgent();

    // Try sending the event which should fail
    try {
      client.append(EventBuilder.withBody("test".getBytes()));
      Assert.fail("EventDeliveryException expected but not raised");
    } catch (EventDeliveryException ex) {
      System.out.println("Attempting to close client");
      client.close();
    }
  } finally {
    if (StagedInstall.getInstance().isRunning()) {
      StagedInstall.getInstance().stopAgent();
    }
  }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@Test
public void testRpcClient() throws Exception {
  StagedInstall.waitUntilPortOpens("localhost", 12121, 20000);
  RpcClient client = RpcClientFactory.getDefaultInstance("localhost", 12121);
  String[] text = {"foo", "bar", "xyz", "abc"};
  for (String str : text) {
    client.append(EventBuilder.withBody(str.getBytes()));
  }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
private synchronized RpcClient createClient() {
  if (client == null) {
    loggingContext.addInfo("Creating a new Flume Client with properties: " + connectionProps);
    try {
      client = RpcClientFactory.getInstance(connectionProps);
    } catch ( Exception e ) {
      loggingContext.addError(e.getLocalizedMessage(), e);
    }
  }

  return client;
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
private synchronized RpcClient createClient() {
  if (client == null) {
    logger.info("Creating a new Flume Client with properties: " + connectionProps);
    try {
      client = RpcClientFactory.getInstance(connectionProps);
    } catch ( Exception e ) {
      logger.error(e.getLocalizedMessage(), e);
    }
  }

  return client;
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
/**
 * Create new instance of <code>AvroAsyncRpcClient</code>.
 *
 * @param starterProp the properties of starter
 * @param numberOfClientThreads is number of client's threads
 */
public AvroAsyncRpcClient(Properties starterProp, int numberOfClientThreads) {
  clientQueue = new ArrayBlockingQueue<RpcClient>(numberOfClientThreads);

  for (int i = 0; i < numberOfClientThreads; i++) {
    RpcClient client = RpcClientFactory.getInstance(starterProp);
    clientQueue.add(client);
  }

  LOG.info("Number of Threads:" + numberOfClientThreads);
  executorService = MoreExecutors
      .listeningDecorator(Executors.newFixedThreadPool(numberOfClientThreads));
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
/**
 * Async append event to RPC client, return listenable future.
 *
 * @param event to adding by RPC
 * @return listenable future
 */
public ListenableFuture<AppendAsyncResultPojo> appendAsync(final Event event)
    throws EventDeliveryException {
  ListenableFuture<AppendAsyncResultPojo> future = executorService.submit(
      new Callable<AppendAsyncResultPojo>() {
        public AppendAsyncResultPojo call() throws Exception {
          RpcClient client = clientQueue.poll();
          client.append(event);
          clientQueue.add(client);
          return new AppendAsyncResultPojo(true, event);
        }
      });
  return future;
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
/**
 * Async append some events to RPC client, return listenable future.
 *
 * @param events to adding by RPC
 * @return listenable future
 */
public ListenableFuture<AppendBatchAsyncResultPojo> appendBatchAsync(final List<Event> events)
    throws EventDeliveryException {
  ListenableFuture<AppendBatchAsyncResultPojo> future = executorService
      .submit(new Callable<AppendBatchAsyncResultPojo>() {
        public AppendBatchAsyncResultPojo call() throws Exception {
          RpcClient client = clientQueue.poll();
          client.appendBatch(events);
          clientQueue.add(client);
          return new AppendBatchAsyncResultPojo(true, events);
        }
      });

  return future;
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
/**
 * There is a very good chance that this will always return the first agent even if it isn't available.
 * @param agents The list of agents to choose from
 * @return The FlumeEventAvroServer.
 */

private RpcClient connect(final Agent[] agents, int retries, final int connectTimeout, final int requestTimeout) {
    try {
        final Properties props = new Properties();

        props.put("client.type", agents.length > 1 ? "default_failover" : "default");

        int count = 1;
        final StringBuilder sb = new StringBuilder();
        for (final Agent agent : agents) {
            if (sb.length() > 0) {
                sb.append(" ");
            }
            final String hostName = "host" + count++;
            props.put("hosts." + hostName, agent.getHost() + ":" + agent.getPort());
            sb.append(hostName);
        }
        props.put("hosts", sb.toString());
        if (batchSize > 0) {
            props.put("batch-size", Integer.toString(batchSize));
        }
        if (retries > 1) {
            if (retries > MAX_RECONNECTS) {
                retries = MAX_RECONNECTS;
            }
            props.put("max-attempts", Integer.toString(retries * agents.length));
        }
        if (requestTimeout >= MINIMUM_TIMEOUT) {
            props.put("request-timeout", Integer.toString(requestTimeout));
        }
        if (connectTimeout >= MINIMUM_TIMEOUT) {
            props.put("connect-timeout", Integer.toString(connectTimeout));
        }
        return RpcClientFactory.getInstance(props);
    } catch (final Exception ex) {
        LOGGER.error("Unable to create Flume RPCClient: {}", ex.getMessage());
        return null;
    }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
/**
 * There is a very good chance that this will always return the first agent even if it isn't available.
 * @param agents The list of agents to choose from
 * @return The FlumeEventAvroServer.
 */
private RpcClient connect(final Agent[] agents, int retries, final int connectTimeoutMillis, final int requestTimeoutMillis) {
    try {
        final Properties props = new Properties();

        props.put("client.type", "default_failover");

        int agentCount = 1;
        final StringBuilder sb = new StringBuilder();
        for (final Agent agent : agents) {
            if (sb.length() > 0) {
                sb.append(' ');
            }
            final String hostName = "host" + agentCount++;
            props.put("hosts." + hostName, agent.getHost() + ':' + agent.getPort());
            sb.append(hostName);
        }
        props.put("hosts", sb.toString());
        if (batchSize > 0) {
            props.put("batch-size", Integer.toString(batchSize));
        }
        if (retries > 1) {
            if (retries > MAX_RECONNECTS) {
                retries = MAX_RECONNECTS;
            }
            props.put("max-attempts", Integer.toString(retries * agents.length));
        }
        if (requestTimeoutMillis >= MINIMUM_TIMEOUT) {
            props.put("request-timeout", Integer.toString(requestTimeoutMillis));
        }
        if (connectTimeoutMillis >= MINIMUM_TIMEOUT) {
            props.put("connect-timeout", Integer.toString(connectTimeoutMillis));
        }
        return RpcClientFactory.getInstance(props);
    } catch (final Exception ex) {
        LOGGER.error("Unable to create Flume RPCClient: {}", ex.getMessage());
        return null;
    }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@VisibleForTesting
RpcClient getUnderlyingClient() {
  return client;
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@Override
protected RpcClient initializeRpcClient(Properties props) {
  logger.info("Attempting to create Avro Rpc client.");
  return RpcClientFactory.getInstance(props);
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
private void run() throws IOException, FlumeException,
    EventDeliveryException {

  EventReader reader = null;

  RpcClient rpcClient;
  if (rpcClientPropsFile != null) {
    rpcClient = RpcClientFactory.getInstance(new File(rpcClientPropsFile));
  } else {
    rpcClient = RpcClientFactory.getDefaultInstance(hostname, port,
        BATCH_SIZE);
  }

  try {
    if (fileName != null) {
      reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
    } else if (dirName != null) {
      reader = new ReliableSpoolingFileEventReader.Builder()
          .spoolDirectory(new File(dirName)).build();
    } else {
      reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
    }

    long lastCheck = System.currentTimeMillis();
    long sentBytes = 0;

    int batchSize = rpcClient.getBatchSize();
    List<Event> events;
    while (!(events = reader.readEvents(batchSize)).isEmpty()) {
      for (Event event : events) {
        event.setHeaders(headers);
        sentBytes += event.getBody().length;
        sent++;

        long now = System.currentTimeMillis();
        if (now >= lastCheck + 5000) {
          logger.debug("Packed {} bytes, {} events", sentBytes, sent);
          lastCheck = now;
        }
      }
      rpcClient.appendBatch(events);
      if (reader instanceof ReliableEventReader) {
        ((ReliableEventReader) reader).commit();
      }
    }
    logger.debug("Finished");
  } finally {
    if (reader != null) {
      logger.debug("Closing reader");
      reader.close();
    }

    logger.debug("Closing RPC client");
    rpcClient.close();
  }
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
@Test
public void testReset() throws Exception {

  setUp();
  Server server = createServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  context.put("reset-connection-interval", String.valueOf("5"));

  sink.setChannel(channel);
  Configurables.configure(sink, context);
  sink.start();
  RpcClient firstClient = sink.getUnderlyingClient();
  Thread.sleep(6000);
  Transaction t = channel.getTransaction();
  t.begin();
  channel.put(EventBuilder.withBody("This is a test", Charset.defaultCharset()));
  t.commit();
  t.close();
  sink.process();
  // Make sure they are not the same object, connection should be reset
  Assert.assertFalse(firstClient == sink.getUnderlyingClient());
  sink.stop();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  context.put("reset-connection-interval", String.valueOf("0"));

  sink.setChannel(channel);
  Configurables.configure(sink, context);
  sink.start();
  firstClient = sink.getUnderlyingClient();
  Thread.sleep(6000);
  // Make sure they are the same object, since connection should not be reset
  Assert.assertTrue(firstClient == sink.getUnderlyingClient());
  sink.stop();

  context.clear();
  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  sink.setChannel(channel);
  Configurables.configure(sink, context);
  sink.start();
  firstClient = sink.getUnderlyingClient();
  Thread.sleep(6000);
  // Make sure they are the same object, since connection should not be reset
  Assert.assertTrue(firstClient == sink.getUnderlyingClient());
  sink.stop();
  server.close();
}
 

import org.apache.flume.api.RpcClient; //导入依赖的package包/类
public RpcClient getRpcClient() {
  return client;
}