Java Config类代码示例

import org.apache.samza.config.Config; //导入依赖的package包/类
@BeforeClass
public static void initTest() throws IOException {
    InputStream inputStream = new FileInputStream("src/test/resources/streaming.properties");
    properties.load(inputStream);

    config = mock(Config.class);
    when(config.getList("redborder.stores", Collections.<String>emptyList())).thenReturn(stores);
    String storesListAsString = properties.getProperty("redborder.stores");

    for (String store : storesListAsString.split(",")) {
        List<String> keys = Arrays.asList(properties.getProperty("redborder.stores." + store + ".keys").split(","));
        String storeOverwriteStr = properties.getProperty("redborder.stores." + store + ".overwrite");
        boolean storeOverwrite = (storeOverwriteStr == null || storeOverwriteStr.equals("true"));

        when(config.getList("redborder.stores." + store + ".keys", Collections.singletonList("key"))).thenReturn(keys);
        when(config.getBoolean("redborder.stores." + store + ".overwrite", true)).thenReturn(storeOverwrite);
        stores.add(store);
    }

    context = new MockTaskContext();
    storeManager = new StoreManager(config, context);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@BeforeClass
public static void initTest() throws IOException {
    InputStream inputStream = new FileInputStream("src/test/resources/streaming.properties");
    properties.load(inputStream);

    config = mock(Config.class);
    when(config.getList("redborder.stores", Collections.<String>emptyList())).thenReturn(stores);
    String storesListAsString = properties.getProperty("redborder.stores");

    for (String store : storesListAsString.split(",")) {
        List<String> keys = Arrays.asList(properties.getProperty("redborder.stores." + store + ".keys").split(","));
        String storeOverwriteStr = properties.getProperty("redborder.stores." + store + ".overwrite");
        boolean storeOverwrite = (storeOverwriteStr == null || storeOverwriteStr.equals("true"));

        when(config.getList("redborder.stores." + store + ".keys", Collections.singletonList("key"))).thenReturn(keys);
        when(config.getBoolean("redborder.stores." + store + ".overwrite", true)).thenReturn(storeOverwrite);
        stores.add(store);
    }


    context = new MockTaskContext();
    storeManager = new StoreManager(config, context);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testBroadcastChain() {
  ApplicationRunner mockRunner = mock(ApplicationRunner.class);
  when(mockRunner.getStreamSpec(eq("input"))).thenReturn(new StreamSpec("input", "input-stream", "input-system"));
  StreamGraphImpl streamGraph = new StreamGraphImpl(mockRunner, mock(Config.class));

  MessageStream<Object> inputStream = streamGraph.getInputStream("input");
  inputStream.filter(mock(FilterFunction.class));
  inputStream.map(mock(MapFunction.class));

  TaskContextImpl mockTaskContext = mock(TaskContextImpl.class);
  when(mockTaskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
  OperatorImplGraph opImplGraph =
      new OperatorImplGraph(streamGraph, mock(Config.class), mockTaskContext, mock(Clock.class));

  InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream("input-system", "input-stream"));
  assertEquals(2, inputOpImpl.registeredOperators.size());
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER));
  assertTrue(inputOpImpl.registeredOperators.stream()
      .anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP));
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
private void convertConfigToCoordinatorMessage(Config config) {
  try {
    for (Map.Entry<String, String> configPair : config.entrySet()) {
      byte[] keyBytes = null;
      byte[] messgeBytes = null;
      if (configPair.getKey().startsWith(CHANGELOGPREFIX)) {
        String[] changelogInfo = configPair.getKey().split(":");
        String changeLogPartition = configPair.getValue();
        SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition));
        keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8");
        messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8");
      } else {
        SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue());
        keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8");
        messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8");
      }
      // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp.
      put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes));
    }
    setIsAtHead(systemStreamPartition, true);
  } catch (Exception e) {
    throw new SamzaException(e);
  }
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
/**
 * Returns a consumer that sends all configs to the coordinator stream.
 *
 * @param config Along with the configs, you can pass checkpoints and changelog stream messages into the stream.
 *               The expected pattern is cp:source:taskname -> ssp,offset for checkpoint (Use sspToString util)
 *               ch:source:taskname -> changelogPartition for changelog
 *               Everything else is processed as normal config
 */
public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {

  if (useCachedConsumer && mockConsumer != null) {
    return mockConsumer;
  }

  String jobName = config.get("job.name");
  String jobId = config.get("job.id");
  if (jobName == null) {
    throw new ConfigException("Must define job.name.");
  }
  if (jobId == null) {
    jobId = "1";
  }
  String streamName = Util.getCoordinatorStreamName(jobName, jobId);
  SystemStreamPartition systemStreamPartition = new SystemStreamPartition(systemName, streamName, new Partition(0));
  mockConsumer = new MockCoordinatorStreamWrappedConsumer(systemStreamPartition, config);
  return mockConsumer;
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testResourceMapWithInvalidTypeFailure() {
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("No enum constant org.apache.hadoop.yarn.api.records.LocalResourceType.INVALIDTYPE");

  Map<String, String> configMap = new HashMap<>();
  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
  configMap.put("yarn.resources.myResource1.local.name", "readme");
  configMap.put("yarn.resources.myResource1.local.type", "invalidType");
  configMap.put("yarn.resources.myResource1.local.visibility", "public");
  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
  yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Override public void init(StreamGraph graph, Config config) {
  MessageStream<PageViewEvent> pageViewEvents =
      graph.getInputStream("pageViewEventStream", new JsonSerdeV2<>(PageViewEvent.class));
  OutputStream<KV<String, PageViewCount>> pageViewEventPerMemberStream =
      graph.getOutputStream("pageViewEventPerMemberStream",
          KVSerde.of(new StringSerde(), new JsonSerdeV2<>(PageViewCount.class)));

  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> foldLeftFn = (m, c) -> c + 1;
  pageViewEvents
      .window(Windows.keyedTumblingWindow(m -> m.memberId, Duration.ofSeconds(10), initialValue, foldLeftFn, null, null)
          .setEarlyTrigger(Triggers.repeat(Triggers.count(5)))
          .setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow")
      .map(windowPane -> KV.of(windowPane.getKey().getKey(), new PageViewCount(windowPane)))
      .sendTo(pageViewEventPerMemberStream);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testLocalityMapWithoutHostAffinity() {
  Config config = new MapConfig(new HashMap<String, String>() {
    {
      put("cluster-manager.container.count", "1");
      put("cluster-manager.container.memory.mb", "512");
      put("cluster-manager.container.retry.count", "1");
      put("cluster-manager.container.retry.window.ms", "1999999999");
      put("cluster-manager.allocator.sleep.ms", "10");
      put("yarn.package.path", "/foo");
      put("task.inputs", "test-system.test-stream");
      put("systems.test-system.samza.factory", "org.apache.samza.system.MockSystemFactory");
      put("systems.test-system.samza.key.serde", "org.apache.samza.serializers.JsonSerde");
      put("systems.test-system.samza.msg.serde", "org.apache.samza.serializers.JsonSerde");
      put("job.host-affinity.enabled", "false");
    }
  });

  this.localityMappings.put("0", new HashMap<String, String>() { {
      put(SetContainerHostMapping.HOST_KEY, "abc-affinity");
    } });
  this.jobModelManager = JobModelManagerTestUtil.getJobModelManagerUsingReadModel(config, 1, mockStreamMetadataCache, mockLocalityManager, server);

  assertEquals(jobModelManager.jobModel().getAllContainerLocality(), new HashMap<String, String>() { { this.put("0", null); } });
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
public HdfsSystemConsumer(String systemName, Config config, HdfsSystemConsumerMetrics consumerMetrics) {
  super(consumerMetrics.getMetricsRegistry());
  hdfsConfig = new HdfsConfig(config);
  readerType = HdfsReaderFactory.getType(hdfsConfig.getFileReaderType(systemName));
  stagingDirectory = hdfsConfig.getStagingDirectory();
  bufferCapacity = hdfsConfig.getConsumerBufferCapacity(systemName);
  numMaxRetires = hdfsConfig.getConsumerNumMaxRetries(systemName);
  readers = new ConcurrentHashMap<>();
  readerRunnableStatus = new ConcurrentHashMap<>();
  isShutdown = false;
  this.consumerMetrics = consumerMetrics;
  cachedPartitionDescriptorMap = CacheBuilder.newBuilder().build(new CacheLoader<String, Map<Partition, List<String>>>() {
      @Override
      public Map<Partition, List<String>> load(String streamName)
        throws Exception {
        Validate.notEmpty(streamName);
        if (StringUtils.isBlank(stagingDirectory)) {
          throw new SamzaException("Staging directory can't be empty. "
              + "Is this not a yarn job (currently hdfs system consumer only works in "
              + "the same yarn environment on which hdfs is running)? " + "Is STAGING_DIRECTORY ("
              + HdfsConfig.STAGING_DIRECTORY() + ") not set (see HdfsConfig.scala)?");
        }
        return HdfsSystemAdmin.obtainPartitionDescriptorMap(stagingDirectory, streamName);
      }
    });
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
/**
 * Static method to load the {@link ApplicationRunner}
 *
 * @param config  configuration passed in to initialize the Samza processes
 * @return  the configure-driven {@link ApplicationRunner} to run the user-defined stream applications
 */
public static ApplicationRunner fromConfig(Config config) {
  try {
    Class<?> runnerClass = Class.forName(config.get(RUNNER_CONFIG, DEFAULT_RUNNER_CLASS));
    if (ApplicationRunner.class.isAssignableFrom(runnerClass)) {
      Constructor<?> constructor = runnerClass.getConstructor(Config.class); // *sigh*
      return (ApplicationRunner) constructor.newInstance(config);
    }
  } catch (Exception e) {
    throw new ConfigException(String.format("Problem in loading ApplicationRunner class %s", config.get(
        RUNNER_CONFIG)), e);
  }
  throw new ConfigException(String.format(
      "Class %s does not extend ApplicationRunner properly",
      config.get(RUNNER_CONFIG)));
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  Map<String, String> configMap = new HashMap<String, String>();
  Set<SystemStreamPartition> ssp = new HashSet<>();
  configMap.put("a", "b");
  Config config = new MapConfig(configMap);
  TaskName taskName = new TaskName("test");
  ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
  TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
  Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
  tasks.put(taskName, taskModel);
  ContainerModel containerModel = new ContainerModel("1", 1, tasks);
  Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
  containerMap.put("1", containerModel);
  jobModel = new JobModel(config, containerMap);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testGetIntermediateStreamWithDefaultDefaultSerde() {
  ApplicationRunner mockRunner = mock(ApplicationRunner.class);
  Config mockConfig = mock(Config.class);
  StreamSpec mockStreamSpec = mock(StreamSpec.class);
  String mockStreamName = "mockStreamName";
  when(mockRunner.getStreamSpec(mockStreamName)).thenReturn(mockStreamSpec);

  StreamGraphImpl graph = new StreamGraphImpl(mockRunner, mockConfig);
  IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl =
      graph.getIntermediateStream(mockStreamName, null);

  assertEquals(graph.getInputOperators().get(mockStreamSpec), intermediateStreamImpl.getOperatorSpec());
  assertEquals(graph.getOutputStreams().get(mockStreamSpec), intermediateStreamImpl.getOutputStream());
  assertEquals(mockStreamSpec, intermediateStreamImpl.getStreamSpec());
  assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde);
  assertTrue(intermediateStreamImpl.getOutputStream().getValueSerde() instanceof NoOpSerde);
  assertTrue(((InputOperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde);
  assertTrue(((InputOperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde() instanceof NoOpSerde);
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
/**
 * Construct a table manager instance
 * @param config the job configuration
 * @param serdes Serde instances for tables
 */
public TableManager(Config config, Map<String, Serde<Object>> serdes) {

  new JavaTableConfig(config).getTableIds().forEach(tableId -> {

      // Construct the table provider
      String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId));

      // Construct the KVSerde
      JavaTableConfig tableConfig = new JavaTableConfig(config);
      KVSerde serde = KVSerde.of(
          serdes.get(tableConfig.getKeySerde(tableId)),
          serdes.get(tableConfig.getValueSerde(tableId)));

      TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory,
          config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + "."));

      addTable(tableSpec);

      logger.info("Added table " + tableSpec.getId());
    });
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testStreamOperatorClose() {
  StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockOp = mock(StreamOperatorSpec.class);
  FlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> txfmFn = mock(FlatMapFunction.class);
  when(mockOp.getTransformFn()).thenReturn(txfmFn);
  Config mockConfig = mock(Config.class);
  TaskContext mockContext = mock(TaskContext.class);

  StreamOperatorImpl<TestMessageEnvelope, TestOutputMessageEnvelope> opImpl =
      new StreamOperatorImpl<>(mockOp, mockConfig, mockContext);

  // ensure that close is not called yet
  verify(txfmFn, times(0)).close();
  opImpl.handleClose();
  // ensure that close is called once inside handleClose()
  verify(txfmFn, times(1)).close();
}
 

import org.apache.samza.config.Config; //导入依赖的package包/类
@Test
public void testKinesisConfigs() {
  Map<String, String> kv = new HashMap<>();
  String system = "kinesis";
  String stream = "kinesis-stream";
  String systemConfigPrefix = String.format("systems.%s.", system);
  String ssConfigPrefix = String.format("systems.%s.streams.%s.", system, stream);

  kv.put("sensitive." + ssConfigPrefix + "aws.secretKey", "secretKey");
  kv.put(systemConfigPrefix + "aws.region", "us-east-1");
  kv.put(ssConfigPrefix + "aws.accessKey", "accessKey");

  Config config = new MapConfig(kv);
  KinesisConfig kConfig = new KinesisConfig(config);

  assertEquals("us-east-1", kConfig.getRegion(system, stream).getName());
  assertEquals("accessKey", kConfig.getStreamAccessKey(system, stream));
  assertEquals("secretKey", kConfig.getStreamSecretKey(system, stream));
}