Java StringUtf8Coder类代码示例

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

    Options options = PipelineOptionsFactory.fromArgs(args).withValidation()
        .as(Options.class);
    options.setRunner(FlinkRunner.class);

    Pipeline p = Pipeline.create(options);

    KafkaIO.Read<byte[], String> kafkaIOReader = KafkaIO.read()
        .withBootstrapServers("192.168.99.100:32771")
        .withTopics(Arrays.asList("beam".split(",")))
        .updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest"))
        .withValueCoder(StringUtf8Coder.of());

    p.apply(kafkaIOReader.withoutMetadata())
        .apply(Values.<String>create())
        .apply(Window.<String>into(
          FixedWindows.of(Duration.standardMinutes(options.getWindowSize()))))
        .apply(new CountWords())
        .apply(MapElements.via(new FormatAsTextFn()))
        .apply("WriteCounts", TextIO.Write.to(options.getOutput()));

    p.run();
  }
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
/** Tests ExtractAndSumScore("user"). */
@Test
@Category(ValidatesRunner.class)
public void testUserScoreSums() throws Exception {

  PCollection<String> input = p.apply(Create.of(GAME_EVENTS).withCoder(StringUtf8Coder.of()));

  PCollection<KV<String, Integer>> output = input
    .apply(ParDo.of(new ParseEventFn()))
    // Extract and sum username/score pairs from the event data.
    .apply("ExtractUserScore", new ExtractAndSumScore("user"));

  // Check the user score sums.
  PAssert.that(output).containsInAnyOrder(USER_SUMS);

  p.run().waitUntilFinish();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void bundleWorkingCoderSucceedsClonesOutput() {
  PCollection<Integer> created = p.apply(Create.of(1, 3).withCoder(VarIntCoder.of()));
  PCollection<KV<String, Integer>> kvs =
      created
          .apply(WithKeys.<String, Integer>of("foo"))
          .setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
  WindowedValue<KV<String, Integer>> fooOne = WindowedValue.valueInGlobalWindow(KV.of("foo", 1));
  WindowedValue<KV<String, Integer>> fooThree =
      WindowedValue.valueInGlobalWindow(KV.of("foo", 3));
  CommittedBundle<KV<String, Integer>> bundle =
      factory.createBundle(kvs).add(fooOne).add(fooThree).commit(Instant.now());

  assertThat(bundle.getElements(), containsInAnyOrder(fooOne, fooThree));
  assertThat(
      bundle.getElements(), not(containsInAnyOrder(theInstance(fooOne), theInstance(fooThree))));
  for (WindowedValue<KV<String, Integer>> foo : bundle.getElements()) {
    assertThat(
        foo.getValue(),
        not(anyOf(theInstance(fooOne.getValue()), theInstance(fooThree.getValue()))));
  }
  assertThat(bundle.getPCollection(), equalTo(kvs));
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and
 * {@link CombineFn}, creating a {@link TriggerStateMachine} from the
 * {@link Trigger} in the {@link WindowingStrategy}.
 */
public static <W extends BoundedWindow, AccumT, OutputT>
    ReduceFnTester<Integer, OutputT, W> combining(
        WindowingStrategy<?, W> strategy,
        CombineFn<Integer, AccumT, OutputT> combineFn,
        Coder<OutputT> outputCoder)
        throws Exception {

  CoderRegistry registry = CoderRegistry.createDefault();
  AppliedCombineFn<String, Integer, AccumT, OutputT> fn =
      AppliedCombineFn.<String, Integer, AccumT, OutputT>withInputCoder(
          combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));

  return combining(
      strategy,
      TriggerStateMachines.stateMachineForTrigger(
          TriggerTranslation.toProto(strategy.getTrigger())),
      combineFn,
      outputCoder);
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void testCommitWithUnderlying() {
  CopyOnAccessInMemoryStateInternals<String> underlying =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);

  StateNamespace namespace = new StateNamespaceForTest("foo");
  StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
  BagState<String> stringBag = underlying.state(namespace, bagTag);
  assertThat(stringBag.read(), emptyIterable());

  stringBag.add("bar");
  stringBag.add("baz");

  internals.commit();
  BagState<String> reReadStringBag = internals.state(namespace, bagTag);
  assertThat(reReadStringBag.read(), containsInAnyOrder("baz", "bar"));

  reReadStringBag.add("spam");

  BagState<String> underlyingState = underlying.state(namespace, bagTag);
  assertThat(underlyingState.read(), containsInAnyOrder("spam", "bar", "baz"));
  assertThat(underlyingState, is(theInstance(stringBag)));
  assertThat(internals.isEmpty(), is(false));
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
private void runPipeline() {
  final List<String> words =
      Arrays.asList("hi there", "hi", "hi sue bob", "hi sue", "", "bob hi");

  final Set<String> expectedCounts =
      ImmutableSet.of("hi: 5", "there: 1", "sue: 2", "bob: 2");

  final PCollection<String> output =
      pipeline
      .apply(Create.of(words).withCoder(StringUtf8Coder.of()))
      .apply(new WordCount.CountWords())
      .apply(MapElements.via(new WordCount.FormatAsTextFn()));

  PAssert.that(output).containsInAnyOrder(expectedCounts);

  pipeline.run();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void testGetWithEmpty() {
  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  StateNamespace namespace = new StateNamespaceForTest("foo");
  StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
  BagState<String> stringBag = internals.state(namespace, bagTag);
  assertThat(stringBag.read(), emptyIterable());

  stringBag.add("bar");
  stringBag.add("baz");
  assertThat(stringBag.read(), containsInAnyOrder("baz", "bar"));

  BagState<String> reReadStringBag = internals.state(namespace, bagTag);
  assertThat(reReadStringBag.read(), containsInAnyOrder("baz", "bar"));
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void testCommitWithAddedUnderlying() {
  CopyOnAccessInMemoryStateInternals<String> underlying =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying);

  internals.commit();

  StateNamespace namespace = new StateNamespaceForTest("foo");
  StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
  BagState<String> stringBag = underlying.state(namespace, bagTag);
  assertThat(stringBag.read(), emptyIterable());

  stringBag.add("bar");
  stringBag.add("baz");

  BagState<String> internalState = internals.state(namespace, bagTag);
  assertThat(internalState.read(), emptyIterable());

  BagState<String> reReadUnderlyingState = underlying.state(namespace, bagTag);
  assertThat(reReadUnderlyingState.read(), containsInAnyOrder("bar", "baz"));
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testDistinct() {
  List<String> strings = Arrays.asList(
      "k1",
      "k5",
      "k5",
      "k2",
      "k1",
      "k2",
      "k3");

  PCollection<String> input =
      p.apply(Create.of(strings)
          .withCoder(StringUtf8Coder.of()));

  PCollection<String> output =
      input.apply(Distinct.<String>create());

  PAssert.that(output)
      .containsInAnyOrder("k1", "k5", "k2", "k3");
  p.run();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Override
protected StateInternals createStateInternals() {
  MemoryStateBackend backend = new MemoryStateBackend();
  try {
    AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend = backend.createKeyedStateBackend(
        new DummyEnvironment("test", 1, 0),
        new JobID(),
        "test_op",
        new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()),
        1,
        new KeyGroupRange(0, 0),
        new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));

    keyedStateBackend.setCurrentKey(
        ByteBuffer.wrap(CoderUtils.encodeToByteArray(StringUtf8Coder.of(), "Hello")));

    return new FlinkStateInternals<>(keyedStateBackend, StringUtf8Coder.of());
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void testClear() throws Exception {
  FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
      key("A"), encode("A1", "A2", "A3")));
  BagUserState<String> userState =
      new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));

  userState.clear();
  userState.append("A1");
  userState.clear();
  userState.asyncClose();

  assertNull(fakeClient.getData().get(key("A")));
  thrown.expect(IllegalStateException.class);
  userState.clear();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
/**
 * Snapshots the state {@code (stateName -> (valueCoder && (namespace -> value)))} for a given
 * {@code keyGroupIdx}.
 *
 * @param keyGroupIdx the id of the key-group to be put in the snapshot.
 * @param out the stream to write to.
 */
public void snapshotKeyGroupState(int keyGroupIdx, DataOutputStream out) throws Exception {
  int localIdx = getIndexForKeyGroup(keyGroupIdx);
  Map<String, Tuple2<Coder<?>, Map<String, ?>>> stateTable = stateTables[localIdx];
  Preconditions.checkState(stateTable.size() <= Short.MAX_VALUE,
      "Too many States: " + stateTable.size() + ". Currently at most "
          + Short.MAX_VALUE + " states are supported");
  out.writeShort(stateTable.size());
  for (Map.Entry<String, Tuple2<Coder<?>, Map<String, ?>>> entry : stateTable.entrySet()) {
    out.writeUTF(entry.getKey());
    Coder coder = entry.getValue().f0;
    InstantiationUtil.serializeObject(out, coder);
    Map<String, ?> map = entry.getValue().f1;
    out.writeInt(map.size());
    for (Map.Entry<String, ?> entry1 : map.entrySet()) {
      StringUtf8Coder.of().encode(entry1.getKey(), out);
      coder.encode(entry1.getValue(), out);
    }
  }
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
public void getExecutionContextDifferentKeysIndependentState() {
  DirectExecutionContext fooContext =
      context.getExecutionContext(createdProducer,
          StructuralKey.of("foo", StringUtf8Coder.of()));

  StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());

  fooContext
      .getStepContext("s1")
      .stateInternals()
      .state(StateNamespaces.global(), intBag)
      .add(1);

  DirectExecutionContext barContext =
      context.getExecutionContext(createdProducer,
          StructuralKey.of("bar", StringUtf8Coder.of()));
  assertThat(barContext, not(equalTo(fooContext)));
  assertThat(
      barContext
          .getStepContext("s1")
          .stateInternals()
          .state(StateNamespaces.global(), intBag)
          .read(),
      emptyIterable());
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
@Category({NeedsRunner.class, UsesTestStream.class})
public void testElementsAtAlmostPositiveInfinity() {
  Instant endOfGlobalWindow = GlobalWindow.INSTANCE.maxTimestamp();
  TestStream<String> stream =
      TestStream.create(StringUtf8Coder.of())
          .addElements(
              TimestampedValue.of("foo", endOfGlobalWindow),
              TimestampedValue.of("bar", endOfGlobalWindow))
          .advanceWatermarkToInfinity();

  FixedWindows windows = FixedWindows.of(Duration.standardHours(6));
  PCollection<String> windowedValues =
      p.apply(stream)
          .apply(Window.<String>into(windows))
          .apply(WithKeys.<Integer, String>of(1))
          .apply(GroupByKey.<Integer, String>create())
          .apply(Values.<Iterable<String>>create())
          .apply(Flatten.<String>iterables());

  PAssert.that(windowedValues)
      .inWindow(windows.assignWindow(endOfGlobalWindow))
      .containsInAnyOrder("foo", "bar");
  p.run();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入依赖的package包/类
@Test
@Category(ValidatesRunner.class)
public void testJustReshuffle() {

  PCollection<KV<String, Integer>> input = pipeline
      .apply(Create.of(ARBITRARY_KVS)
          .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())));

  PCollection<KV<String, Integer>> output = input
      .apply(Reshuffle.<String, Integer>of());

  PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS);

  assertEquals(
      input.getWindowingStrategy(),
      output.getWindowingStrategy());

  pipeline.run();
}