Java StringUtf8Coder.of方法代码示例

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
public void testClear() throws Exception {
  FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
      key("A"), encode("A1", "A2", "A3")));
  BagUserState<String> userState =
      new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));

  userState.clear();
  userState.append("A1");
  userState.clear();
  userState.asyncClose();

  assertNull(fakeClient.getData().get(key("A")));
  thrown.expect(IllegalStateException.class);
  userState.clear();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Override
protected byte[][] getSortedTestData() {
  StringUtf8Coder coder = StringUtf8Coder.of();

  try {
    return new byte[][]{
        CoderUtils.encodeToByteArray(coder, ""),
        CoderUtils.encodeToByteArray(coder, "Lorem Ipsum Dolor Omit Longer"),
        CoderUtils.encodeToByteArray(coder, "aaaa"),
        CoderUtils.encodeToByteArray(coder, "abcd"),
        CoderUtils.encodeToByteArray(coder, "abce"),
        CoderUtils.encodeToByteArray(coder, "abdd"),
        CoderUtils.encodeToByteArray(coder, "accd"),
        CoderUtils.encodeToByteArray(coder, "bbcd")
    };
  } catch (CoderException e) {
    throw new RuntimeException("Could not encode values.", e);
  }
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Override
public void encode(InvoiceGroupingKey value, OutputStream outStream) throws IOException {
  Coder<String> stringCoder = StringUtf8Coder.of();
  stringCoder.encode(value.startDate(), outStream);
  stringCoder.encode(value.endDate(), outStream);
  stringCoder.encode(value.productAccountKey(), outStream);
  stringCoder.encode(value.usageGroupingKey(), outStream);
  stringCoder.encode(value.description(), outStream);
  stringCoder.encode(String.valueOf(value.unitPrice()), outStream);
  stringCoder.encode(value.unitPriceCurrency(), outStream);
  stringCoder.encode(value.poNumber(), outStream);
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
public void testAppend() throws Exception {
  FakeBeamFnStateClient fakeClient = new FakeBeamFnStateClient(ImmutableMap.of(
      key("A"), encode("A1")));
  BagUserState<String> userState =
      new BagUserState<>(fakeClient, "A", StringUtf8Coder.of(), () -> requestForId("A"));
  userState.append("A2");
  userState.append("A3");
  userState.asyncClose();

  assertEquals(encode("A1", "A2", "A3"), fakeClient.getData().get(key("A")));
  thrown.expect(IllegalStateException.class);
  userState.append("A4");
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
  ApexPipelineOptions options = PipelineOptionsFactory.create()
      .as(ApexPipelineOptions.class);
  EmbeddedCollector.RESULTS.clear();
  options.setApplicationName("ReadUnbound");
  options.setRunner(ApexRunner.class);
  Pipeline p = Pipeline.create(options);

  List<String> collection = Lists.newArrayList("1", "2", "3", "4", "5");
  CollectionSource<String> source = new CollectionSource<>(collection, StringUtf8Coder.of());
  p.apply(Read.from(source))
      .apply(ParDo.of(new EmbeddedCollector()));

  ApexRunnerResult result = (ApexRunnerResult) p.run();
  DAG dag = result.getApexDAG();
  DAG.OperatorMeta om = dag.getOperatorMeta("Read(CollectionSource)");
  Assert.assertNotNull(om);
  Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class);

  long timeout = System.currentTimeMillis() + 30000;
  while (System.currentTimeMillis() < timeout) {
    if (EmbeddedCollector.RESULTS.containsAll(collection)) {
      break;
    }
    LOG.info("Waiting for expected results.");
    Thread.sleep(1000);
  }
  Assert.assertEquals(Sets.newHashSet(collection), EmbeddedCollector.RESULTS);
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
public void testCheckpointCoderNulls() throws Exception {
  CheckpointCoder<String> coder = new CheckpointCoder<>(StringUtf8Coder.of());
  Checkpoint<String> emptyCheckpoint = new Checkpoint<>(null, null);
  Checkpoint<String> decodedEmptyCheckpoint = CoderUtils.decodeFromByteArray(
      coder,
      CoderUtils.encodeToByteArray(coder, emptyCheckpoint));
  assertNull(decodedEmptyCheckpoint.getResidualElements());
  assertNull(decodedEmptyCheckpoint.getResidualSource());
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Override
public void translate(Flatten.PCollections<T> transform, TranslationContext context) {
  JavaStream<T> merged = null;
  Set<PCollection<T>> unique = new HashSet<>();
  for (PValue input: context.getInputs().values()) {
    PCollection<T> collection = (PCollection<T>) input;
    JavaStream<T> inputStream = context.getInputStream(collection);
    if (null == merged) {
      merged = inputStream;
    } else {
      // duplicate edges are not allowed in Gearpump graph
      // so we route through a dummy node
      if (unique.contains(collection)) {
        inputStream = inputStream.map(new DummyFunction<T>(), "dummy");
      }

      merged = merged.merge(inputStream, 1, transform.getName());
    }
    unique.add(collection);
  }

  if (null == merged) {
    UnboundedSourceWrapper<String, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
        new ValuesSource<>(Lists.newArrayList("dummy"),
            StringUtf8Coder.of()), context.getPipelineOptions());
    merged = context.getSourceStream(unboundedSourceWrapper);
  }
  context.setOutputStream(context.getOutput(), merged);
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Override
public Coder<String> getOutputCoder() {
  return StringUtf8Coder.of();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
private static GrowthTracker<String, String, Integer> newTracker(
    GrowthState<String, String, Integer> state) {
  return new GrowthTracker<>(
      SerializableFunctions.<String>identity(), StringUtf8Coder.of(), state, never());
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Nullable
@Override
public Coder<String> getDestinationCoder() {
  return StringUtf8Coder.of();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testWriteWithWritableByteChannelFactory() throws Exception {
  Coder<String> coder = StringUtf8Coder.of();
  String outputName = "file.txt";
  ResourceId baseDir =
      FileSystems.matchNewResource(
          Files.createTempDirectory(tempFolder.getRoot().toPath(), "testwrite").toString(), true);

  PCollection<String> input = p.apply(Create.of(Arrays.asList(LINES2_ARRAY)).withCoder(coder));

  final WritableByteChannelFactory writableByteChannelFactory =
      new DrunkWritableByteChannelFactory();
  TextIO.Write write =
      TextIO.write()
          .to(
              baseDir
                  .resolve(outputName, ResolveOptions.StandardResolveOptions.RESOLVE_FILE)
                  .toString())
          .withoutSharding()
          .withWritableByteChannelFactory(writableByteChannelFactory);
  DisplayData displayData = DisplayData.from(write);
  assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "DRUNK"));

  input.apply(write);

  p.run();

  final List<String> drunkElems = new ArrayList<>(LINES2_ARRAY.length * 2 + 2);
  for (String elem : LINES2_ARRAY) {
    drunkElems.add(elem);
    drunkElems.add(elem);
  }
  assertOutputFiles(
      drunkElems.toArray(new String[0]),
      null,
      null,
      1,
      baseDir.resolve(
          outputName + writableByteChannelFactory.getSuggestedFilenameSuffix(),
          ResolveOptions.StandardResolveOptions.RESOLVE_FILE),
      write.inner.getShardTemplate());
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
private SerializedMutationCoder() {
  byteArrayCoder = ByteArrayCoder.of();
  stringCoder = StringUtf8Coder.of();
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Override
protected StateInternals createStateInternals() {
  KeyedStateBackend keyedStateBackend =
      getKeyedStateBackend(2, new KeyGroupRange(0, 1));
  return new FlinkKeyGroupStateInternals<>(StringUtf8Coder.of(), keyedStateBackend);
}
 

import org.apache.beam.sdk.coders.StringUtf8Coder; //导入方法依赖的package包/类
@Test
public void shouldWriteAndReadSnapshotForConcreteClassCoder() throws Exception {
  Coder<String> concreteClassCoder = StringUtf8Coder.of();
  testWriteAndReadConfigSnapshot(concreteClassCoder);
}