Java KV.of方法代码示例

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testCachePipeline() throws Exception {

    final KV<String, String> kv = KV.of("trellis:repository/resource", null);

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/root").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new CacheWriter(dataConfiguration)));

    PAssert.that(pCollection).containsInAnyOrder(asList(kv));

    pipeline.run();
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDirectoryPipeline() throws Exception {

    final KV<String, String> kv = KV.of("trellis:repository/resource", null);

    final Map<String, String> dataConfiguration = singletonMap("foo",
            getClass().getResource("/root").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new CacheWriter(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/dataDirectory").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), false)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final String dataConfiguration = "http://localhost/";

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new EventProcessor(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final Map<String, String> dataConfiguration = singletonMap("repository", "http://localhost/");

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new EventProcessor(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testFormatCounts() throws Exception {
  DoFnTester<KV<Integer, Long>, TableRow> formatCountsFn =
      DoFnTester.of(new FormatCountsFn());
  KV empty[] = {};
  List<TableRow> results = formatCountsFn.processBundle(empty);
  Assert.assertTrue(results.size() == 0);
  KV input[] = { KV.of(3, 0L),
                 KV.of(4, Long.MAX_VALUE),
                 KV.of(5, Long.MIN_VALUE) };
  results = formatCountsFn.processBundle(input);
  Assert.assertEquals(results.size(), 3);
  Assert.assertEquals(results.get(0).get("month"), 3);
  Assert.assertEquals(results.get(0).get("tornado_count"), 0L);
  Assert.assertEquals(results.get(1).get("month"), 4);
  Assert.assertEquals(results.get(1).get("tornado_count"), Long.MAX_VALUE);
  Assert.assertEquals(results.get(2).get("month"), 5);
  Assert.assertEquals(results.get(2).get("tornado_count"), Long.MIN_VALUE);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
private static ArrayList<KV<String, String>> createTestData() {
  String[] scientists = {
    "Einstein",
    "Darwin",
    "Copernicus",
    "Pasteur",
    "Curie",
    "Faraday",
    "Newton",
    "Bohr",
    "Galilei",
    "Maxwell"
  };
  ArrayList<KV<String, String>> data = new ArrayList<>();
  for (int i = 0; i < NUM_ELEMENTS; i++) {
    int index = i % scientists.length;
    KV<String, String> element = KV.of("key", scientists[index]);
    data.add(element);
  }
  return data;
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public KV<K, V> getCurrent() {
  K key = null;
  V value = null;
  try {
    // Transform key if translation function is provided.
    key =
        transformKeyOrValue((T1) recordReader.getCurrentKey(), keyTranslationFunction,
            keyCoder);
    // Transform value if translation function is provided.
    value =
        transformKeyOrValue((T2) recordReader.getCurrentValue(), valueTranslationFunction,
            valueCoder);
  } catch (IOException | InterruptedException e) {
    LOG.error("Unable to read data: " + "{}", e);
    throw new IllegalStateException("Unable to read data: " + "{}", e);
  }
  return KV.of(key, value);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
	TableRow row = c.element();
	String url = row.get("WebResourceHash").toString();
	String lastCountTime = row.get("LastCountTime").toString();
	if (url != null && !url.isEmpty())
	{
		Long l = IndexerPipelineUtils.parseDateToLong(IndexerPipelineUtils.dateTimeFormatYMD_HMS_MSTZ, lastCountTime);
		if (l == null) l = 1L;
		KV<String,Long> kv = KV.of(url, l);
		c.output(kv);
	}
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
	TableRow row = c.element();
	String url = IndexerPipelineUtils.getTableRowStringFieldIfNotNull(row,"Url");
	String processingTime = IndexerPipelineUtils.getTableRowStringFieldIfNotNull(row,"ProcessingTime");
	if (url != null && !url.isEmpty())
	{
		Long l = IndexerPipelineUtils.parseDateToLong(IndexerPipelineUtils.dateTimeFormatYMD_HMS_MSTZ, processingTime);
		if (l == null) l = 1L;
		KV<String,Long> kv = KV.of(url, l);
		c.output(kv);
	}
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public Object structuralValue(KV<K, V> kv) {
  if (consistentWithEquals()) {
    return kv;
  } else {
    return KV.of(getKeyCoder().structuralValue(kv.getKey()),
                 getValueCoder().structuralValue(kv.getValue()));
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
    public KV<BigDecimal, VarAgg> mergeAccumulators(
            Iterable<KV<BigDecimal, VarAgg>> accumulators) {
      BigDecimal variance = new BigDecimal(0);
      long count = 0;
      BigDecimal sum = new BigDecimal(0);

      Iterator<KV<BigDecimal, VarAgg>> ite = accumulators.iterator();
      while (ite.hasNext()) {
        KV<BigDecimal, VarAgg> r = ite.next();

        BigDecimal b = r.getValue().sum;

        count += r.getValue().count;
        sum = sum.add(b);

//        t = ( r.count / count ) * sum - b;
//        d = t^2 * ( ( count / r.count ) / ( count + r.count ) );
        BigDecimal t = new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
                .multiply(sum).subtract(b);
        BigDecimal d = t.pow(2)
                .multiply(new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
                          .divide(new BigDecimal(count)
                                  .add(new BigDecimal(r.getValue().count))), mc);
        variance = variance.add(r.getKey().add(d));
      }

      return KV.of(variance, new VarAgg(count, sum));
    }
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/** Tests trying to call add after calling sort. Should throw an exception. */
public static void testAddAfterSort(Sorter sorter, ExpectedException thrown) throws Exception {
  thrown.expect(IllegalStateException.class);
  thrown.expectMessage(is("Records can only be added before sort()"));
  KV<byte[], byte[]> kv = KV.of(new byte[] {4, 7}, new byte[] {1, 2});
  sorter.add(kv);
  sorter.sort();
  sorter.add(kv);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/** A pair to {@link KV} function . */
static <K, V> Function<Tuple2<K, V>, KV<K, V>> fromPairFunction() {
  return new Function<Tuple2<K, V>, KV<K, V>>() {
    @Override
    public KV<K, V> call(Tuple2<K, V> t2) {
      return KV.of(t2._1(), t2._2());
    }
  };
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public KV<Integer, WindowedValue<?>> decode(InputStream in)
    throws IOException {
  Integer id = VarIntCoder.of().decode(in);
  Coder<WindowedValue<?>> coder = idsToCoders.get(id);
  WindowedValue<?> value = coder.decode(in);
  return KV.<Integer, WindowedValue<?>>of(id, value);
}