Java KV.getValue方法代码示例

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
	KV<String, Iterable<InputContent>> kv = c.element();
	String documentHash = kv.getKey();
	Iterable<InputContent> dupes = kv.getValue();
	boolean isFirst = true;
	int groupSize = Iterables.size(dupes);
	for (InputContent ic : dupes) {
	
		// Check if this doc was already processed and stored in BQ
		Map<String,Long> sideInputMap = c.sideInput(alreadyProcessedDocsSideInput);
		Long proTime = sideInputMap.get(ic.expectedDocumentHash);
		if (proTime!=null) {
			c.output(PipelineTags.contentNotToIndexExactDupesTag,ic);
			continue;
		}
		
		if (isFirst) {
			isFirst = false;
			c.output(ic);
		} else {
			c.output(PipelineTags.contentNotToIndexExactDupesTag,ic);
		}
	}
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/**
 * Renames a {@link List} of file-like resources from one location to another.
 *
 * <p>The number of source resources must equal the number of destination resources.
 * Destination resources will be created recursively.
 *
 * <p>{@code srcResourceIds} and {@code destResourceIds} must have the same scheme.
 *
 * <p>It doesn't support renaming globs.
 *
 * @param srcResourceIds the references of the source resources
 * @param destResourceIds the references of the destination resources
 */
public static void rename(
    List<ResourceId> srcResourceIds, List<ResourceId> destResourceIds, MoveOptions... moveOptions)
    throws IOException {
  validateSrcDestLists(srcResourceIds, destResourceIds);
  if (srcResourceIds.isEmpty()) {
    // Short-circuit.
    return;
  }

  List<ResourceId> srcToRename = srcResourceIds;
  List<ResourceId> destToRename = destResourceIds;
  if (Sets.newHashSet(moveOptions).contains(
      MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES)) {
    KV<List<ResourceId>, List<ResourceId>> existings =
        filterMissingFiles(srcResourceIds, destResourceIds);
    srcToRename = existings.getKey();
    destToRename = existings.getValue();
  }
  if (srcToRename.isEmpty()) {
    return;
  }
  getFileSystemInternal(srcToRename.iterator().next().getScheme())
      .rename(srcToRename, destToRename);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
  MutationGroupEncoder mutationGroupEncoder = new MutationGroupEncoder(c.sideInput(schemaView));
  KV<String, Iterable<SerializedMutation>> element = c.element();
  for (SerializedMutation kv : element.getValue()) {
    byte[] value = kv.getMutationGroupBytes();
    MutationGroup mg = mutationGroupEncoder.decode(value);
    Iterables.addAll(mutations, mg);
    batchSizeBytes += MutationSizeEstimator.sizeOf(mg);
    if (batchSizeBytes >= maxBatchSizeBytes || mutations.size() > MAX_NUM_MUTATIONS) {
      c.output(mutations);
      mutations = new ArrayList<>();
      batchSizeBytes = 0;
    }
  }
  if (!mutations.isEmpty()) {
    c.output(mutations);
    mutations = new ArrayList<>();
    batchSizeBytes = 0;
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) {
  for (KV<String, Long> item : c.element()) {
    String session = item.getKey();
    long count = item.getValue();
    c.output(session + " : " + count + " : " + ((IntervalWindow) window).start());
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
  KV<Integer, String> record = c.element();
  int timestamp = record.getKey();
  String userName = record.getValue();
  if (userName != null) {
    // Sets the implicit timestamp field to be used in windowing.
    c.outputWithTimestamp(userName, new Instant(timestamp));
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public WindowedValue<KV<K, List<V>>> map(KV<org.joda.time.Instant,
    WindowedValue<KV<K, List<V>>>> kv) {
  Instant timestamp = kv.getKey();
  WindowedValue<KV<K, List<V>>> wv = kv.getValue();
  return WindowedValue.of(wv.getValue(), timestamp, wv.getWindows(), wv.getPane());
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
    public KV<BigDecimal, VarAgg> addInput(KV<BigDecimal, VarAgg> accumulator, T input) {
      BigDecimal v;
      if (input == null) {
        return accumulator;
      } else {
        v = new BigDecimal(input.toString());
        accumulator.getValue().count++;
        accumulator.getValue().sum = accumulator.getValue().sum
                .add(new BigDecimal(input.toString()));
        BigDecimal variance;
        if (accumulator.getValue().count > 1) {

//          pseudo code for the formula
//          t = count * v - sum;
//          variance = (t^2) / (count * (count - 1));
          BigDecimal t = v.multiply(new BigDecimal(accumulator.getValue().count))
                                    .subtract(accumulator.getValue().sum);
          variance = t.pow(2)
                  .divide(new BigDecimal(accumulator.getValue().count)
                            .multiply(new BigDecimal(accumulator.getValue().count)
                                      .subtract(BigDecimal.ONE)), mc);
        } else {
          variance = BigDecimal.ZERO;
        }
       return KV.of(accumulator.getKey().add(variance), accumulator.getValue());
      }
    }
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
    public KV<BigDecimal, VarAgg> mergeAccumulators(
            Iterable<KV<BigDecimal, VarAgg>> accumulators) {
      BigDecimal variance = new BigDecimal(0);
      long count = 0;
      BigDecimal sum = new BigDecimal(0);

      Iterator<KV<BigDecimal, VarAgg>> ite = accumulators.iterator();
      while (ite.hasNext()) {
        KV<BigDecimal, VarAgg> r = ite.next();

        BigDecimal b = r.getValue().sum;

        count += r.getValue().count;
        sum = sum.add(b);

//        t = ( r.count / count ) * sum - b;
//        d = t^2 * ( ( count / r.count ) / ( count + r.count ) );
        BigDecimal t = new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
                .multiply(sum).subtract(b);
        BigDecimal d = t.pow(2)
                .multiply(new BigDecimal(r.getValue().count).divide(new BigDecimal(count), mc)
                          .divide(new BigDecimal(count)
                                  .add(new BigDecimal(r.getValue().count))), mc);
        variance = variance.add(r.getKey().add(d));
      }

      return KV.of(variance, new VarAgg(count, sum));
    }
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/** Extract window from a {@link KV} with {@link WindowedValue} value. */
static <K, V> Function<KV<K, WindowedValue<V>>, WindowedValue<KV<K, V>>> toKVByWindowInValue() {
  return new Function<KV<K, WindowedValue<V>>, WindowedValue<KV<K, V>>>() {
    @Override
    public WindowedValue<KV<K, V>> call(KV<K, WindowedValue<V>> kv) throws Exception {
      WindowedValue<V> wv = kv.getValue();
      return wv.withValue(KV.of(kv.getKey(), wv.getValue()));
    }
  };
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public boolean canStopPolling(Instant now, KV<Instant, ReadableDuration> state) {
  Instant timeOfLastNewOutput = state.getKey();
  ReadableDuration maxTimeSinceNewOutput = state.getValue();
  return timeOfLastNewOutput != null
      && new Duration(timeOfLastNewOutput, now).isLongerThan(maxTimeSinceNewOutput);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/** {@link KV} to pair function. */
public static <K, V> PairFunction<KV<K, V>, K, V> toPairFunction() {
  return new PairFunction<KV<K, V>, K, V>() {
    @Override
    public Tuple2<K, V> call(KV<K, V> kv) {
      return new Tuple2<>(kv.getKey(), kv.getValue());
    }
  };
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
  Iterator<KV<W, Long>> iterator = c.element().getValue().iterator();
  KV<W, Long> currentValue = iterator.next();
  Object currentWindowStructuralValue = windowCoder.structuralValue(currentValue.getKey());
  long size = 0;
  while (iterator.hasNext()) {
    KV<W, Long> nextValue = iterator.next();
    Object nextWindowStructuralValue = windowCoder.structuralValue(nextValue.getKey());

    size += currentValue.getValue();
    if (!currentWindowStructuralValue.equals(nextWindowStructuralValue)) {
      c.output(IsmRecord.<WindowedValue<V>>meta(
          ImmutableList.of(IsmFormat.getMetadataKey(), currentValue.getKey(), 0L),
          CoderUtils.encodeToByteArray(VarLongCoder.of(), size)));
      size = 0;
    }

    currentValue = nextValue;
    currentWindowStructuralValue = nextWindowStructuralValue;
  }

  size += currentValue.getValue();
  // Output the final value since it is guaranteed to be on a window boundary.
  c.output(IsmRecord.<WindowedValue<V>>meta(
      ImmutableList.of(IsmFormat.getMetadataKey(), currentValue.getKey(), 0L),
      CoderUtils.encodeToByteArray(VarLongCoder.of(), size)));
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c, BoundedWindow window) {
  BoundedWindow w = window;
  KV<Integer, CoGbkResult> e = c.element();
  CoGbkResult row = e.getValue();
  Iterable<String> clicks = row.getAll(clicksTag);
  Iterable<String> purchases = row.getAll(purchasesTag);
  for (String click : clicks) {
    for (String purchase : purchases) {
      c.output(KV.of(click + ":" + purchase,
                     c.timestamp().getMillis() + ":" + w.maxTimestamp().getMillis()));
    }
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public ListenableFuture<MutateRowResponse> writeRecord(
    KV<ByteString, Iterable<Mutation>> record) {
  service.verifyTableExists(tableId);
  Map<ByteString, ByteString> table = service.getTable(tableId);
  ByteString key = record.getKey();
  for (Mutation m : record.getValue()) {
    SetCell cell = m.getSetCell();
    if (cell.getValue().isEmpty()) {
      return Futures.immediateFailedCheckedFuture(new IOException("cell value missing"));
    }
    table.put(key, cell.getValue());
  }
  return Futures.immediateFuture(MutateRowResponse.getDefaultInstance());
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public boolean advance() throws IOException {
  if (iterator.hasNext()) {
    KV<String, Instant> kv = iterator.next();
    collected = false;
    currentRecord = kv.getKey();
    currentTimestamp = kv.getValue();
    return true;
  } else {
    return false;
  }
}