Java KV.getKey方法代码示例

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
	KV<String, Iterable<InputContent>> kv = c.element();
	String documentHash = kv.getKey();
	Iterable<InputContent> dupes = kv.getValue();
	boolean isFirst = true;
	int groupSize = Iterables.size(dupes);
	for (InputContent ic : dupes) {
	
		// Check if this doc was already processed and stored in BQ
		Map<String,Long> sideInputMap = c.sideInput(alreadyProcessedDocsSideInput);
		Long proTime = sideInputMap.get(ic.expectedDocumentHash);
		if (proTime!=null) {
			c.output(PipelineTags.contentNotToIndexExactDupesTag,ic);
			continue;
		}
		
		if (isFirst) {
			isFirst = false;
			c.output(ic);
		} else {
			c.output(PipelineTags.contentNotToIndexExactDupesTag,ic);
		}
	}
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/**
 * A method for processing each element
 * @param c the context
 */
@ProcessElement
public void processElement(final ProcessContext c) {
    final KV<String, String> element = c.element();
    final String key = element.getKey();
    final File dir = resourceDirectory(config, key);
    if (!isNull(dir)) {
        LOGGER.debug("Writing cache for: {}", key);
        if (CachedResource.write(dir, key)) {
            c.output(element);
        } else {
            LOGGER.error("Error writing cached resource for {}", key);
        }
    } else {
        LOGGER.error("Error accessing cached resource location for {}", key);
    }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/**
 * A method for processing each element
 * @param c the context
 */
@ProcessElement
public void processElement(final ProcessContext c) {
    final KV<String, String> element = c.element();
    final String key = element.getKey();
    final File dir = resourceDirectory(dataLocation, key);
    if (!isNull(dir)) {
        LOGGER.debug("Writing cache for: {}", key);
        if (CachedResource.write(dir, key)) {
            c.output(element);
        } else {
            LOGGER.error("Error writing cached resource for {}", key);
        }
    } else {
        LOGGER.error("Error accessing cached resource location for {}", key);
    }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public void processElement(WindowedValue<KV<K, V>> element) {
  KV<K, V> kv = element.getValue();
  K key = kv.getKey();
  byte[] encodedKey;
  try {
    encodedKey = encodeToByteArray(keyCoder, key);
  } catch (CoderException exn) {
    // TODO: Put in better element printing:
    // truncate if too long.
    throw new IllegalArgumentException(
        String.format("unable to encode key %s of input to %s using %s", key, this, keyCoder),
        exn);
  }
  GroupingKey<K> groupingKey = new GroupingKey<>(key, encodedKey);
  List<WindowedValue<V>> values = groupingMap.get(groupingKey);
  if (values == null) {
    values = new ArrayList<>();
    groupingMap.put(groupingKey, values);
  }
  values.add(element.withValue(kv.getValue()));
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/**
 * Copies a {@link List} of file-like resources from one location to another.
 *
 * <p>The number of source resources must equal the number of destination resources.
 * Destination resources will be created recursively.
 *
 * <p>{@code srcResourceIds} and {@code destResourceIds} must have the same scheme.
 *
 * <p>It doesn't support copying globs.
 *
 * @param srcResourceIds the references of the source resources
 * @param destResourceIds the references of the destination resources
 */
public static void copy(
    List<ResourceId> srcResourceIds, List<ResourceId> destResourceIds, MoveOptions... moveOptions)
    throws IOException {
  validateSrcDestLists(srcResourceIds, destResourceIds);
  if (srcResourceIds.isEmpty()) {
    // Short-circuit.
    return;
  }

  List<ResourceId> srcToCopy = srcResourceIds;
  List<ResourceId> destToCopy = destResourceIds;
  if (Sets.newHashSet(moveOptions).contains(
      MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES)) {
    KV<List<ResourceId>, List<ResourceId>> existings =
        filterMissingFiles(srcResourceIds, destResourceIds);
    srcToCopy = existings.getKey();
    destToCopy = existings.getValue();
  }
  if (srcToCopy.isEmpty()) {
    return;
  }
  getFileSystemInternal(srcToCopy.iterator().next().getScheme())
      .copy(srcToCopy, destToCopy);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public WindowedValue<KV<K, List<V>>> map(KV<org.joda.time.Instant,
    WindowedValue<KV<K, List<V>>>> kv) {
  Instant timestamp = kv.getKey();
  WindowedValue<KV<K, List<V>>> wv = kv.getValue();
  return WindowedValue.of(wv.getValue(), timestamp, wv.getWindows(), wv.getPane());
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public ListenableFuture<MutateRowResponse> writeRecord(
    KV<ByteString, Iterable<Mutation>> record) {
  service.verifyTableExists(tableId);
  Map<ByteString, ByteString> table = service.getTable(tableId);
  ByteString key = record.getKey();
  for (Mutation m : record.getValue()) {
    SetCell cell = m.getSetCell();
    if (cell.getValue().isEmpty()) {
      return Futures.immediateFailedCheckedFuture(new IOException("cell value missing"));
    }
    table.put(key, cell.getValue());
  }
  return Futures.immediateFuture(MutateRowResponse.getDefaultInstance());
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public void add(KV<byte[], byte[]> record) throws IOException {
  checkState(!sortCalled, "Records can only be added before sort()");

  initHadoopSorter();

  BytesWritable key = new BytesWritable(record.getKey());
  BytesWritable value = new BytesWritable(record.getValue());

  writer.append(key, value);
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/**
 * Generates random records and executes a test with the provided number of sorters and number of
 * records per sorter.
 */
public static void testRandom(
    SorterGenerator sorterGenerator, int numSorters, int numRecordsPerSorter) throws Exception {
  long start = System.currentTimeMillis();
  for (int i = 0; i < numSorters; ++i) {
    Sorter sorter = sorterGenerator.generateSorter();
    Random rnd = new Random(0L);
    for (int j = 0; j < numRecordsPerSorter; ++j) {
      byte[] key = new byte[8];
      byte[] value = new byte[8];
      rnd.nextBytes(key);
      rnd.nextBytes(value);
      sorter.add(KV.of(key, value));
    }

    byte[] prevKey = null;
    for (KV<byte[], byte[]> record : sorter.sort()) {
      assertTrue(
          prevKey == null
              || UnsignedBytes.lexicographicalComparator().compare(prevKey, record.getKey()) < 0);
      prevKey = record.getKey();
    }
  }
  long end = System.currentTimeMillis();
  System.out.println(
      "Took "
          + (end - start)
          + "ms for "
          + numRecordsPerSorter * numSorters * 1000.0 / (end - start)
          + " records/s");
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public boolean advance() throws IOException {
  if (iterator.hasNext()) {
    KV<String, Instant> kv = iterator.next();
    collected = false;
    currentRecord = kv.getKey();
    currentTimestamp = kv.getValue();
    return true;
  } else {
    return false;
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
  KV<Integer, String> record = c.element();
  int timestamp = record.getKey();
  String userName = record.getValue();
  if (userName != null) {
    // Sets the implicit timestamp field to be used in windowing.
    c.outputWithTimestamp(userName, new Instant(timestamp));
  }
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
/** {@link KV} to pair function. */
public static <K, V> PairFunction<KV<K, V>, K, V> toPairFunction() {
  return new PairFunction<KV<K, V>, K, V>() {
    @Override
    public Tuple2<K, V> call(KV<K, V> kv) {
      return new Tuple2<>(kv.getKey(), kv.getValue());
    }
  };
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
protected BigDecimal prepareOutput(KV<BigDecimal, VarAgg> accumulator){
  BigDecimal decimalVar;
  if (accumulator.getValue().count > 1) {
    BigDecimal a = accumulator.getKey();
    BigDecimal b = new BigDecimal(accumulator.getValue().count)
            .subtract(this.isSamp ? BigDecimal.ONE : BigDecimal.ZERO);

    decimalVar = a.divide(b, mc);
  } else {
    decimalVar = BigDecimal.ZERO;
  }
  return decimalVar;
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public String toString(KV<Instant, ReadableDuration> state) {
  return "AfterTotalOf{"
      + "timeStarted="
      + state.getKey()
      + ", maxTimeSinceInput="
      + state.getValue()
      + '}';
}
 

import org.apache.beam.sdk.values.KV; //导入方法依赖的package包/类
@Override
public String getDestination(ValueInSingleWindow<KV<MsSqlTable, DbRow>> element) {
	KV<MsSqlTable, DbRow> kv = element.getValue();
	MsSqlTable table = kv.getKey();
	return datasetName + "." + table.getFullName();
}