Java Intersection.update方法代码示例

import com.yahoo.sketches.theta.Intersection; //导入方法依赖的package包/类
/**
 * Main logic called by hive if sketchSize is also passed in. Computes the
 * intersection of two sketches of same or different column.
 *
 * @param firstSketchBytes
 *          first sketch to be intersected.
 * @param secondSketchBytes
 *          second sketch to be intersected.
 * @param hashSeed
 *          Only required if input sketches were constructed using an update seed that was not the default.
 * @return resulting sketch of intersection.
 */
public BytesWritable evaluate(final BytesWritable firstSketchBytes,
    final BytesWritable secondSketchBytes, final long hashSeed) {
  Sketch firstSketch = null;
  if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) {
    firstSketch = Sketch.wrap(Memory.wrap(firstSketchBytes.getBytes()), hashSeed);
  }

  Sketch secondSketch = null;
  if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) {
    secondSketch = Sketch.wrap(Memory.wrap(secondSketchBytes.getBytes()), hashSeed);
  }

  final Intersection intersect = SetOperation.builder().setSeed(hashSeed).buildIntersection();
  intersect.update(firstSketch);
  intersect.update(secondSketch);
  return new BytesWritable(intersect.getResult().toByteArray());
}
 

import com.yahoo.sketches.theta.Intersection; //导入方法依赖的package包/类
/*************************************************************************************************
 * Updates an intersection from a bag of sketches
 *
 * @param bag A bag of sketchTuples.
 * @param intersection The intersection to update
 * @param seed to check against incoming sketches
 */
private static void updateIntersection(final DataBag bag, final Intersection intersection,
    final long seed) {
  //Bag is not empty. process each innerTuple in the bag
  for (Tuple innerTuple : bag) {
    //validate the inner Tuples
    final Object f0 = extractFieldAtIndex(innerTuple, 0);
    if (f0 == null) {
    continue;
  }
    final Byte type = extractTypeAtIndex(innerTuple, 0);
  // add only the first field of the innerTuple to the intersection
  if (type == DataType.BYTEARRAY) {
    final DataByteArray dba = (DataByteArray) f0;
    final Memory srcMem = Memory.wrap(dba.get());
    final Sketch sketch = Sketch.wrap(srcMem, seed);
    intersection.update(sketch);
  }
  else {
    throw new IllegalArgumentException(
        "Field type was not DataType.BYTEARRAY: " + type);
    }
  }
}
 

import com.yahoo.sketches.theta.Intersection; //导入方法依赖的package包/类
@Override //IntermediateFinal exec
public Tuple exec(final Tuple inputTuple) throws IOException { //throws is in API

  final Intersection intersection = SetOperation.builder().setSeed(mySeed_).buildIntersection();
  final DataBag outerBag = extractBag(inputTuple); //InputTuple.bag0
  if (outerBag == null) {  //must have non-empty outer bag at field 0.
    return myEmptyCompactOrderedSketchTuple_;
  }
  //Bag is not empty.

  for (Tuple dataTuple : outerBag) {
    final Object f0 = extractFieldAtIndex(dataTuple, 0); //inputTuple.bag0.dataTupleN.f0
    //must have non-null field zero
    if (f0 == null) {
      continue; //go to next dataTuple if there is one.  If none, exception is thrown.
    }
    //f0 is not null
    if (f0 instanceof DataBag) {
      final DataBag innerBag = (DataBag)f0; //inputTuple.bag0.dataTupleN.f0:bag
      if (innerBag.size() == 0) {
        continue; //go to next dataTuple if there is one.  If none, exception is thrown.
      }
      //If field 0 of a dataTuple is again a Bag all tuples of this inner bag
      // will be passed into the union.
      //It is due to system bagged outputs from multiple mapper Initial functions.
      //The Intermediate stage was bypassed.
      updateIntersection(innerBag, intersection, mySeed_); //process all tuples of innerBag

    }
    else if (f0 instanceof DataByteArray) { //inputTuple.bag0.dataTupleN.f0:DBA
      //If field 0 of a dataTuple is a DataByteArray we assume it is a sketch from a prior call
      //It is due to system bagged outputs from multiple mapper Intermediate functions.
      // Each dataTuple.DBA:sketch will merged into the union.
      final DataByteArray dba = (DataByteArray) f0;
      final Memory srcMem = Memory.wrap(dba.get());
      final Sketch sketch = Sketch.wrap(srcMem, mySeed_);
      intersection.update(sketch);
    }
    else { // we should never get here.
      throw new IllegalArgumentException("dataTuple.Field0: Is not a DataByteArray: "
          + f0.getClass().getName());
    }
  }

  final CompactSketch compactSketch = intersection.getResult(true, null);
  return compactOrderedSketchToTuple(compactSketch);
}