Java DataBag.add方法代码示例

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void checkNotDBAExcep() throws IOException {
  DataToSketch inter = new DataToSketch();
  //create inputTuple and a bag, add bag to inputTuple
  Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  inputTuple.set(0, bag);
  
  Tuple innerTuple = TupleFactory.getInstance().newTuple(1);
  bag.add(innerTuple);
  inter.accumulate(inputTuple); //add empty tuple
  
  innerTuple.set(0, new Double(1.0));  //not a DBA
  inter = new DataToSketch();
  inter.accumulate(inputTuple); //add wrong type
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void checkExactAlgebraicInitial() throws IOException {
  EvalFunc<Tuple> unionFuncInit = new Union.Initial("256");
  
  Tuple inputTuple = TupleFactory.getInstance().newTuple(1);
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  inputTuple.set(0, bag);
  
  for (int i = 0; i < 4; i++ ) { //4 sketches with one value each
    Tuple dataTuple = TupleFactory.getInstance().newTuple(1);
    dataTuple.set(0, createDbaFromQssRange(16, i, 1));
    
    bag.add(dataTuple);
  }
  
  Tuple resultTuple = unionFuncInit.exec(inputTuple);
  assertTrue(resultTuple == inputTuple);  //returns the inputTuple
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void algebraicIntermediateFinalNormalCase() throws Exception {
  EvalFunc<Tuple> func = new DataToDoublesSketch.IntermediateFinal();
  DataBag bag = bagFactory.newDefaultBag();

  { // this is to simulate an output from Initial
    DataBag innerBag = bagFactory.newDefaultBag();
    innerBag.add(tupleFactory.newTuple(1.0));
    bag.add(tupleFactory.newTuple(innerBag));
  }

  { // this is to simulate an output from a prior call of IntermediateFinal
    UpdateDoublesSketch qs = DoublesSketch.builder().build();
    qs.update(2.0);
    bag.add(tupleFactory.newTuple(new DataByteArray(qs.toByteArray())));
  }

  Tuple resultTuple = func.exec(tupleFactory.newTuple(bag));
  DoublesSketch sketch = getSketch(resultTuple);
  Assert.assertFalse(sketch.isEmpty());
  Assert.assertEquals(sketch.getN(), 2);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
static DataBag createDataBagFromSketch(final VarOptItemsSketch<Tuple> sketch) {
  final DataBag output = BAG_FACTORY.newDefaultBag();

  final VarOptItemsSamples<Tuple> samples = sketch.getSketchSamples();

  try {
    // create (weight, item) tuples to add to output bag
    for (final VarOptItemsSamples<Tuple>.WeightedSample ws : samples) {
      final Tuple weightedSample = TUPLE_FACTORY.newTuple(2);
      weightedSample.set(0, ws.getWeight());
      weightedSample.set(1, ws.getItem());
      output.add(weightedSample);
    }
  } catch (final ExecException e) {
    throw new RuntimeException("Pig error: " + e.getMessage(), e);
  }

  return output;
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void algebraicIntermediateFromInitial() throws Exception {
  @SuppressWarnings("unchecked")
  EvalFunc<Tuple> func =
      (EvalFunc<Tuple>) Class.forName(new UnionSketch().getIntermed())
      .getConstructor(String.class, String.class).newInstance("10", "HLL_6");
  HllSketch inputSketch = new HllSketch(12);
  inputSketch.update(1);
  inputSketch.update(2);
  inputSketch.update(3);
  DataBag outerBag = bagFactory.newDefaultBag();
  DataBag innerBag = bagFactory.newDefaultBag();
  innerBag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toCompactByteArray())));
  outerBag.add(tupleFactory.newTuple(innerBag));
  Tuple result = func.exec(tupleFactory.newTuple(outerBag));
  HllSketch sketch = DataToSketchTest.getSketch((DataByteArray) result.get(0));
  Assert.assertFalse(sketch.isEmpty());
  Assert.assertEquals(sketch.getEstimate(), 3.0, 0.01);
  Assert.assertEquals(sketch.getLgConfigK(), 10);
  Assert.assertEquals(sketch.getTgtHllType(), TgtHllType.HLL_6);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void execNormalCase() throws Exception {
  EvalFunc<Tuple> func = new DataToStringsSketch();
  DataBag bag = BAG_FACTORY.newDefaultBag();
  bag.add(TUPLE_FACTORY.newTuple("a"));
  Tuple resultTuple = func.exec(TUPLE_FACTORY.newTuple(bag));
  ItemsSketch<String> sketch = getSketch(resultTuple);
  Assert.assertFalse(sketch.isEmpty());
  Assert.assertEquals(sketch.getN(), 1);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test(expectedExceptions = IllegalArgumentException.class)
public void execWrongSizeOfInnerTuple() throws Exception {
  EvalFunc<Tuple> func = new DataToDoubleSummarySketch();
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  bag.add(PigUtil.objectsToTuple(1));
  Tuple inputTuple = PigUtil.objectsToTuple(bag);
  func.exec(inputTuple);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void checkSmall() throws IOException {
  EvalFunc<Tuple> func = new DataToSketch("32");

  Tuple inputTuple = null;
  Tuple resultTuple = func.exec(inputTuple);
  Sketch sketch = tupleToSketch(resultTuple, seed_);
  
  assertTrue(sketch.isEmpty());
  
  inputTuple = TupleFactory.getInstance().newTuple(1);
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  inputTuple.set(0, bag);

  int u = 32;
  for (int ii = 0; ii < u; ii++ ) {
    Tuple dataTuple = TupleFactory.getInstance().newTuple(1);
    dataTuple.set(0, ii);

    bag.add(dataTuple);
  }

  resultTuple = func.exec(inputTuple);
  assertNotNull(resultTuple);
  assertEquals(resultTuple.size(), 1);
  DataByteArray bytes = (DataByteArray) resultTuple.get(0);
  assertTrue(bytes.size() > 0);
  sketch = tupleToSketch(resultTuple, seed_);
  assertEquals(sketch.getEstimate(), u, 0.0);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void algebraicInitial() throws Exception {
  EvalFunc<Tuple> func = new UnionFrequentStringsSketch.Initial(null);
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  bag.add(null);
  bag.add(null);
  bag.add(null);

  Tuple resultTuple = func.exec(PigUtil.objectsToTuple(bag));
  Assert.assertNotNull(resultTuple);
  Assert.assertEquals(resultTuple.size(), 1);
  DataBag resultBag = (DataBag) resultTuple.get(0);
  Assert.assertEquals(resultBag.size(), 3);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
private static DataBag numberArrayToDataBag(Number... values) {
  DataBag bag = DefaultBagFactory.getInstance().newDefaultBag();
  for (Number value : values) {
    bag.add(TupleFactory.getInstance().newTuple(value));
  }
  return bag;
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test(expectedExceptions = IllegalArgumentException.class)
public void execWrongKeyType() throws Exception {
  EvalFunc<Tuple> func = new DataToDoubleSummarySketch();
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  bag.add(PigUtil.objectsToTuple(new Object(), 1.0)); // Object in place of key is not supported
  Tuple inputTuple = PigUtil.objectsToTuple(bag);
  func.exec(inputTuple);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test(expectedExceptions = ClassCastException.class)
public void execWrongValueType() throws Exception {
  EvalFunc<Tuple> func = new DataToStringsSketch();
  DataBag bag = BAG_FACTORY.newDefaultBag();
  bag.add(TUPLE_FACTORY.newTuple(1.0));
  func.exec(TUPLE_FACTORY.newTuple(bag));
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test(expectedExceptions = IllegalArgumentException.class)
public void execWrongSizeOfInnerTuple() throws Exception {
  EvalFunc<Tuple> func = new DataToArrayOfDoublesSketch("32", "1");
  DataBag bag = BagFactory.getInstance().newDefaultBag();
  bag.add(PigUtil.objectsToTuple(1));
  Tuple inputTuple = PigUtil.objectsToTuple(bag);
  func.exec(inputTuple);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test
public void algebraicInitial() throws Exception {
  EvalFunc<Tuple> func = new UnionDoublesSketch.Initial();
  DataBag bag = bagFactory.newDefaultBag();
  bag.add(tupleFactory.newTuple());
  Tuple resultTuple = func.exec(tupleFactory.newTuple(bag));
  Assert.assertNotNull(resultTuple);
  Assert.assertEquals(resultTuple.size(), 1);
  Assert.assertTrue(resultTuple.get(0) instanceof DataBag);
  Assert.assertEquals(((DataBag) resultTuple.get(0)).size(), 1);
}
 

import org.apache.pig.data.DataBag; //导入方法依赖的package包/类
@Test(expectedExceptions = ClassCastException.class)
public void execWrongValueType() throws Exception {
  EvalFunc<Tuple> func = new DataToDoublesSketch();
  DataBag bag = bagFactory.newDefaultBag();
  bag.add(tupleFactory.newTuple("a"));
  func.exec(tupleFactory.newTuple(bag));
}