Java VectorWrapper.getValueVector方法代码示例

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
/**
 * Method is invoked when we have a straight aggregate (no group by expression) and our input is empty.
 * In this case we construct an outgoing batch with record count as 1. For the nullable vectors we don't set anything
 * as we want the output to be NULL. For the required vectors (only for count()) we set the value to be zero since
 * we don't zero out our buffers initially while allocating them.
 */
private void constructSpecialBatch() {
  int exprIndex = 0;
  for (final VectorWrapper<?> vw: container) {
    final ValueVector vv = vw.getValueVector();
    AllocationHelper.allocateNew(vv, SPECIAL_BATCH_COUNT);
    vv.getMutator().setValueCount(SPECIAL_BATCH_COUNT);
    if (vv.getField().getType().getMode() == TypeProtos.DataMode.REQUIRED) {
      if (vv instanceof FixedWidthVector) {
        /*
         * The only case we should have a required vector in the aggregate is for count function whose output is
         * always a FixedWidthVector (BigIntVector). Zero out the vector.
         */
        ((FixedWidthVector) vv).zeroVector();
      } else {
        /*
         * If we are in this else block it means that we have a required vector which is of variable length. We
         * should not be here, raising an error since we have set the record count to be 1 and not cleared the
         * buffer
         */
        throw new DrillRuntimeException("FixedWidth vectors is the expected output vector type. " +
            "Corresponding expression: " + popConfig.getExprs()[exprIndex].toString());
      }
    }
    exprIndex++;
  }
  container.setRecordCount(SPECIAL_BATCH_COUNT);
  recordCount = SPECIAL_BATCH_COUNT;
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
private void allocateOutgoing() {
  for (final VectorWrapper w : outgoingContainer) {
    final ValueVector v = w.getValueVector();
    if (v instanceof FixedWidthVector) {
      AllocationHelper.allocate(v, OUTGOING_BATCH_SIZE, 1);
    } else {
      v.allocateNewSafe();
    }
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
private void cleanPartition() {
  partition = null;
  resetValues();
  for (VectorWrapper<?> vw : internal) {
    if ((vw.getValueVector() instanceof BaseDataValueVector)) {
      ((BaseDataValueVector) vw.getValueVector()).reset();
    }
  }
  lagCopiedToInternal = false;
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
protected static String getPlanInString(String sql, String columnName)
    throws Exception {
  final List<QueryDataBatch> results = testSqlWithResults(sql);
  final RecordBatchLoader loader = new RecordBatchLoader(getDrillbitContext().getAllocator());
  final StringBuilder builder = new StringBuilder();

  for (final QueryDataBatch b : results) {
    if (!b.hasData()) {
      continue;
    }

    loader.load(b.getHeader().getDef(), b.getData());

    final VectorWrapper<?> vw;
    try {
        vw = loader.getValueAccessorById(
            NullableVarCharVector.class,
            loader.getValueVectorId(SchemaPath.getSimplePath(columnName)).getFieldIds());
    } catch (Throwable t) {
      throw new Exception("Looks like you did not provide an explain plan query, please add EXPLAIN PLAN FOR to the beginning of your query.");
    }

    System.out.println(vw.getValueVector().getField().toExpr());
    final ValueVector vv = vw.getValueVector();
    for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
      final Object o = vv.getAccessor().getObject(i);
      builder.append(o);
      System.out.println(vv.getAccessor().getObject(i));
    }
    loader.clear();
    b.release();
  }

  return builder.toString();
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
public void testFilterPlan() throws Exception {
  final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();

  try (final Drillbit bit1 = new Drillbit(config, serviceSet);
      final DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
    bit1.run();
    client.connect();
    final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
        Resources.toString(Resources.getResource("physical_filter.json"), Charsets.UTF_8));
    final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
    for (final QueryDataBatch b : results) {
      System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
      loader.load(b.getHeader().getDef(), b.getData());
      for (final VectorWrapper<?> vw : loader) {
        System.out.println(vw.getValueVector().getField().toExpr());
        final ValueVector vv = vw.getValueVector();
        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
          final Object o = vv.getAccessor().getObject(i);
          System.out.println(vv.getAccessor().getObject(i));
        }
      }
      loader.clear();
      b.release();
    }
    client.close();
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
public void testJoinPlan() throws Exception {
  final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();

  try (final Drillbit bit1 = new Drillbit(config, serviceSet);
      final DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
    bit1.run();
    client.connect();
    final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
        Resources.toString(Resources.getResource("physical_join.json"), Charsets.UTF_8));
    final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
    for (final QueryDataBatch b : results) {
      System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
      loader.load(b.getHeader().getDef(), b.getData());
      for (final VectorWrapper<?> vw : loader) {
        System.out.println(vw.getValueVector().getField().toExpr());
        final ValueVector vv = vw.getValueVector();
        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
          final Object o = vv.getAccessor().getObject(i);
          System.out.println(vv.getAccessor().getObject(i));
        }
      }
      loader.clear();
      b.release();
    }
    client.close();
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
public void testFilterString() throws Exception {
  final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();

  try (final Drillbit bit1 = new Drillbit(config, serviceSet);
      final DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
    bit1.run();
    client.connect();
    final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.LOGICAL,
        Resources.toString(Resources.getResource("logical_string_filter.json"), Charsets.UTF_8));
    final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
    for (final QueryDataBatch b : results) {
      System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
      loader.load(b.getHeader().getDef(), b.getData());
      for (final VectorWrapper<?> vw : loader) {
        System.out.println(vw.getValueVector().getField().toExpr());
        final ValueVector vv = vw.getValueVector();
        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
          final Object o = vv.getAccessor().getObject(i);
          if (vv instanceof VarBinaryVector) {
            final VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
            final VarBinaryHolder vbh = new VarBinaryHolder();
            x.get(i, vbh);
            System.out.printf("%d..%d", vbh.start, vbh.end);

            System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
          } else {
            System.out.println(vv.getAccessor().getObject(i));
          }

        }
      }
      loader.clear();
      b.release();
    }
    client.close();
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
public void testLogicalJsonScan() throws Exception {
  final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();

  try (final Drillbit bit1 = new Drillbit(config, serviceSet);
      final DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
    bit1.run();
    client.connect();
    final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.LOGICAL,
        Resources.toString(Resources.getResource("logical_json_scan.json"), Charsets.UTF_8));
    final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
    for (final QueryDataBatch b : results) {
      System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
      loader.load(b.getHeader().getDef(), b.getData());
      for (final VectorWrapper vw : loader) {
        System.out.println(vw.getValueVector().getField().toExpr());
        final ValueVector vv = vw.getValueVector();
        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
          final Object o = vv.getAccessor().getObject(i);
          if (vv instanceof VarBinaryVector) {
            final VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
            final VarBinaryHolder vbh = new VarBinaryHolder();
            x.get(i, vbh);
            System.out.printf("%d..%d", vbh.start, vbh.end);

            System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
          } else {
            System.out.println(vv.getAccessor().getObject(i));
          }

        }
      }
      loader.clear();
      b.release();
    }
    client.close();
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
public void testOrderVarbinary() throws Exception {
  final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();

  try (final Drillbit bit1 = new Drillbit(config, serviceSet);
      final DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
    bit1.run();
    client.connect();
    final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
        Resources.toString(Resources.getResource("physical_order_varbinary.json"), Charsets.UTF_8));
    final RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
    for (final QueryDataBatch b : results) {
      System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
      loader.load(b.getHeader().getDef(), b.getData());
      for (final VectorWrapper vw : loader) {
        System.out.println(vw.getValueVector().getField().toExpr());
        final ValueVector vv = vw.getValueVector();
        for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
          final Object o = vv.getAccessor().getObject(i);
          if (vv instanceof VarBinaryVector) {
            final VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
            final VarBinaryHolder vbh = new VarBinaryHolder();
            x.get(i, vbh);
            System.out.printf("%d..%d", vbh.start, vbh.end);

            System.out.println("[" + new String((byte[]) vv.getAccessor().getObject(i)) + "]");
          } else {
            System.out.println(vv.getAccessor().getObject(i));
          }

        }
      }
      loader.clear();
      b.release();
    }
    client.close();
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
  @Ignore
  public void testParseParquetPhysicalPlan() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    DrillConfig config = DrillConfig.create();

    try (Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) {
      bit1.run();
      client.connect();
      List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
      int count = 0;
      for (QueryDataBatch b : results) {
        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
        count += b.getHeader().getRowCount();
        loader.load(b.getHeader().getDef(), b.getData());
        for (VectorWrapper vw : loader) {
          System.out.print(vw.getValueVector().getField().toExpr() + ": ");
          ValueVector vv = vw.getValueVector();
          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
            Object o = vv.getAccessor().getObject(i);
            if (o instanceof byte[]) {
              System.out.print(" [" + new String((byte[]) o) + "]");
            } else {
              System.out.print(" [" + vv.getAccessor().getObject(i) + "]");
            }
//            break;
          }
          System.out.println();
        }
        loader.clear();
        b.release();
      }
      client.close();
      System.out.println(String.format("Got %d total results", count));
    }
  }
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
private boolean outputKeys(VectorContainer outContainer, int outStartIndex, int numRecords) {

      /** for debugging
      BigIntVector vv0 = getValueVector(0);
      BigIntHolder holder = new BigIntHolder();
      */

      // set the value count for htContainer's value vectors before the transfer ..
      setValueCount();

      Iterator<VectorWrapper<?>> outgoingIter = outContainer.iterator();

      for (VectorWrapper<?> sourceWrapper : htContainer) {
        ValueVector sourceVV = sourceWrapper.getValueVector();
        ValueVector targetVV = outgoingIter.next().getValueVector();
        TransferPair tp = sourceVV.makeTransferPair(targetVV);
        tp.splitAndTransfer(outStartIndex, numRecords);
      }

/*
      logger.debug("Attempting to output keys for batch index: {} from index {} to maxOccupiedIndex {}.",
      this.batchIndex, 0, maxOccupiedIdx);
      for (int i = batchOutputCount; i <= maxOccupiedIdx; i++) {
        if (outputRecordKeys(i, batchOutputCount) ) {
          if (EXTRA_DEBUG) logger.debug("Outputting keys to output index: {}", batchOutputCount) ;

          // debugging
          // holder.value = vv0.getAccessor().get(i);
          // if (holder.value == 100018 || holder.value == 100021) {
          //  logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i,
          //      batchOutputCount);
          // }

          batchOutputCount++;
        } else {
          return false;
        }
      }
 */
      return true;
    }
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
private void setValueCount() {
  for (VectorWrapper<?> vw : htContainer) {
    ValueVector vv = vw.getValueVector();
    vv.getMutator().setValueCount(maxOccupiedIdx + 1);
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Test
@SuppressWarnings("static-method")
public void test() throws Exception {
  final List<ValueVector> vectorList = Lists.newArrayList();
  final DrillConfig config = DrillConfig.create();
  try (final RemoteServiceSet serviceSet = RemoteServiceSet
      .getLocalServiceSet();
      final Drillbit bit = new Drillbit(config, serviceSet)) {
    bit.run();
    final DrillbitContext context = bit.getContext();

    final MaterializedField intField = MaterializedField.create(
        new SchemaPath("int", ExpressionPosition.UNKNOWN),
        Types.required(TypeProtos.MinorType.INT));
    final MaterializedField binField = MaterializedField.create(
        new SchemaPath("binary", ExpressionPosition.UNKNOWN),
        Types.required(TypeProtos.MinorType.VARBINARY));
    try (final IntVector intVector = (IntVector) TypeHelper.getNewVector(intField, context.getAllocator());
        final VarBinaryVector binVector =
            (VarBinaryVector) TypeHelper.getNewVector(binField, context.getAllocator())) {
      AllocationHelper.allocate(intVector, 4, 4);
      AllocationHelper.allocate(binVector, 4, 5);
      vectorList.add(intVector);
      vectorList.add(binVector);

      intVector.getMutator().setSafe(0, 0);
      binVector.getMutator().setSafe(0, "ZERO".getBytes());
      intVector.getMutator().setSafe(1, 1);
      binVector.getMutator().setSafe(1, "ONE".getBytes());
      intVector.getMutator().setSafe(2, 2);
      binVector.getMutator().setSafe(2, "TWO".getBytes());
      intVector.getMutator().setSafe(3, 3);
      binVector.getMutator().setSafe(3, "THREE".getBytes());
      intVector.getMutator().setValueCount(4);
      binVector.getMutator().setValueCount(4);

      VectorContainer container = new VectorContainer();
      container.addCollection(vectorList);
      container.setRecordCount(4);
      WritableBatch batch = WritableBatch.getBatchNoHVWrap(
          container.getRecordCount(), container, false);
      VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(
          batch, context.getAllocator());

      Configuration conf = new Configuration();
      conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");

      final VectorAccessibleSerializable newWrap = new VectorAccessibleSerializable(
          context.getAllocator());
      try (final FileSystem fs = FileSystem.get(conf)) {
        final File tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        final Path path = new Path(tempDir.getAbsolutePath(), "drillSerializable");
        try (final FSDataOutputStream out = fs.create(path)) {
          wrap.writeToStream(out);
          out.close();
        }

        try (final FSDataInputStream in = fs.open(path)) {
          newWrap.readFromStream(in);
        }
      }

      final VectorAccessible newContainer = newWrap.get();
      for (VectorWrapper<?> w : newContainer) {
        try (ValueVector vv = w.getValueVector()) {
          int values = vv.getAccessor().getValueCount();
          for (int i = 0; i < values; i++) {
            final Object o = vv.getAccessor().getObject(i);
            if (o instanceof byte[]) {
              System.out.println(new String((byte[]) o));
            } else {
              System.out.println(o);
            }
          }
        }
      }
    }
  }
}
 

import org.apache.drill.exec.record.VectorWrapper; //导入方法依赖的package包/类
@Override
synchronized public void dataArrived(QueryDataBatch result, ConnectionThrottle throttle) {
  logger.debug("result arrived in test batch listener.");
  int columnValCounter = 0;
  FieldInfo currentField;
  count += result.getHeader().getRowCount();
  boolean schemaChanged = false;
  final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
  try {
    schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
    // TODO:  Clean:  DRILL-2933:  That load(...) no longer throws
    // SchemaChangeException, so check/clean catch clause below.
  } catch (SchemaChangeException e) {
    throw new RuntimeException(e);
  }

  // used to make sure each vector in the batch has the same number of records
  int valueCount = batchLoader.getRecordCount();

  // print headers.
  if (schemaChanged) {
  } // do not believe any change is needed for when the schema changes, with the current mock scan use case

  for (final VectorWrapper vw : batchLoader) {
    final ValueVector vv = vw.getValueVector();
    currentField = props.fields.get(vv.getField().getPath().getRootSegment().getPath());
    if (!valuesChecked.containsKey(vv.getField().getPath().getRootSegment().getPath())) {
      valuesChecked.put(vv.getField().getPath().getRootSegment().getPath(), 0);
      columnValCounter = 0;
    } else {
      columnValCounter = valuesChecked.get(vv.getField().getPath().getRootSegment().getPath());
    }
    printColumnMajor(vv);

    if (testValues) {
      for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
        assertField(vv, j, currentField.type,
            currentField.values[columnValCounter % 3], currentField.name + "/");
        columnValCounter++;
      }
    } else {
      columnValCounter += vv.getAccessor().getValueCount();
    }

    valuesChecked.remove(vv.getField().getPath().getRootSegment().getPath());
    assertEquals("Mismatched value count for vectors in the same batch.", valueCount, vv.getAccessor().getValueCount());
    valuesChecked.put(vv.getField().getPath().getRootSegment().getPath(), columnValCounter);
  }

  if (ParquetRecordReaderTest.VERBOSE_DEBUG){
    printRowMajor(batchLoader);
  }
  batchCounter++;

  batchLoader.clear();
  result.release();
}