Java NullableVarBinaryVector类代码示例

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Override
protected void readField(long recordsToReadInThisPass) {
  this.bytebuf = pageReader.pageData;
  if (usingDictionary) {
    NullableVarBinaryVector.Mutator mutator =  castedVector.getMutator();
    Binary currDictValToWrite;
    for (int i = 0; i < recordsReadInThisIteration; i++){
      currDictValToWrite = pageReader.dictionaryValueReader.readBytes();
      mutator.setSafe(valuesReadInCurrentPass + i, currDictValToWrite.toByteBuffer(), 0,
          currDictValToWrite.length());
    }
    // Set the write Index. The next page that gets read might be a page that does not use dictionary encoding
    // and we will go into the else condition below. The readField method of the parent class requires the
    // writer index to be set correctly.
    int writerIndex = castedBaseVector.getBuffer().writerIndex();
    castedBaseVector.getBuffer().setIndex(0, writerIndex + (int)readLength);
  } else {
    super.readField(recordsToReadInThisPass);
    // TODO - replace this with fixed binary type in drill
    // for now we need to write the lengths of each value
    int byteLength = dataTypeLengthInBits / 8;
    for (int i = 0; i < recordsToReadInThisPass; i++) {
      castedVector.getMutator().setValueLengthSafe(valuesReadInCurrentPass + i, byteLength);
    }
  }
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
private Copier<?> getCopier(int jdbcType, int offset, ResultSet result, ValueVector v) {

    if (v instanceof NullableBigIntVector) {
      return new BigIntCopier(offset, result, (NullableBigIntVector.Mutator) v.getMutator());
    } else if (v instanceof NullableFloat4Vector) {
      return new Float4Copier(offset, result, (NullableFloat4Vector.Mutator) v.getMutator());
    } else if (v instanceof NullableFloat8Vector) {
      return new Float8Copier(offset, result, (NullableFloat8Vector.Mutator) v.getMutator());
    } else if (v instanceof NullableIntVector) {
      return new IntCopier(offset, result, (NullableIntVector.Mutator) v.getMutator());
    } else if (v instanceof NullableVarCharVector) {
      return new VarCharCopier(offset, result, (NullableVarCharVector.Mutator) v.getMutator());
    } else if (v instanceof NullableVarBinaryVector) {
      return new VarBinaryCopier(offset, result, (NullableVarBinaryVector.Mutator) v.getMutator());
    } else if (v instanceof NullableDateVector) {
      return new DateCopier(offset, result, (NullableDateVector.Mutator) v.getMutator());
    } else if (v instanceof NullableTimeVector) {
      return new TimeCopier(offset, result, (NullableTimeVector.Mutator) v.getMutator());
    } else if (v instanceof NullableTimeStampVector) {
      return new TimeStampCopier(offset, result, (NullableTimeStampVector.Mutator) v.getMutator());
    } else if (v instanceof NullableBitVector) {
      return new BitCopier(offset, result, (NullableBitVector.Mutator) v.getMutator());
    }

    throw new IllegalArgumentException("Unknown how to handle vector.");
  }
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
NullableVarBinaryColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                        ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableVarBinaryVector v,
                        SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  nullableVarBinaryVector = v;
  mutator = v.getMutator();
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Test
public void testByteSubstring(@Injectable final DrillbitContext bitContext,
                                @Injectable UserServer.UserClientConnection connection) throws Throwable {
  new NonStrictExpectations() {{
    bitContext.getMetrics(); result = new MetricRegistry();
    bitContext.getAllocator(); result = RootAllocatorFactory.newRoot(c);
    bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
    bitContext.getConfig(); result = c;
    bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c);
  }};

  final PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

  while(exec.next()) {
    final NullableVarBinaryVector c1 = exec.getValueVectorById(new SchemaPath("col3", ExpressionPosition.UNKNOWN), NullableVarBinaryVector.class);
    final NullableVarBinaryVector.Accessor a1 = c1.getAccessor();

    int count = 0;
    for(int i = 0; i < c1.getAccessor().getValueCount(); i++) {
      if (!a1.isNull(i)) {
        final NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
        a1.get(i, holder);
        assertEquals("aa", StringFunctionHelpers.toStringFromUTF8(holder.start,  holder.end,  holder.buffer));
        ++count;
      }
    }
    assertEquals(50, count);
  }

  if(context.getFailureCause() != null) {
    throw context.getFailureCause();
  }
  assertTrue(!context.isFailed());
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
private NullableVarBinaryVector getOrCreateColumnVector(MapVector mv, String qualifier) {
  int oldSize = mv.size();
  NullableVarBinaryVector v = mv.addOrGet(qualifier, COLUMN_TYPE, NullableVarBinaryVector.class);
  if (oldSize != mv.size()) {
    v.allocateNew();
  }
  return v;
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Override
public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
  final SequenceFileAsBinaryInputFormat inputFormat = new SequenceFileAsBinaryInputFormat();
  final JobConf jobConf = new JobConf(dfs.getConf());
  jobConf.setInputFormat(inputFormat.getClass());
  reader = getRecordReader(inputFormat, jobConf);
  final MaterializedField keyField = MaterializedField.create(keySchema, KEY_TYPE);
  final MaterializedField valueField = MaterializedField.create(valueSchema, VALUE_TYPE);
  try {
    keyVector = output.addField(keyField, NullableVarBinaryVector.class);
    valueVector = output.addField(valueField, NullableVarBinaryVector.class);
  } catch (SchemaChangeException sce) {
    throw new ExecutionSetupException("Error in setting up sequencefile reader.", sce);
  }
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Override
protected void readField(long recordsToReadInThisPass) {
  this.bytebuf = pageReader.pageData;
  if (usingDictionary) {
    NullableVarBinaryVector.Mutator mutator =  valueVec.getMutator();
    Binary currDictValToWrite;
    for (int i = 0; i < recordsReadInThisIteration; i++){
      currDictValToWrite = pageReader.dictionaryValueReader.readBytes();
      ByteBuffer buf = currDictValToWrite.toByteBuffer();
      mutator.setSafe(valuesReadInCurrentPass + i, buf, buf.position(),
          currDictValToWrite.length());
    }
    // Set the write Index. The next page that gets read might be a page that does not use dictionary encoding
    // and we will go into the else condition below. The readField method of the parent class requires the
    // writer index to be set correctly.
    int writerIndex = castedBaseVector.getBuffer().writerIndex();
    castedBaseVector.getBuffer().setIndex(0, writerIndex + (int)readLength);
  } else {
    super.readField(recordsToReadInThisPass);
    // TODO - replace this with fixed binary type in drill
    // for now we need to write the lengths of each value
    int byteLength = dataTypeLengthInBits / 8;
    for (int i = 0; i < recordsToReadInThisPass; i++) {
      valueVec.getMutator().setValueLengthSafe(valuesReadInCurrentPass + i, byteLength);
    }
  }
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Test
public void testByteSubstring(@Injectable final DrillbitContext bitContext,
                                @Injectable UserClientConnection connection) throws Throwable {
  mockDrillbitContext(bitContext);

  final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
  final PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(DrillFileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8));
  final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
  final FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
  final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));

  while(exec.next()) {
    final NullableVarBinaryVector c1 = exec.getValueVectorById(new SchemaPath("col3", ExpressionPosition.UNKNOWN), NullableVarBinaryVector.class);
    final NullableVarBinaryVector.Accessor a1 = c1.getAccessor();

    int count = 0;
    for(int i = 0; i < c1.getAccessor().getValueCount(); i++) {
      if (!a1.isNull(i)) {
        final NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
        a1.get(i, holder);
        assertEquals("aa", StringFunctionHelpers.toStringFromUTF8(holder.start,  holder.end,  holder.buffer));
        ++count;
      }
    }
    assertEquals(50, count);
  }

  if(context.getFailureCause() != null) {
    throw context.getFailureCause();
  }
  assertTrue(!context.isFailed());
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
NullableFixedBinaryReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                               ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableVarBinaryVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  castedVector = v;
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
static VarLengthValuesColumn getReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                                        ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v,
                                        SchemaElement schemaElement
) throws ExecutionSetupException {
  ConvertedType convertedType = schemaElement.getConverted_type();
  switch (descriptor.getMaxDefinitionLevel()) {
    case 0:
      if (convertedType == null) {
        return new VarLengthColumnReaders.VarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement);
      }
      switch (convertedType) {
        case UTF8:
          return new VarLengthColumnReaders.VarCharColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarCharVector) v, schemaElement);
        case DECIMAL:
          if (v instanceof Decimal28SparseVector) {
            return new VarLengthColumnReaders.Decimal28Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal28SparseVector) v, schemaElement);
          } else if (v instanceof Decimal38SparseVector) {
            return new VarLengthColumnReaders.Decimal38Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal38SparseVector) v, schemaElement);
          }
        default:
          return new VarLengthColumnReaders.VarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement);
      }
    default:
      if (convertedType == null) {
        return new VarLengthColumnReaders.NullableVarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarBinaryVector) v, schemaElement);
      }

      switch (convertedType) {
        case UTF8:
          return new VarLengthColumnReaders.NullableVarCharColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarCharVector) v, schemaElement);
        case DECIMAL:
          if (v instanceof NullableDecimal28SparseVector) {
            return new VarLengthColumnReaders.NullableDecimal28Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableDecimal28SparseVector) v, schemaElement);
          } else if (v instanceof NullableDecimal38SparseVector) {
            return new VarLengthColumnReaders.NullableDecimal38Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableDecimal38SparseVector) v, schemaElement);
          }
        default:
          return new VarLengthColumnReaders.NullableVarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarBinaryVector) v, schemaElement);
      }
  }
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
public static NullableColumnReader getNullableColumnReader(ParquetRecordReader parentReader, int allocateSize,
                                                           ColumnDescriptor columnDescriptor,
                                                           ColumnChunkMetaData columnChunkMetaData,
                                                           boolean fixedLength,
                                                           ValueVector valueVec,
                                                           SchemaElement schemaElement) throws ExecutionSetupException {
  ConvertedType convertedType = schemaElement.getConverted_type();

  if (! columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
    if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT96) {
      return new NullableFixedByteAlignedReaders.NullableFixedBinaryReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (NullableVarBinaryVector) valueVec, schemaElement);
    }else{
      return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, valueVec, schemaElement);
    }
  } else {
    switch (columnDescriptor.getType()) {
      case INT32:
        if (convertedType == null) {
          return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableIntVector) valueVec, schemaElement);
        }
        switch (convertedType) {
          case DECIMAL:
            return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal9Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal9Vector) valueVec, schemaElement);
          case TIME_MILLIS:
            return new NullableFixedByteAlignedReaders.NullableDictionaryTimeReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeVector)valueVec, schemaElement);
          default:
            throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT32");
        }
      case INT64:
        if (convertedType == null) {
          return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableBigIntVector)valueVec, schemaElement);
        }
        switch (convertedType) {
          case DECIMAL:
            return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal18Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal18Vector)valueVec, schemaElement);
          case TIMESTAMP_MILLIS:
            return new NullableFixedByteAlignedReaders.NullableDictionaryTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeStampVector)valueVec, schemaElement);
          default:
            throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT64");
        }
      case INT96:
         return new NullableFixedByteAlignedReaders.NullableFixedBinaryReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, true, (NullableVarBinaryVector) valueVec, schemaElement);
      case FLOAT:
        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat4Vector)valueVec, schemaElement);
      case DOUBLE:
        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat8Vector)valueVec, schemaElement);
      default:
        throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
    }
  }
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
public VarBinaryCopier(int columnIndex, ResultSet result, NullableVarBinaryVector.Mutator mutator) {
  super(columnIndex, result, mutator);
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
  ((NullableVarBinaryVector) outputVV).getMutator().setSafe(outputIndex, value, 0, value.length);
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
NullableFixedBinaryReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                               ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableVarBinaryVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

import org.apache.drill.exec.vector.NullableVarBinaryVector; //导入依赖的package包/类
static VarLengthValuesColumn<?> getReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                                        ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v,
                                        SchemaElement schemaElement
) throws ExecutionSetupException {
  ConvertedType convertedType = schemaElement.getConverted_type();
  switch (descriptor.getMaxDefinitionLevel()) {
    case 0:
      if (convertedType == null) {
        return new VarLengthColumnReaders.VarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement);
      }
      switch (convertedType) {
        case UTF8:
        case ENUM:
          return new VarLengthColumnReaders.VarCharColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarCharVector) v, schemaElement);
        case DECIMAL:
          if (v instanceof Decimal28SparseVector) {
            return new VarLengthColumnReaders.Decimal28Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal28SparseVector) v, schemaElement);
          } else if (v instanceof Decimal38SparseVector) {
            return new VarLengthColumnReaders.Decimal38Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal38SparseVector) v, schemaElement);
          }
        default:
          return new VarLengthColumnReaders.VarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement);
      }
    default:
      if (convertedType == null) {
        return new VarLengthColumnReaders.NullableVarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarBinaryVector) v, schemaElement);
      }

      switch (convertedType) {
        case UTF8:
        case ENUM:
          return new VarLengthColumnReaders.NullableVarCharColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarCharVector) v, schemaElement);
        case DECIMAL:
          if (v instanceof NullableDecimal28SparseVector) {
            return new VarLengthColumnReaders.NullableDecimal28Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableDecimal28SparseVector) v, schemaElement);
          } else if (v instanceof NullableDecimal38SparseVector) {
            return new VarLengthColumnReaders.NullableDecimal38Column(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableDecimal38SparseVector) v, schemaElement);
          }
        default:
          return new VarLengthColumnReaders.NullableVarBinaryColumn(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (NullableVarBinaryVector) v, schemaElement);
      }
  }
}