Java ArrowType类代码示例

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@Override
protected String visitGeneric(ArrowType type) {
  String typeStr = describe(type);
  if(!includeName) {
    return typeStr;
  }

  return field.getName() + "::" + typeStr;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
public static CompleteType union(Iterable<Field> fields){
  ImmutableList<Field> listOfFields = ImmutableList.copyOf(fields);
  int[] typeIds = new int[listOfFields.size()];
  for (int i =0; i < typeIds.length; i++) {
    typeIds[i] = MajorTypeHelper.getArrowMinorType(CompleteType.fromField(listOfFields.get(i)).toMinorType()).ordinal();
  }
  return new CompleteType(new ArrowType.Union(UnionMode.Sparse, typeIds), listOfFields);
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@JsonCreator
public GlobalDictionaryFieldInfo(
  @JsonProperty("dictionaryVersion") long dictionaryVersion,
  @JsonProperty("fieldName") String fieldName,
  @JsonProperty("storagePluginId") StoragePluginId storagePluginId,
  @JsonProperty("arrowType") ArrowType arrowType,
  @JsonProperty("dictionaryPath") String dictionaryPath) {
  this(dictionaryVersion, fieldName, storagePluginId, arrowType, dictionaryPath, null);
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@Override
public Field visit(ArrowType.List type) {
  if(field.getName().equals(column.getAsUnescapedPath())){
    Field child = field.getChildren().get(0);
    return new Field(field.getName(), child.isNullable(), child.getType(), child.getChildren());
  }
  return field;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@Override
public Field visitGeneric(ArrowType type) {
  if(field.getName().equals(column.getAsUnescapedPath())){
    throw UserException.validationError().message("You're trying to flatten a field that is not a list. The offending field is %s.", Describer.describe(field)).build(logger);
  }
  return super.visitGeneric(type);
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static VectorContainer buildIntegerGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(32, true), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final NullableIntVector intVector = input.addOrGet(field);
  intVector.allocateNew();
  final SortedSet<Integer> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToInt(i));
    }
  }
  if (existingDict != null) {
    final NullableIntVector existingDictValues = existingDict.getValueAccessorById(NullableIntVector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.getAccessor().get(i));
    }
  }
  final Iterator<Integer> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    intVector.getMutator().setSafe(recordCount++, iter.next());
  }
  intVector.getMutator().setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static VectorContainer buildLongGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Int(64, true), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final NullableBigIntVector longVector = input.addOrGet(field);
  longVector.allocateNew();
  SortedSet<Long> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToLong(i));
    }
  }
  if (existingDict != null) {
    final NullableBigIntVector existingDictValues = existingDict.getValueAccessorById(NullableBigIntVector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.getAccessor().get(i));
    }
  }
  final Iterator<Long> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    longVector.getMutator().setSafe(recordCount++, iter.next());
  }
  longVector.getMutator().setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static VectorContainer buildDoubleGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final NullableFloat8Vector doubleVector = input.addOrGet(field);
  doubleVector.allocateNew();
  SortedSet<Double> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToDouble(i));
    }
  }
  if (existingDict != null) {
    final NullableFloat8Vector existingDictValues = existingDict.getValueAccessorById(NullableFloat8Vector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.getAccessor().get(i));
    }
  }
  final Iterator<Double> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    doubleVector.getMutator().setSafe(recordCount++, iter.next());
  }
  doubleVector.getMutator().setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static VectorContainer buildFloatGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final NullableFloat4Vector floatVector = input.addOrGet(field);
  floatVector.allocateNew();
  SortedSet<Float> values = Sets.newTreeSet();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToFloat(i));
    }
  }
  if (existingDict != null) {
    final NullableFloat4Vector existingDictValues = existingDict.getValueAccessorById(NullableFloat4Vector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(existingDictValues.getAccessor().get(i));
    }
  }
  final Iterator<Float> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    floatVector.getMutator().setSafe(recordCount++, iter.next());
  }
  floatVector.getMutator().setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static VectorContainer buildBinaryGlobalDictionary(List<Dictionary> dictionaries, VectorContainer existingDict, ColumnDescriptor columnDescriptor, BufferAllocator bufferAllocator) {
  final Field field = new Field(SchemaPath.getCompoundPath(columnDescriptor.getPath()).getAsUnescapedPath(), true, new ArrowType.Binary(), null);
  final VectorContainer input = new VectorContainer(bufferAllocator);
  final NullableVarBinaryVector binaryVector = input.addOrGet(field);
  binaryVector.allocateNew();
  final SortedSet<Binary> values = new TreeSet<>();
  for (Dictionary dictionary : dictionaries) {
    for (int i = 0; i <= dictionary.getMaxId(); ++i) {
      values.add(dictionary.decodeToBinary(i));
    }
  }
  if (existingDict != null) {
    final NullableVarBinaryVector existingDictValues = existingDict.getValueAccessorById(NullableVarBinaryVector.class, 0).getValueVector();
    for (int i = 0; i < existingDict.getRecordCount(); ++i) {
      values.add(Binary.fromConstantByteArray(existingDictValues.getAccessor().get(i)));
    }
  }
  final Iterator<Binary> iter = values.iterator();
  int recordCount = 0;
  while (iter.hasNext()) {
    final byte[] data = iter.next().getBytes();
    binaryVector.getMutator().setSafe(recordCount++, data, 0, data.length);
  }
  binaryVector.getMutator().setValueCount(recordCount);
  input.setRecordCount(recordCount);
  input.buildSchema(BatchSchema.SelectionVectorMode.NONE);
  return input;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
public CompleteType getOutputType(CompleteType baseReturn, List<LogicalExpression> args) {
  Preconditions.checkArgument(args.size() == 1);
  CompleteType type = args.get(0).getCompleteType();
  if(!type.isStruct()){
    throw UserException.functionError().message("The kvgen function can only be used when operating against maps. The type you were attempting to apply it to was a %s.", type.toString()).build(logger);
  }

  List<Field> children = type.getChildren();

  if(children.isEmpty()){
    throw UserException.functionError().message("The kvgen function can only be used when operating against maps. The type you were attempting to apply it to was a %s.", type.toString()).build(logger);
  }


  CompleteType valueType = CompleteType.fromField(children.get(0));
  for(int i =0; i < children.size(); i++){
    valueType = valueType.merge(CompleteType.fromField(children.get(i)));
  }

  return new CompleteType(
      ArrowType.List.INSTANCE,
      new CompleteType(
          ArrowType.Struct.INSTANCE,
          CompleteType.VARCHAR.toField(MappifyUtility.fieldKey),
          valueType.toField(MappifyUtility.fieldValue))
      .toField("list"));
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private static SourceTableDefinition newDataset(final String dsPath) {
  final List<String> path = SqlUtils.parseSchemaPath(dsPath);

  SourceTableDefinition ret = mock(SourceTableDefinition.class);
  NamespaceKey datasetName = new NamespaceKey(path);
  when(ret.getName()).thenReturn(datasetName);

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(new Field("string", FieldType.nullable(ArrowType.Utf8.INSTANCE), null))
    .build();
  DatasetConfig dsConfig = new DatasetConfig()
    .setName(Util.last(path))
    .setFullPathList(path)
    .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE)
    .setRecordSchema(ByteString.EMPTY)
    .setPhysicalDataset(
      new PhysicalDataset()
        .setFormatSettings(null))
    .setSchemaVersion(DatasetHelper.CURRENT_VERSION)
    .setRecordSchema(schema.toByteString())
    .setReadDefinition(new ReadDefinition());
  try {
    when(ret.getDataset()).thenReturn(dsConfig);
  } catch (Exception ex) {}

  when(ret.getType()).thenReturn(DatasetType.PHYSICAL_DATASET_SOURCE_FILE);

  when(ret.isSaveable()).thenReturn(true);
  return ret;
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
private void assertDatasetSchemasDefined(List<NamespaceKey> datasetKeys) throws Exception {
  for (NamespaceKey datasetKey : datasetKeys) {
    DatasetConfig dsConfig = ns.getDataset(datasetKey);
    BatchSchema schema = BatchSchema.deserialize(dsConfig.getRecordSchema());
    assertEquals(schema.getFieldCount(), 1);
    assertEquals(schema.getColumn(0).getName(), "string");
    assertEquals(schema.getColumn(0).getType(), ArrowType.Utf8.INSTANCE);
  }
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@Test
public void trueWhenAllColumnsAreSelected() {
  BatchSchema schema = mock(BatchSchema.class);
  when(schema.iterator())
      .thenReturn(Lists.newArrayList(Field.nullable("a1", new ArrowType.Bool())).iterator());
  assertTrue(EasyScanOperatorCreator.selectsAllColumns(schema,
      Lists.<SchemaPath>newArrayList(SchemaPath.getSimplePath("a1"))));
}
 

import org.apache.arrow.vector.types.pojo.ArrowType; //导入依赖的package包/类
@Test
public void selectionIgnoresIncremental() {
  BatchSchema schema = mock(BatchSchema.class);
  when(schema.iterator())
      .thenReturn(Lists.newArrayList(Field.nullable("a1", new ArrowType.Bool()),
          Field.nullable(IncrementalUpdateUtils.UPDATE_COLUMN, new ArrowType.Bool())).iterator());
  assertTrue(EasyScanOperatorCreator.selectsAllColumns(schema,
    Lists.<SchemaPath>newArrayList(SchemaPath.getSimplePath("a1"))));
}