Java DataMode.REPEATED屬性代碼示例

private parquet.schema.Type getType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  DataMode dataMode = field.getType().getMode();
  switch(minorType) {
    case MAP:
      List<parquet.schema.Type> types = Lists.newArrayList();
      for (MaterializedField childField : field.getChildren()) {
        types.add(getType(childField));
      }
      return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getLastName(), types);
    case LIST:
      throw new UnsupportedOperationException("Unsupported type " + minorType);
    default:
      return getPrimitiveType(field);
  }
}
 

protected R apply(ValueVector vector, A arg) {
  MaterializedField schema = vector.getField();
  MajorType majorType = schema.getType();
  MinorType type = majorType.getMinorType();
  DataMode mode = majorType.getMode();
  switch (type) {
  case MAP:
    if (mode == DataMode.REPEATED) {
      return visitRepeatedMap((RepeatedMapVector) vector, arg);
    } else {
      return visitMap((AbstractMapVector) vector, arg);
    }
  case LIST:
    if (mode == DataMode.REPEATED) {
      return visitRepeatedList((RepeatedListVector) vector, arg);
    } else {
      return visitList((ListVector) vector, arg);
    }
  default:
    if (mode == DataMode.REPEATED) {
      return visitRepeatedPrimitive((BaseRepeatedValueVector) vector, arg);
    } else {
      return visitPrimitive(vector, arg);
    }
  }
}
 

@Override
public String toString() {
  StringBuilder buf = new StringBuilder()
      .append(prefix)
      .append(metadata.getName())
      .append("(type: ")
      .append(metadata.getType().getMode().name())
      .append(" ")
      .append(metadata.getType().getMinorType().name())
      .append(", count: ")
      .append(valueCount);
  if (metadata.getDataMode() == DataMode.REPEATED) {
    buf.append(", elements: ")
       .append(elementCount)
       .append(", per-array: ")
       .append(estElementCountPerArray);
  }
  buf .append(", std size: ")
      .append(stdSize)
      .append(", actual size: ")
      .append(estSize)
      .append(", data size: ")
      .append(dataSize)
      .append(")");
  return buf.toString();
}
 

private Type getType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  DataMode dataMode = field.getType().getMode();
  switch(minorType) {
    case MAP:
      List<Type> types = Lists.newArrayList();
      for (MaterializedField childField : field.getChildren()) {
        types.add(getType(childField));
      }
      return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types);
    case LIST:
      throw new UnsupportedOperationException("Unsupported type " + minorType);
    default:
      return getPrimitiveType(field);
  }
}
 

private MajorType getLastPathType() {
  if((this.getField().getType().getMinorType() == MinorType.LIST  &&
      this.getField().getType().getMode() == DataMode.REPEATED)) {  // Use Repeated scalar type instead of Required List.
    VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
    ValueVector v = vord.vector;
    if (! (v instanceof  AbstractContainerVector)) {
      return v.getField().getType();
    }
  } else if (this.getField().getType().getMinorType() == MinorType.MAP  &&
      this.getField().getType().getMode() == DataMode.REPEATED) {  // Use Required Map
    return this.getField().getType().toBuilder().setMode(DataMode.REQUIRED).build();
  }

  return this.getField().getType();
}
 

public static boolean isCastableWithNullHandling(MajorType from, MajorType to, NullHandling nullHandling) {
  if ((from.getMode() == DataMode.REPEATED || to.getMode() == DataMode.REPEATED) && from.getMode() != to.getMode()) {
    return false;
  }
  if (nullHandling == NullHandling.INTERNAL && from.getMode() != to.getMode()) {
    return false;
  }
  return isCastable(from.getMinorType(), to.getMinorType());
}
 

@Override
public void addField(Field field) {
    if (field.getFieldType().getMode() == DataMode.REPEATED || fields.isEmpty() || !isSingleTyped() ||
            !Iterables.getOnlyElement(fields).equals(field.getFieldType())) {
        fields.add(field);
    }
}
 

private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
  if (column.getMaxRepetitionLevel() > 0 ) {
    return DataMode.REPEATED;
  } else if (column.getMaxDefinitionLevel() == 0) {
    return TypeProtos.DataMode.REQUIRED;
  } else {
    return TypeProtos.DataMode.OPTIONAL;
  }
}
 

@Override
public Void visitIfExpression(IfExpression ifExpr, ErrorCollector errors) throws RuntimeException {
  // confirm that all conditions are required boolean values.
  IfCondition cond = ifExpr.ifCondition;
  MajorType majorType = cond.condition.getMajorType();
  if ( majorType
      .getMinorType() != MinorType.BIT) {
    errors
        .addGeneralError(
            cond.condition.getPosition(),
            String
                .format(
                    "Failure composing If Expression.  All conditions must return a boolean type.  Condition was of Type %s.",
                    majorType.getMinorType()));
  }

  // confirm that all outcomes are the same type.
  final MajorType mt = ifExpr.elseExpression.getMajorType();
  cond = ifExpr.ifCondition;
  MajorType innerT = cond.expression.getMajorType();
  if ((innerT.getMode() == DataMode.REPEATED && mt.getMode() != DataMode.REPEATED) || //
      ((innerT.getMinorType() != mt.getMinorType()) &&
      (innerT.getMode() != DataMode.OPTIONAL && mt.getMode() != DataMode.OPTIONAL &&
      (innerT.getMinorType() != MinorType.NULL && mt.getMinorType() != MinorType.NULL)))) {
    errors
        .addGeneralError(
            cond.condition.getPosition(),
            String
                .format(
                    "Failure composing If Expression.  All expressions must return the same MinorType as the else expression.  The if condition returned type type %s but the else expression was of type %s",
                    innerT, mt));
  }
  return null;
}
 

private AbstractObjectReader buildVectorReader(ValueVector vector, VectorDescrip descrip) {
  MajorType type = vector.getField().getType();
  if (type.getMinorType() == MinorType.MAP) {
    if (type.getMode() == DataMode.REPEATED) {
      return buildMapArrayReader((RepeatedMapVector) vector, descrip);
    } else {
      return buildMapReader((MapVector) vector, descrip);
    }
  } else {
    return buildPrimitiveReader(vector, descrip);
  }
}
 

@SuppressWarnings("unchecked")
private AbstractObjectReader buildVectorReader(VectorWrapper<?> vw, VectorDescrip descrip) {
  MajorType type = vw.getField().getType();
  if (type.getMinorType() == MinorType.MAP) {
    if (type.getMode() == DataMode.REPEATED) {
      return buildMapArrayReader((HyperVectorWrapper<? extends AbstractMapVector>) vw, descrip);
    } else {
      return buildMapReader((HyperVectorWrapper<? extends AbstractMapVector>) vw, descrip);
    }
  } else {
    return buildPrimitiveReader(vw, descrip);
  }
}
 

public ColumnSize(ValueVector v, String prefix) {
  this.prefix = prefix;
  valueCount = v.getAccessor().getValueCount();
  metadata = v.getField();
  isVariableWidth = v instanceof VariableWidthVector;

  // The amount of memory consumed by the payload: the actual
  // data stored in the vectors.

  if (v.getField().getDataMode() == DataMode.REPEATED) {
    elementCount = buildRepeated(v);
    estElementCountPerArray = valueCount == 0 ? 0 : elementCount * 1.0f / valueCount;
  } else {
    elementCount = 1;
    estElementCountPerArray = 1;
  }
  switch (metadata.getType().getMinorType()) {
  case LIST:
    buildList(v);
    break;
  case MAP:
  case UNION:
    // No standard size for Union type
    dataSize = v.getPayloadByteCount(valueCount);
    break;
  default:
    dataSize = v.getPayloadByteCount(valueCount);
    stdSize = TypeHelper.getSize(metadata.getType()) * elementCount;
  }
  estSize = safeDivide(dataSize, valueCount);
}
 

/**
 * Add a single vector initializer to a collection for the entire batch.
 * Uses the observed column size information to predict the size needed
 * when allocating a new vector for the same data. Adds a hint only for
 * variable-width or repeated types; no extra information is needed for
 * fixed width, non-repeated columns.
 *
 * @param initializer the vector initializer to hold the hints
 * for this column
 */

private void buildVectorInitializer(VectorInitializer initializer) {
  int width = 0;
  switch(metadata.getType().getMinorType()) {
  case VAR16CHAR:
  case VARBINARY:
  case VARCHAR:

    // Subtract out the offset vector width
    width = estSize - 4;

    // Subtract out the bits (is-set) vector width
    if (metadata.getDataMode() == DataMode.OPTIONAL) {
      width -= 1;
    }
    break;
  default:
    break;
  }
  String name = prefix + metadata.getName();
  if (metadata.getDataMode() == DataMode.REPEATED) {
    if (width > 0) {
      // Estimated width is width of entire column. Divide
      // by element count to get per-element size.
      initializer.variableWidthArray(name, width / estElementCountPerArray, estElementCountPerArray);
    } else {
      initializer.fixedWidthArray(name, estElementCountPerArray);
    }
  }
  else if (width > 0) {
    initializer.variableWidth(name, width);
  }
}
 

private Type getType(MajorType t) {

    if(t.getMode() == DataMode.REPEATED){
      throw UserException
      .dataWriteError()
      .message("Kudu does not support array types.")
      .build(logger);
    }

    switch (t.getMinorType()) {
    case BIGINT:
      return Type.INT64;
    case BIT:
      return Type.BOOL;
    case FLOAT4:
      return Type.FLOAT;
    case FLOAT8:
      return Type.DOUBLE;
    case INT:
      return Type.INT32;
    case TIMESTAMP:
      return Type.UNIXTIME_MICROS;
    case VARCHAR:
      return Type.STRING;
    case VARBINARY:
      return Type.BINARY;
    default:
      throw UserException
        .dataWriteError()
          .message("Data type: '%s' not supported in Kudu.", t.getMinorType().name())
          .build(logger);
    }
  }
 

private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
  if (isRepeated()) {
    return DataMode.REPEATED;
  } else if (column.getMaxDefinitionLevel() == 0) {
    return TypeProtos.DataMode.REQUIRED;
  } else {
    return TypeProtos.DataMode.OPTIONAL;
  }
}