Java DataType.UNKNOWN属性代码示例

/**
 * Copy Constructor.
 * 
 * @param fs
 *           Source FieldSchema
 * 
 */
public FieldSchema(FieldSchema fs)  {
    if(null != fs) {
        alias = fs.alias;
        if(null != fs.schema) {
            schema = new Schema(fs.schema);
        } else {
            schema = null;
        }
        type = fs.type;
    } else {
        alias = null;
        schema = null;
        type = DataType.UNKNOWN;
    }
    canonicalName = CanonicalNamer.getNewName();
}
 

private void mergeMROperProperties(MapReduceOper from, MapReduceOper to) {

        if (from.isEndOfAllInputSetInMap()) {
            to.setEndOfAllInputInMap(true);
        }

        if (from.isEndOfAllInputSetInReduce()) {
            to.setEndOfAllInputInReduce(true);
        }
        
        if (from.getRequestedParallelism() > to.getRequestedParallelism()) {
            to.requestedParallelism = from.requestedParallelism;
        }

        if (!from.UDFs.isEmpty()) {
            to.UDFs.addAll(from.UDFs);
        }

        if (from.needsDistinctCombiner()) {
            to.setNeedsDistinctCombiner(true);
        }

        if (to.mapKeyType == DataType.UNKNOWN) {
            to.mapKeyType = from.mapKeyType;
        }
    }
 

private void mergeMROperProperties(MapReduceOper from, MapReduceOper to) {

        if (from.isEndOfAllInputSetInMap()) {
            to.setEndOfAllInputInMap(true);
        }

        if (from.isEndOfAllInputSetInReduce()) {
            to.setEndOfAllInputInReduce(true);
        }

        if (from.getRequestedParallelism() > to.getRequestedParallelism()) {
            to.requestedParallelism = from.requestedParallelism;
        }

        if (!from.UDFs.isEmpty()) {
            to.UDFs.addAll(from.UDFs);
        }

        if (from.needsDistinctCombiner()) {
            to.setNeedsDistinctCombiner(true);
        }

        if (to.mapKeyType == DataType.UNKNOWN) {
            to.mapKeyType = from.mapKeyType;
        }
    }
 

@Override
public Schema outputSchema(Schema inputSch) {
    try {
        if (returnType == DataType.UNKNOWN) {
            return Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
        } else {
            Schema outputTupleSchema = new Schema(new Schema.FieldSchema(returnName, returnType));
            return new Schema(new Schema.FieldSchema(
                    getSchemaName(this.getClass().getName().toLowerCase(), inputSch),
                        outputTupleSchema, 
                        DataType.BAG));
        }
    } catch (FrontendException fe) {
        throw new RuntimeException("Unable to create nested schema", fe);
    }
}
 

@Override
public Tuple getNext() throws IOException {
  boolean next = false;
  try {
    next = reader.nextKeyValue();
  } catch (InterruptedException e) {
    throw new IOException(e);
  }
  
  if (!next) return null;
  
  key = reader.getCurrentKey();
  value = reader.getCurrentValue();
  
  if (keyType == DataType.UNKNOWN && key != null) {
      setKeyType(key.getClass());
  }
  if (valType == DataType.UNKNOWN && value != null) {
      setValueType(value.getClass());
  }
  
  mProtoTuple.add(translateWritableToPigDataType(key, keyType));
  mProtoTuple.add(translateWritableToPigDataType(value, valType));
  Tuple t =  mTupleFactory.newTuple(mProtoTuple);
  mProtoTuple.clear();
  return t;
}
 

@Override
public Tuple getNext() throws IOException {
  boolean next = false;
  try {
    next = reader.nextKeyValue();
  } catch (InterruptedException e) {
    throw new IOException(e);
  }

  if (!next) return null;

  key = reader.getCurrentKey();
  value = reader.getCurrentValue();

  if (keyType == DataType.UNKNOWN && key != null) {
      setKeyType(key.getClass());
  }
  if (valType == DataType.UNKNOWN && value != null) {
      setValueType(value.getClass());
  }

  mProtoTuple.add(translateWritableToPigDataType(key, keyType));
  mProtoTuple.add(translateWritableToPigDataType(value, valType));
  Tuple t =  mTupleFactory.newTuple(mProtoTuple);
  mProtoTuple.clear();
  return t;
}
 

private boolean isNullOrUnknownType(FieldSchema fs) {
    return (fs.type == DataType.NULL || fs.type == DataType.UNKNOWN);
}
 

private Object readField(String line, ResourceFieldSchema field, FixedWidthField column) 
                         throws IOException, IllegalArgumentException {

    int start = column.start;
    int end = Math.min(column.end, line.length());

    if (start > line.length())
        return null;

    if (end <= start)
        return null;

    String s  = line.substring(start, end);
    String sTrim = s.trim();

    switch (field.getType()) {
        case DataType.UNKNOWN:
        case DataType.BYTEARRAY:
        case DataType.CHARARRAY:
            if (s.trim().length() == 0)
                return null;
            return s.trim();

        case DataType.BOOLEAN:
            return Boolean.parseBoolean(sTrim);

        case DataType.INTEGER:
            return Integer.parseInt(sTrim);

        case DataType.LONG:
            return Long.parseLong(sTrim);

        case DataType.FLOAT:
            return Float.parseFloat(sTrim);
        
        case DataType.DOUBLE:
            return Double.parseDouble(sTrim);

        case DataType.DATETIME:
            return (new DateTime(sTrim)).toDateTime(DateTimeZone.UTC);

        case DataType.MAP:
        case DataType.TUPLE:
        case DataType.BAG:
            throw new IllegalArgumentException("Object types (map, tuple, bag) are not supported by FixedWidthLoader");
        
        default:
            throw new IllegalArgumentException(
                "Unknown type in input schema: " + field.getType());
    }
}
 

public Over() {
    initialized = false;
    udfArgs = null;
    func = null;
    returnType = DataType.UNKNOWN;
}
 

/**
 * 
 * @param inputColumn
 *            A pair of integers representing the input number and the
 *            input column number
 */
public Column(Pair<Integer, Integer> inputColumn) {
    this(inputColumn, false, DataType.UNKNOWN);
}