Java DataType.INTEGER属性代码示例

@Override
public ResourceSchema getSchema(final String s, final Job job) throws IOException {
    ResourceSchema.ResourceFieldSchema remoteAddrFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("remoteAddr", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema accessTimeFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("accessTime", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema methodFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("method", DataType.BYTEARRAY));

    ResourceSchema.ResourceFieldSchema urlFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("url", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema protocolFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("protocol", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema agentFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("agent", DataType.BYTEARRAY));

    ResourceSchema.ResourceFieldSchema referFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("refer", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema statusFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("status", DataType.INTEGER));
    ResourceSchema.ResourceFieldSchema lengthFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("length", DataType.INTEGER));

    ResourceSchema resourceSchema = new ResourceSchema();
    resourceSchema.setFields(new ResourceSchema.ResourceFieldSchema[]{remoteAddrFieldSchema, accessTimeFieldSchema, methodFieldSchema, urlFieldSchema, protocolFieldSchema, agentFieldSchema, referFieldSchema, statusFieldSchema, lengthFieldSchema});
    return resourceSchema;
}
 

/**
 * Convert Pig primitive type to Avro type
 * 
 */
protected static Schema convertPrimitiveType(byte pigType) throws IOException {

    if (pigType == DataType.BOOLEAN) {
        return AvroStorageUtils.BooleanSchema;
    } else if (pigType == DataType.BYTEARRAY) {
        return AvroStorageUtils.BytesSchema;
    } else if (pigType == DataType.CHARARRAY
                                    || pigType == DataType.BIGCHARARRAY) {
        return AvroStorageUtils.StringSchema;
    } else if (pigType == DataType.DOUBLE) {
        return AvroStorageUtils.DoubleSchema;
    } else if (pigType == DataType.FLOAT) {
        return AvroStorageUtils.FloatSchema;
    } else if (pigType == DataType.INTEGER) {
        return AvroStorageUtils.IntSchema;
    } else if (pigType == DataType.LONG) {
        return AvroStorageUtils.LongSchema;
    } else
        throw new IOException("unsupported pig type:"
                                        + DataType.findTypeName(pigType));

}
 

@Override
public List<String> getPredicateFields(String s, Job job) throws IOException {
  if(!job.getConfiguration().getBoolean(ENABLE_PREDICATE_FILTER_PUSHDOWN, DEFAULT_PREDICATE_PUSHDOWN_ENABLED)) {
    return null;
  }

  List<String> fields = new ArrayList<String>();

  for(FieldSchema field : schema.getFields()) {
    switch(field.type) {
      case DataType.BOOLEAN:
      case DataType.INTEGER:
      case DataType.LONG:
      case DataType.FLOAT:
      case DataType.DOUBLE:
      case DataType.CHARARRAY:
        fields.add(field.alias);
        break;
      default:
        // Skip BYTEARRAY, TUPLE, MAP, BAG, DATETIME, BIGINTEGER, BIGDECIMAL
        break;
    }
  }

  return fields;
}
 

/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation2() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
 

private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
    if (data == null) {
        return null;
    }
    byte type = detail.type;
    switch (type) {
        case DataType.BOOLEAN:
            return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
        case DataType.INTEGER:
        case DataType.LONG:
            return TypeSystem.asTypedObject(DataType.toLong(data, type));
        case DataType.FLOAT:
        case DataType.DOUBLE:
            return TypeSystem.asTypedObject(DataType.toDouble(data, type));
        case DataType.DATETIME:
            return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
        case DataType.BYTE:
        case DataType.BYTEARRAY:
        case DataType.CHARARRAY:
            return TypeSystem.asTypedObject(DataType.toString(data, type));
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
        default:
            //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
            return null;
    }
}
 

/**
 * Validates format of input schema and returns a matching schema
 * @param input Expects input to be a bag of sketches: <tt>(n, k, {(samples...)})</tt>
 * @return Schema based on the
 */
@Override
public Schema outputSchema(final Schema input) {
  if (input != null && input.size() > 0) {
    try {
      Schema source = input;

      // if we have a bag, grab one level down to get a tuple
      if (source.size() == 1 && source.getField(0).type == DataType.BAG) {
        source = source.getField(0).schema;
      }

      if (source.size() == 1 && source.getField(0).type == DataType.TUPLE) {
        source = source.getField(0).schema;
      }

      final List<Schema.FieldSchema> fields = source.getFields();
      if (fields.size() == 3
              && fields.get(0).type == DataType.LONG
              && fields.get(1).type == DataType.INTEGER
              && fields.get(2).type == DataType.BAG) {
        return new Schema(new Schema.FieldSchema(getSchemaName(this
                .getClass().getName().toLowerCase(), source), source, DataType.TUPLE));
      }
    } catch (final FrontendException e) {
      throw new RuntimeException(e);
    }
  }

  return null;
}
 

private Result getResult(ExpressionOperator op) throws ExecException {
    Result res = ERR_RESULT;
    switch (op.getResultType()) {
    case DataType.BAG:
    case DataType.BOOLEAN:
    case DataType.BYTEARRAY:
    case DataType.CHARARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.LONG:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.DATETIME:
    case DataType.MAP:
    case DataType.TUPLE:
        res = op.getNext(op.getResultType());
        break;
    default:
        String msg = "Invalid result type: "
                + DataType.findType(op.getResultType());
        throw new ExecException(msg, 2270, PigException.BUG);
    }

    // allow null as group by key
    if (res.returnStatus == POStatus.STATUS_OK
            || res.returnStatus == POStatus.STATUS_NULL) {
        return res;
    }
    return ERR_RESULT;
}
 

@Override
public Schema outputSchema(Schema input) {
  try {
      return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.INTEGER));
  } catch (Exception e) {
    return null;
  }
}
 

protected Number mod(Number a, Number b, byte dataType) throws ExecException {
    switch(dataType) {
    case DataType.INTEGER:
        return Integer.valueOf((Integer) a % (Integer) b);
    case DataType.LONG:
        return Long.valueOf((Long) a % (Long) b);
    case DataType.BIGINTEGER:
        return ((BigInteger)a).mod((BigInteger)b);
    default:
        throw new ExecException("called on unsupported Number class " + DataType.findTypeName(dataType));
    }
}
 

@Override
public Schema outputSchema(Schema input) {
  try {
    if (!(input.size() == 2 || input.size() == 3))
    {
      throw new RuntimeException("Expected input to have two or three fields");
    }
    
    Schema.FieldSchema inputFieldSchema = input.getField(0);

    if (inputFieldSchema.type != DataType.BAG) {
      throw new RuntimeException("Expected a BAG as first input, got: "+inputFieldSchema.type);
    }
    
    if (input.getField(1).type != DataType.INTEGER) {
      throw new RuntimeException("Expected an INT as second input, got: "+input.getField(1).type);
    }      
    
    if (input.size() == 3 && !(input.getField(2).type == DataType.INTEGER || input.getField(2).type == DataType.LONG)) {
      throw new RuntimeException("Expected an INT or LONG as second input, got: "+input.getField(2).type);
    }
    
    return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
                                             inputFieldSchema.schema, DataType.BAG));    
  } catch (FrontendException e) {
    e.printStackTrace();
    throw new RuntimeException(e);
  }
}
 

protected byte inferPigDataType(Type t) {
  if (t == DataByteArray.class) return DataType.BYTEARRAY;
  else if (t == Text.class) return DataType.CHARARRAY;
  else if (t == IntWritable.class) return DataType.INTEGER;
  else if (t == LongWritable.class) return DataType.LONG;
  else if (t == FloatWritable.class) return DataType.FLOAT;
  else if (t == DoubleWritable.class) return DataType.DOUBLE;
  else if (t == BooleanWritable.class) return DataType.BOOLEAN;
  else if (t == ByteWritable.class) return DataType.BYTE;
  else if (t == DateTimeWritable.class) return DataType.DATETIME;
  // not doing maps or other complex types for now
  else return DataType.ERROR;
}
 

public static byte findTypeFromNullableWritable(PigNullableWritable o) throws ExecException {
    if (o instanceof NullableBooleanWritable)
        return DataType.BOOLEAN;
    else if (o instanceof NullableBytesWritable)
        return DataType.BYTEARRAY;
    else if (o instanceof NullableText)
        return DataType.CHARARRAY;
    else if (o instanceof NullableFloatWritable)
        return DataType.FLOAT;
    else if (o instanceof NullableDoubleWritable)
        return DataType.DOUBLE;
    else if (o instanceof NullableIntWritable)
        return DataType.INTEGER;
    else if (o instanceof NullableLongWritable)
        return DataType.LONG;
    else if (o instanceof NullableBigIntegerWritable)
        return DataType.BIGINTEGER;
    else if (o instanceof NullableBigDecimalWritable)
        return DataType.BIGDECIMAL;
    else if (o instanceof NullableDateTimeWritable)
        return DataType.DATETIME;
    else if (o instanceof NullableBag)
        return DataType.BAG;
    else if (o instanceof NullableTuple)
        return DataType.TUPLE;
    else {
        int errCode = 2044;
        String msg = "Cannot find Pig type for " + o.getClass().getName();
        throw new ExecException(msg, errCode, PigException.BUG);
    }
}
 

public void testCharArray2Numeric(){
	byte[] numbericTypes=new byte[]{DataType.DOUBLE,DataType.FLOAT,DataType.LONG,DataType.INTEGER};
	Schema.FieldSchema inputFieldSchema=new Schema.FieldSchema("",DataType.CHARARRAY);
	for (byte type:numbericTypes){
		Schema.FieldSchema castFieldSchema=new Schema.FieldSchema("",type);
		assertTrue(Schema.FieldSchema.castable(castFieldSchema, inputFieldSchema));
	}
}
 

@Test
public void testQuery86() throws Exception {
    LogicalPlan lp;
    String query = "a = load 'myfile' as (name:Chararray, age:Int, gpa:Float);" +
                   "b = group a by (name, age);" +
                   "store b into 'output';";
    lp = buildPlan( query );
    Operator store = lp.getSinks().get(0);
    LOCogroup cogroup = (LOCogroup) lp.getPredecessors(store).get(0);

    Schema.FieldSchema nameFs = new Schema.FieldSchema("name", DataType.CHARARRAY);
    Schema.FieldSchema ageFs = new Schema.FieldSchema("age", DataType.INTEGER);
    Schema.FieldSchema gpaFs = new Schema.FieldSchema("gpa", DataType.FLOAT);

    Schema groupSchema = new Schema(nameFs);
    groupSchema.add(ageFs);
    Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);

    Schema loadSchema = new Schema(nameFs);
    loadSchema.add(ageFs);
    loadSchema.add(gpaFs);

    Schema.FieldSchema bagFs = new Schema.FieldSchema("a", loadSchema, DataType.BAG);

    Schema cogroupExpectedSchema = new Schema(groupFs);
    cogroupExpectedSchema.add(bagFs);
    Assert.assertTrue(cogroup.getSchema().toString(false).equals("group:tuple(name:chararray,age:int),a:bag{:tuple(name:chararray,age:int,gpa:float)}"));
}
 

/**
 * Test that ResourceSchema is correctly with SortInfo
 */
@Test
public void testResourceFlatSchemaCreationWithSortInfo() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
    List<SortColInfo> colList = new ArrayList<SortColInfo>();
    SortColInfo col1 = new SortColInfo("f1", 0, SortColInfo.Order.ASCENDING);
    SortColInfo col2 = new SortColInfo("f1", 1, SortColInfo.Order.DESCENDING);
    colList.add(col1);
    colList.add(col2);
    SortInfo sortInfo = new SortInfo(colList);
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema, sortInfo);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
    assertTrue(rsSchema.getSortKeys()[0]==0);
    assertTrue(rsSchema.getSortKeys()[1]==1);
    assertTrue(rsSchema.getSortKeyOrders()[0]==ResourceSchema.Order.ASCENDING);
    assertTrue(rsSchema.getSortKeyOrders()[1]==ResourceSchema.Order.DESCENDING);
}