Java DataType.CHARARRAY属性代码示例

@Test
public void testOutputSchema() throws Exception {
 // given
    Schema schema = new Schema();
    schema.add(new FieldSchema(null, DataType.CHARARRAY));
    schema.add(new FieldSchema(null, DataType.CHARARRAY));
    FieldSchema idFieldSchema = new FieldSchema("id", DataType.CHARARRAY);
    schema.add(idFieldSchema);
    FieldSchema dedupFieldSchema = new FieldSchema("dedup", DataType.BOOLEAN);
    schema.add(dedupFieldSchema);
    
    // execute
    Schema resultSchema = idReplacer.outputSchema(schema);
    
    // assert
    assertNotNull(resultSchema);
    assertEquals(1, resultSchema.getFields().size());
    assertEquals(idFieldSchema, resultSchema.getField(0).schema.getField(0));
    assertEquals(dedupFieldSchema, resultSchema.getField(0).schema.getField(1));
}
 

/**
 * Convert Pig primitive type to Avro type
 * 
 */
protected static Schema convertPrimitiveType(byte pigType) throws IOException {

    if (pigType == DataType.BOOLEAN) {
        return AvroStorageUtils.BooleanSchema;
    } else if (pigType == DataType.BYTEARRAY) {
        return AvroStorageUtils.BytesSchema;
    } else if (pigType == DataType.CHARARRAY
                                    || pigType == DataType.BIGCHARARRAY) {
        return AvroStorageUtils.StringSchema;
    } else if (pigType == DataType.DOUBLE) {
        return AvroStorageUtils.DoubleSchema;
    } else if (pigType == DataType.FLOAT) {
        return AvroStorageUtils.FloatSchema;
    } else if (pigType == DataType.INTEGER) {
        return AvroStorageUtils.IntSchema;
    } else if (pigType == DataType.LONG) {
        return AvroStorageUtils.LongSchema;
    } else
        throw new IOException("unsupported pig type:"
                                        + DataType.findTypeName(pigType));

}
 

/**
 * Test one-level Pig Schema: multiple fields for a bag
 */
@Test
public void testResourceSchemaWithInvalidPigSchema() 
throws FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema level0 = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    Schema.FieldSchema fld0 = 
        new Schema.FieldSchema("f0", level0, DataType.BAG);
    Schema level1 = new Schema(fld0);
    try {
        Schema.getPigSchema(new ResourceSchema(level1));
        Assert.fail();
    } catch(FrontendException e) {
        assertTrue(e.getErrorCode()==2218);
    }
}
 

public Schema outputSchema(Schema input) {
	// Check that we were passed one field
	if ( input.size() != 1 )
		throw new RuntimeException("Excpeted input (String), input need only one field");
	try{
		// Get the types for column and check it.
		// If they are wring, figure out what types were passed and give a good error message.
		if (input.getField(0).type != DataType.CHARARRAY){
			String msg = "Excepted input (String), received schema (";
			msg += DataType.findTypeName(input.getField(0).type);
			msg += ")";
			throw new RuntimeException(msg);
		}
	} catch ( Exception e) {
		throw new RuntimeException(e);
	}
	// Assemble output schema
	Schema output = new Schema();
	FieldSchema paramMap = new FieldSchema("urlparams", DataType.MAP);
	output.add(paramMap);
	return output;
}
 

/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}
 

/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation2() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
 

@Override
public List<String> getPredicateFields(String location, Job job) throws IOException {
    ResourceSchema schema = getSchema(location, job);
    List<String> predicateFields = new ArrayList<String>();
    for (ResourceFieldSchema field : schema.getFields()) {
        switch(field.getType()) {
        case DataType.BOOLEAN:
        case DataType.INTEGER:
        case DataType.LONG:
        case DataType.FLOAT:
        case DataType.DOUBLE:
        case DataType.DATETIME:
        case DataType.CHARARRAY:
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            predicateFields.add(field.getName());
            break;
        default:
            // Skip DataType.BYTEARRAY, DataType.TUPLE, DataType.MAP and DataType.BAG
            break;
        }
    }
    return predicateFields;
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    Schema s = new Schema(new Schema.FieldSchema(null, DataType.CHARARRAY)); 
    
    Schema s1 = new Schema();
    s1.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s1.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    
    Schema s2 = new Schema();
    s2.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s2.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
    s2.add(new Schema.FieldSchema(null, DataType.INTEGER));
    
    funcList.add(new FuncSpec(this.getClass().getName(), s));
    funcList.add(new FuncSpec(this.getClass().getName(), s1));
    funcList.add(new FuncSpec(this.getClass().getName(), s2));
    return funcList;
}
 

@Override
public Schema outputSchema(Schema input) {
    Schema.FieldSchema charFs = new FieldSchema(null, DataType.CHARARRAY);
    Schema.FieldSchema intFs = new FieldSchema(null, DataType.INTEGER);
    Schema.FieldSchema floatFs = new FieldSchema(null, DataType.FLOAT);
    Schema bagSchema = new Schema();
    bagSchema.add(charFs);
    bagSchema.add(intFs);
    bagSchema.add(floatFs);
    Schema.FieldSchema bagFs;
    try {
        bagFs = new Schema.FieldSchema(null, bagSchema, DataType.BAG);
    } catch (FrontendException fee) {
        return null;
    }
    return new Schema(bagFs);
}
 

private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
    if (data == null) {
        return null;
    }
    byte type = detail.type;
    switch (type) {
        case DataType.BOOLEAN:
            return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
        case DataType.INTEGER:
        case DataType.LONG:
            return TypeSystem.asTypedObject(DataType.toLong(data, type));
        case DataType.FLOAT:
        case DataType.DOUBLE:
            return TypeSystem.asTypedObject(DataType.toDouble(data, type));
        case DataType.DATETIME:
            return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
        case DataType.BYTE:
        case DataType.BYTEARRAY:
        case DataType.CHARARRAY:
            return TypeSystem.asTypedObject(DataType.toString(data, type));
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
        default:
            //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
            return null;
    }
}
 

@Override
public Schema outputSchema(Schema input) {
    try {
        return new Schema(new Schema.FieldSchema(getSchemaName(this
                .getClass().getName().toLowerCase(), input),
                DataType.CHARARRAY));
    } catch (Exception e) {
        return null;
    }
}
 

public TOBAG(String bagColName, String tupleColName, String fieldType) {

        if ( bagColName== null || tupleColName == null  || fieldType == null)  {
	    throw new RuntimeException("The bagColName  and fieldType cannot be null");
        }

    	this.bagColName   = bagColName;	
    	this.tupleColName = tupleColName;	

        if ( fieldType.equalsIgnoreCase( "CHARARRAY" )){ 
             this.fieldType = DataType.CHARARRAY;

        } else if ( fieldType.equalsIgnoreCase( "DOUBLE" )){ 
            this.fieldType = DataType.DOUBLE;

        } else if ( fieldType.equalsIgnoreCase( "FLOAT" )){ 
            this.fieldType = DataType.FLOAT; 

        } else if ( fieldType.equalsIgnoreCase( "BOOLEAN" )){ 
            this.fieldType = DataType.BOOLEAN; 

        } else if ( fieldType.equalsIgnoreCase( "INTEGER" )){ 
            this.fieldType = DataType.INTEGER;

        } else if ( fieldType.equalsIgnoreCase( "LONG" )){ 
            this.fieldType = DataType.LONG; 

        } else if ( fieldType.equalsIgnoreCase( "MAP" )){ 
            this.fieldType = DataType.MAP; 
        } else {
	    throw new RuntimeException("This type"+ fieldType +"is not supported in TOBAG");
        }

    }
 

/**
 * Assert that without -noWAL, the WAL is enabled the WAL
 * @throws IOException
 * @throws ParseException
 */
@Test
public void testWIthWAL() throws IOException, ParseException {
    HBaseStorage hbaseStorage = new HBaseStorage(TESTCOLUMN_A);

    Object key = "somekey";
    byte type = DataType.CHARARRAY;
    Assert.assertTrue(hbaseStorage.createPut(key, type).getWriteToWAL());
}
 

protected Object translateWritableToPigDataType(Writable w, byte dataType) {
  switch(dataType) {
    case DataType.CHARARRAY: return ((Text) w).toString();
    case DataType.BYTEARRAY: return((DataByteArray) w).get();
    case DataType.BOOLEAN: return ((BooleanWritable) w).get();
    case DataType.INTEGER: return ((IntWritable) w).get();
    case DataType.LONG: return ((LongWritable) w).get();
    case DataType.FLOAT: return ((FloatWritable) w).get();
    case DataType.DOUBLE: return ((DoubleWritable) w).get();
    case DataType.BYTE: return ((ByteWritable) w).get();
    case DataType.DATETIME: return ((DateTimeWritable) w).get();
  }
  
  return null;
}
 

@Override
public Result getNextBoolean() throws ExecException {
    Result left, right;

    switch (operandType) {
    case DataType.BYTEARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.LONG:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.DATETIME:
    case DataType.CHARARRAY: {
        Result r = accumChild(null, operandType);
        if (r != null) {
            return r;
        }
        left = lhs.getNext(operandType);
        right = rhs.getNext(operandType);
        return doComparison(left, right);
    }

    default: {
        int errCode = 2067;
        String msg = this.getClass().getSimpleName() + " does not know how to " +
        "handle type: " + DataType.findTypeName(operandType);
        throw new ExecException(msg, errCode, PigException.BUG);
    }

    }
}