Java ResourceFieldSchema类代码示例

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
private static ResourceFieldSchema getResourceSchemaFromFieldSchema(HCatFieldSchema hfs)
  throws IOException {
  ResourceFieldSchema rfSchema;
  // if we are dealing with a bag or tuple column - need to worry about subschema
  if (hfs.getType() == Type.STRUCT) {
    rfSchema = new ResourceFieldSchema()
      .setName(hfs.getName())
      .setDescription(hfs.getComment())
      .setType(getPigType(hfs))
      .setSchema(getTupleSubSchema(hfs));
  } else if (hfs.getType() == Type.ARRAY) {
    rfSchema = new ResourceFieldSchema()
      .setName(hfs.getName())
      .setDescription(hfs.getComment())
      .setType(getPigType(hfs))
      .setSchema(getBagSubSchema(hfs));
  } else {
    rfSchema = new ResourceFieldSchema()
      .setName(hfs.getName())
      .setDescription(hfs.getComment())
      .setType(getPigType(hfs))
      .setSchema(null); // no munging inner-schemas
  }
  return rfSchema;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
@Test
public void testBytesToBagWithConversion() throws IOException {
    DataBag b = GenRandomData.genFloatDataBag(r,5,100);
    ResourceFieldSchema fs = GenRandomData.getFloatDataBagFieldSchema(5);
    DataBag convertedBag = ps.getLoadCaster().bytesToBag(b.toString().getBytes(), fs);

    Iterator<Tuple> iter1 = b.iterator();
    Iterator<Tuple> iter2 = convertedBag.iterator();
    for (int i=0;i<100;i++) {
        Tuple t1 = (Tuple)iter1.next();
        assertTrue(iter2.hasNext());
        Tuple t2 = (Tuple)iter2.next();
        for (int j=0;j<5;j++) {
            assertTrue(t2.get(j) instanceof Integer);
            Integer expectedValue = ((Float)t1.get(j)).intValue();
            assertEquals(expectedValue, t2.get(j));
        }
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void putNext(Tuple t) throws IOException {
    // Build a ByteArrayOutputStream to write the JSON into
    ByteArrayOutputStream baos = new ByteArrayOutputStream(BUF_SIZE);
    // Build the generator
    JsonGenerator json =
        jsonFactory.createJsonGenerator(baos, JsonEncoding.UTF8);

    // Write the beginning of the top level tuple object
    json.writeStartObject();
    
    ResourceFieldSchema[] fields = schema.getFields();
    for (int i = 0; i < fields.length; i++) {
        writeField(json, fields[i], t.get(i));
    }
    json.writeEndObject();
    json.close();

    // Hand a null key and our string to Hadoop
    try {
        writer.write(null, new Text(baos.toByteArray()));
    } catch (InterruptedException ie) {
        throw new IOException(ie);
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException {
    ResourceFieldSchema stringfs = new ResourceFieldSchema();
    stringfs.setType(DataType.CHARARRAY);
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
    ResourceFieldSchema longfs = new ResourceFieldSchema();
    longfs.setType(DataType.LONG);
    ResourceFieldSchema floatfs = new ResourceFieldSchema();
    floatfs.setType(DataType.FLOAT);
    ResourceFieldSchema doublefs = new ResourceFieldSchema();
    doublefs.setType(DataType.DOUBLE);
    ResourceFieldSchema boolfs = new ResourceFieldSchema();
    boolfs.setType(DataType.BOOLEAN);
    ResourceFieldSchema dtfs = new ResourceFieldSchema();
    dtfs.setType(DataType.DATETIME);
    
    ResourceSchema tupleSchema = new ResourceSchema();
    tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, longfs, intfs, doublefs, floatfs, stringfs, intfs, doublefs, floatfs, boolfs, dtfs});
    ResourceFieldSchema tuplefs = new ResourceFieldSchema();
    tuplefs.setSchema(tupleSchema);
    tuplefs.setType(DataType.TUPLE);
    
    return tuplefs;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/**
 * Convert an Avro schema to a Pig schema
 */
public static ResourceSchema convert(Schema schema) throws IOException {

    if (AvroStorageUtils.containsGenericUnion(schema))
        throw new IOException ("We don't accept schema containing generic unions.");

    Set<Schema> visitedRecords = new HashSet<Schema>();
    ResourceFieldSchema inSchema = inconvert(schema, FIELD, visitedRecords);

    ResourceSchema tupleSchema;
    if (inSchema.getType() == DataType.TUPLE) {
        tupleSchema = inSchema.getSchema();
    } else { // other typs
        ResourceFieldSchema tupleWrapper = AvroStorageUtils.wrapAsTuple(inSchema);

        ResourceSchema topSchema = new ResourceSchema();
        topSchema.setFields(new ResourceFieldSchema[] { tupleWrapper });

        tupleSchema = topSchema;

    }
    return tupleSchema;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/**
 * Test invalid Resource Schema: bag without tuple field
 * @throws IOException 
 */
@Test(expected=FrontendException.class) 
public void testToPigSchemaWithInvalidSchema2() throws IOException {
    ResourceFieldSchema[] level0 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("fld0").setType(DataType.CHARARRAY)
    };
    
    ResourceSchema rSchema0 = new ResourceSchema()
        .setFields(level0);
    
    ResourceFieldSchema[] level2 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t2").setType(DataType.BAG).setSchema(rSchema0)
    };
     
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
@Override
public Tuple bytesToTuple(byte[] b, ResourceFieldSchema fieldSchema) throws IOException {
    if(b == null)
        return null;
    Tuple t;

    try {
        ByteArrayInputStream bis = new ByteArrayInputStream(b);
        PushbackInputStream in = new PushbackInputStream(bis);
        t = consumeTuple(in, fieldSchema);
    }
    catch (IOException e) {
        LogUtils.warn(this, "Unable to interpret value " + Arrays.toString(b) + " in field being " +
                "converted to type tuple, caught ParseException <" +
                e.getMessage() + "> field discarded",
                PigWarning.FIELD_DISCARDED_TYPE_CONVERSION_FAILED, mLog);
        return null;
    }

    return t;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
public void setSchema(ResourceSchema schema) {
    schemaField = new ResourceFieldSchema();
    schemaField.setType(DataType.TUPLE);
    try {
        schemaField.setSchema(schema);
    } catch (IOException ex) {
        throw new EsHadoopIllegalStateException(String.format("Cannot use schema [%s]", schema), ex);
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfInfo cfInfo = getCfInfo(loadSignature);
    CfDef cfDef = cfInfo.cfDef;
    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfDef cfDef = getCfDef(loadSignature);

    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
@Override
public Tuple exec(Tuple input) throws IOException {
    if (input == null || input.size() == 0) {
        return null;
    }
    try {
        String jsonStr = (String) input.get(0);

        JsonParser p = jsonFactory.createJsonParser(jsonStr);
        Tuple t = tupleFactory.newTuple(resourceSchemaFields.length);

        // Create a map of field names to ResourceFieldSchema's,
        // and create a map of field names to positions in the tuple.
        // These are used during parsing to handle extra, missing, and/or out-of-order
        // fields properly.
        Map<String, ResourceFieldSchema> schemaMap = new HashMap<String, ResourceFieldSchema>();
        Map<String, Integer> schemaPositionMap = new HashMap<String, Integer>();
        for (int i = 0; i < resourceSchemaFields.length; i++) {
            schemaMap.put(resourceSchemaFields[i].getName(), resourceSchemaFields[i]);
            schemaPositionMap.put(resourceSchemaFields[i].getName(), i);
        }

        try {
            p.nextToken(); // move to start of object
            JsonLoader.parseObjectIntoTuple(jsonStr, p, schemaMap, schemaPositionMap, t);
        } catch (JsonParseException jpe) {
            // If the line doesn't parse as a valid JSON object, log an error and move on
            log.error("Error parsing input: " + jsonStr + ": " + jpe.toString());
        }

        p.close();
        return t;
    } catch (ExecException e) {
        warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
        return null;
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
@Override
public Map exec(Tuple input) throws IOException {
    if (input == null || input.size() == 0) {
        return null;
    }
    try {
        String jsonStr = (String) input.get(0);
        String schemaStr = "object: map[]";

        ResourceSchema schema = new ResourceSchema(Utils.getSchemaFromString(schemaStr));
        ResourceFieldSchema[] fields = schema.getFields();

        JsonParser p = jsonFactory.createJsonParser(jsonStr);

        Tuple t = tupleFactory.newTuple(1);
        try {
            p.nextToken(); // move to start of object
            t.set(0, JsonLoader.readField(jsonStr, p, fields[0]));
        } catch (JsonParseException jpe) {
            log.error("Error parsing input: " + jsonStr + ": " + jpe.toString());
        }

        p.close();
        return (Map) t.get(0);
    } catch (ExecException e) {
        warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
        return null;
    }
}
 

import org.apache.pig.ResourceSchema.ResourceFieldSchema; //导入依赖的package包/类
public static ResourceSchema getResourceSchema(HCatSchema hcatSchema)
    throws IOException {
  List<ResourceFieldSchema> rfSchemaList = new ArrayList<ResourceFieldSchema>();
  for (HCatFieldSchema hfs : hcatSchema.getFields()) {
    ResourceFieldSchema rfSchema;
    rfSchema = getResourceSchemaFromFieldSchema(hfs);
    rfSchemaList.add(rfSchema);
  }
  ResourceSchema rSchema = new ResourceSchema();
  rSchema.setFields(rfSchemaList.toArray(new ResourceFieldSchema[rfSchemaList.size()]));
  return rSchema;

}