Java ResourceSchema.setFields方法代码示例

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Convert an Avro schema to a Pig schema
 */
public static ResourceSchema convert(Schema schema) throws IOException {

    if (AvroStorageUtils.containsGenericUnion(schema))
        throw new IOException ("We don't accept schema containing generic unions.");

    Set<Schema> visitedRecords = new HashSet<Schema>();
    ResourceFieldSchema inSchema = inconvert(schema, FIELD, visitedRecords);

    ResourceSchema tupleSchema;
    if (inSchema.getType() == DataType.TUPLE) {
        tupleSchema = inSchema.getSchema();
    } else { // other typs
        ResourceFieldSchema tupleWrapper = AvroStorageUtils.wrapAsTuple(inSchema);

        ResourceSchema topSchema = new ResourceSchema();
        topSchema.setFields(new ResourceFieldSchema[] { tupleWrapper });

        tupleSchema = topSchema;

    }
    return tupleSchema;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException {
    ResourceFieldSchema stringfs = new ResourceFieldSchema();
    stringfs.setType(DataType.CHARARRAY);
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
    ResourceFieldSchema longfs = new ResourceFieldSchema();
    longfs.setType(DataType.LONG);
    ResourceFieldSchema floatfs = new ResourceFieldSchema();
    floatfs.setType(DataType.FLOAT);
    ResourceFieldSchema doublefs = new ResourceFieldSchema();
    doublefs.setType(DataType.DOUBLE);
    ResourceFieldSchema boolfs = new ResourceFieldSchema();
    boolfs.setType(DataType.BOOLEAN);
    ResourceFieldSchema dtfs = new ResourceFieldSchema();
    dtfs.setType(DataType.DATETIME);
    
    ResourceSchema tupleSchema = new ResourceSchema();
    tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, longfs, intfs, doublefs, floatfs, stringfs, intfs, doublefs, floatfs, boolfs, dtfs});
    ResourceFieldSchema tuplefs = new ResourceFieldSchema();
    tuplefs.setSchema(tupleSchema);
    tuplefs.setType(DataType.TUPLE);
    
    return tuplefs;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfInfo cfInfo = getCfInfo(loadSignature);
    CfDef cfDef = cfInfo.cfDef;
    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfDef cfDef = getCfDef(loadSignature);

    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static ResourceSchema getResourceSchema(HCatSchema hcatSchema)
    throws IOException {
  List<ResourceFieldSchema> rfSchemaList = new ArrayList<ResourceFieldSchema>();
  for (HCatFieldSchema hfs : hcatSchema.getFields()) {
    ResourceFieldSchema rfSchema;
    rfSchema = getResourceSchemaFromFieldSchema(hfs);
    rfSchemaList.add(rfSchema);
  }
  ResourceSchema rSchema = new ResourceSchema();
  rSchema.setFields(rfSchemaList.toArray(new ResourceFieldSchema[rfSchemaList.size()]));
  return rSchema;

}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
private static ResourceSchema getTupleSubSchema(HCatFieldSchema hfs)
    throws IOException {
  // for each struct subfield, create equivalent ResourceFieldSchema
  ResourceSchema s = new ResourceSchema();
  List<ResourceFieldSchema> lrfs = new ArrayList<ResourceFieldSchema>();
  for (HCatFieldSchema subField : hfs.getStructSubSchema().getFields()) {
    lrfs.add(getResourceSchemaFromFieldSchema(subField));
  }
  s.setFields(lrfs.toArray(new ResourceFieldSchema[lrfs.size()]));
  return s;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Convert an Avro schema to a Pig schema
 */
public static ResourceSchema convert(Schema schema) throws IOException
{

    if (AvroStorageUtils.containsGenericUnion(schema))
        throw new IOException("We don't accept schema containing generic unions.");

    Set<Schema> visitedRecords = new HashSet<Schema>();
    ResourceFieldSchema inSchema = inconvert(schema, FIELD, visitedRecords);

    ResourceSchema tupleSchema;
    if (inSchema.getType() == DataType.TUPLE)
    {
        tupleSchema = inSchema.getSchema();
    }
    else
    { // other typs
        ResourceFieldSchema tupleWrapper = AvroStorageUtils.wrapAsTuple(inSchema);

        ResourceSchema topSchema = new ResourceSchema();
        topSchema.setFields(new ResourceFieldSchema[] { tupleWrapper });

        tupleSchema = topSchema;

    }
    return tupleSchema;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Add a field schema to a bag schema
 */
static protected void add2BagSchema(ResourceFieldSchema fieldSchema,
                                    ResourceFieldSchema subFieldSchema) throws IOException
{

    ResourceFieldSchema wrapped =
            (subFieldSchema.getType() == DataType.TUPLE) ? subFieldSchema
                    : AvroStorageUtils.wrapAsTuple(subFieldSchema);

    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { wrapped });

    fieldSchema.setSchema(listSchema);

}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/** wrap a pig schema as tuple */
public static ResourceFieldSchema wrapAsTuple(ResourceFieldSchema subFieldSchema) throws IOException {
    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { subFieldSchema });

    ResourceFieldSchema tupleWrapper = new ResourceFieldSchema();
    tupleWrapper.setType(DataType.TUPLE);
    tupleWrapper.setName(PIG_TUPLE_WRAPPER);
    tupleWrapper.setSchema(listSchema);

    return tupleWrapper;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
  * Add a field schema to a bag schema
  */
static protected void add2BagSchema(ResourceFieldSchema fieldSchema,
                                ResourceFieldSchema subFieldSchema)
                                throws IOException {

    ResourceFieldSchema wrapped = (subFieldSchema.getType() == DataType.TUPLE)
                                                          ? subFieldSchema
                                                          : AvroStorageUtils.wrapAsTuple(subFieldSchema);

    ResourceSchema listSchema = new ResourceSchema();
    listSchema.setFields(new ResourceFieldSchema[] { wrapped });

    fieldSchema.setSchema(listSchema);

}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public ResourceFieldSchema getBagFieldSchema() throws IOException{
    ResourceFieldSchema tuplefs = getTupleFieldSchema();

    ResourceSchema outBagSchema = new ResourceSchema();
    outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema outBagfs = new ResourceFieldSchema();
    outBagfs.setSchema(outBagSchema);
    outBagfs.setType(DataType.BAG);

    return outBagfs;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static ResourceFieldSchema getSmallTupDataBagFieldSchema() throws IOException {
    ResourceFieldSchema tuplefs = getSmallTupleFieldSchema();
    
    ResourceSchema bagSchema = new ResourceSchema();
    bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema bagfs = new ResourceFieldSchema();
    bagfs.setSchema(bagSchema);
    bagfs.setType(DataType.BAG);
    
    return bagfs;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static ResourceFieldSchema getSmallTupleFieldSchema() throws IOException{
    ResourceFieldSchema stringfs = new ResourceFieldSchema();
    stringfs.setType(DataType.CHARARRAY);
    ResourceFieldSchema intfs = new ResourceFieldSchema();
    intfs.setType(DataType.INTEGER);
    
    ResourceSchema tupleSchema = new ResourceSchema();
    tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
    ResourceFieldSchema tuplefs = new ResourceFieldSchema();
    tuplefs.setSchema(tupleSchema);
    tuplefs.setType(DataType.TUPLE);
    
    return tuplefs;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static ResourceFieldSchema getFullTupTextDataBagFieldSchema() throws IOException{
    ResourceFieldSchema tuplefs = getSmallBagTextTupleFieldSchema();
    
    ResourceSchema outBagSchema = new ResourceSchema();
    outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
    ResourceFieldSchema outBagfs = new ResourceFieldSchema();
    outBagfs.setSchema(outBagSchema);
    outBagfs.setType(DataType.BAG);
    
    return outBagfs;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/** schema: (value, value, value) where keys are in the front. */
public ResourceSchema getSchema(String location, Job job) throws IOException
{
    setLocation(location, job);
    CfInfo cfInfo = getCfInfo(loadSignature);
    CfDef cfDef = cfInfo.cfDef;

    // top-level schema, no type
    ResourceSchema schema = new ResourceSchema();

    // get default marshallers and validators
    Map<MarshallerType, AbstractType> marshallers = getDefaultMarshallers(cfDef);
    Map<ByteBuffer, AbstractType> validators = getValidatorMap(cfDef);

    // will contain all fields for this schema
    List<ResourceFieldSchema> allSchemaFields = new ArrayList<ResourceFieldSchema>();

    for (ColumnDef cdef : cfDef.column_metadata)
    {
        ResourceFieldSchema valSchema = new ResourceFieldSchema();
        AbstractType validator = validators.get(cdef.name);
        if (validator == null)
            validator = marshallers.get(MarshallerType.DEFAULT_VALIDATOR);
        valSchema.setName(new String(cdef.getName()));
        valSchema.setType(getPigType(validator));
        allSchemaFields.add(valSchema);
    }

    // top level schema contains everything
    schema.setFields(allSchemaFields.toArray(new ResourceFieldSchema[allSchemaFields.size()]));
    return schema;
}