Java ResourceSchema.getFields方法代码示例

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public JsonLoader(String schemaStr, String optStr) {
    schema = new ResourceSchema(getEscapedSchemaFromString(schemaStr));
    fields = schema.getFields();
    
    populateValidOptions();
    String[] optsArr = optStr.split(" ");
    
    try {
      configuredOptions_ = parser_.parse(validOptions_, optsArr);
    } catch (org.apache.commons.cli.ParseException e) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp( "[-inputFormat]", validOptions_ );
        throw new RuntimeException(e);
    }
    
    if (configuredOptions_.getOptionValue("inputFormat") != null) {
      this.inputFormatClassName = configuredOptions_.getOptionValue("inputFormat");
    }
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
@Override
public void checkSchema(ResourceSchema s) throws IOException {
  if (myUDFContextSignature == null) {
    throw new IllegalStateException("No UDFContext Signature provided to this UDF! Cannot store field names!");
  }

  ResourceSchema.ResourceFieldSchema[] fields = s.getFields();
  if (fields == null || fields.length == 0) {
    throw new IOException("Input field names not available from schema during front-end processing! " +
        "FusionIndexPipelineStoreFunc must have field names!");
  }

  List<String> fieldNames = new ArrayList<String>(fields.length);
  for (int f = 0; f < fields.length; f++) {
    fieldNames.add(fields[f].getName());
  }

  // Save the fieldIndexToType Mapping in the UDFContext, keyed by our
  // UDFContext Signature so we don't step on other FusionIndexPipelineStoreFunc UDFs
  Properties udfProps =
      UDFContext.getUDFContext().getUDFProperties(getClass(), new String[]{myUDFContextSignature});
  udfProps.put(FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, fieldNames);

  log.info(String.format("Saved %s=%s into UDFContext using signature: %s",
      FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, String.valueOf(fieldNames), myUDFContextSignature));
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    // Store writer to use in putNext()
    this.writer = writer;

    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));
    fields = schema.getFields();
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
@Override
public List<String> getPredicateFields(String location, Job job) throws IOException {
    ResourceSchema schema = getSchema(location, job);
    List<String> predicateFields = new ArrayList<String>();
    for (ResourceFieldSchema field : schema.getFields()) {
        switch(field.getType()) {
        case DataType.BOOLEAN:
        case DataType.INTEGER:
        case DataType.LONG:
        case DataType.FLOAT:
        case DataType.DOUBLE:
        case DataType.DATETIME:
        case DataType.CHARARRAY:
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            predicateFields.add(field.getName());
            break;
        default:
            // Skip DataType.BYTEARRAY, DataType.TUPLE, DataType.MAP and DataType.BAG
            break;
        }
    }
    return predicateFields;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public FromJsonWithSchema(String schemaStr) {
    jsonFactory = new JsonFactory();
    tupleFactory = TupleFactory.getInstance();

    logicalSchema = JsonLoader.getEscapedSchemaFromString(schemaStr);
    resourceSchema = new ResourceSchema(logicalSchema);
    resourceSchemaFields = resourceSchema.getFields();
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
@Override
public Map exec(Tuple input) throws IOException {
    if (input == null || input.size() == 0) {
        return null;
    }
    try {
        String jsonStr = (String) input.get(0);
        String schemaStr = "object: map[]";

        ResourceSchema schema = new ResourceSchema(Utils.getSchemaFromString(schemaStr));
        ResourceFieldSchema[] fields = schema.getFields();

        JsonParser p = jsonFactory.createJsonParser(jsonStr);

        Tuple t = tupleFactory.newTuple(1);
        try {
            p.nextToken(); // move to start of object
            t.set(0, JsonLoader.readField(jsonStr, p, fields[0]));
        } catch (JsonParseException jpe) {
            log.error("Error parsing input: " + jsonStr + ": " + jpe.toString());
        }

        p.close();
        return (Map) t.get(0);
    } catch (ExecException e) {
        warn("Error reading input: " + e.getMessage(), PigWarning.UDF_WARNING_1);
        return null;
    }
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Convert a pig ResourceSchema to avro schema
 * 
 */
public static Schema convert(ResourceSchema pigSchema, boolean nullable) throws IOException {
    ResourceFieldSchema[] pigFields = pigSchema.getFields();

    /* remove the pig tuple wrapper */
    if (pigFields.length == 1) {

        AvroStorageLog.details("Ignore the pig tuple wrapper.");
        return convert(pigFields[0], nullable);
    } else
        return convertRecord(pigFields, nullable);
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Translated a ResourceSchema to an Avro Schema.
 * @param rs Input schema.
 * @param recordName Record name
 * @param recordNameSpace Namespace
 * @param definedRecordNames Map of already defined record names
 * to schema objects
 * @return the translated schema
 * @throws IOException
 */
public static Schema resourceSchemaToAvroSchema(final ResourceSchema rs,
    String recordName, final String recordNameSpace,
    final Map<String, List<Schema>> definedRecordNames,
    final Boolean doubleColonsToDoubleUnderscores) throws IOException {

  if (rs == null) {
    return null;
  }

  recordName = toAvroName(recordName, doubleColonsToDoubleUnderscores);

  List<Schema.Field> fields = new ArrayList<Schema.Field>();
  Schema newSchema = Schema.createRecord(
      recordName, null, recordNameSpace, false);
  if (rs.getFields() != null) {
    Integer i = 0;
    for (ResourceSchema.ResourceFieldSchema rfs : rs.getFields()) {
      String rfsName = toAvroName(rfs.getName(),
          doubleColonsToDoubleUnderscores);
      Schema fieldSchema = resourceFieldSchemaToAvroSchema(
          rfsName, recordNameSpace, rfs.getType(),
          rfs.getDescription().equals("autogenerated from Pig Field Schema")
            ? null : rfs.getDescription(),
          rfs.getSchema(), definedRecordNames,
          doubleColonsToDoubleUnderscores);
      fields.add(new Schema.Field((rfsName != null)
            ? rfsName : recordName + "_" + i.toString(),
          fieldSchema,
          rfs.getDescription().equals(
              "autogenerated from Pig Field Schema")
              ? null : rfs.getDescription(), null));
      i++;

    }
    newSchema.setFields(fields);
  }

  return newSchema;
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * This method adds FieldSchema of 'input source tag/path' as the first
 * field. This will be called only when PigStorage is invoked with
 * '-tagFile' or '-tagPath' option and the schema file is present to be
 * loaded.
 * 
 * @param schema
 * @param fieldName
 * @return ResourceSchema
 */
public static ResourceSchema getSchemaWithInputSourceTag(ResourceSchema schema, String fieldName) {
    ResourceFieldSchema[] fieldSchemas = schema.getFields();
    ResourceFieldSchema sourceTagSchema = new ResourceFieldSchema(new FieldSchema(fieldName, DataType.CHARARRAY));
    ResourceFieldSchema[] fieldSchemasWithSourceTag = new ResourceFieldSchema[fieldSchemas.length + 1];
    fieldSchemasWithSourceTag[0] = sourceTagSchema;
    for(int j = 0; j < fieldSchemas.length; j++) {
        fieldSchemasWithSourceTag[j + 1] = fieldSchemas[j];
    }
    return schema.setFields(fieldSchemasWithSourceTag);
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * Get avro schema from "location" and return the converted
 * PigSchema.
 */
@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {

    /* get avro schema */
    AvroStorageLog.funcCall("getSchema");
    if (inputAvroSchema == null) {
        Set<Path> paths = new HashSet<Path>();
        Configuration conf = job.getConfiguration();
        if (AvroStorageUtils.getAllSubDirs(new Path(location), conf, paths)) {
            setInputAvroSchema(paths, conf);
        }
    }
    if(inputAvroSchema != null) {
        AvroStorageLog.details( "avro input schema:"  + inputAvroSchema);

        /* convert to pig schema */
        ResourceSchema pigSchema = AvroSchema2Pig.convert(inputAvroSchema);
        AvroStorageLog.details("pig input schema:" + pigSchema);
        if (pigSchema.getFields().length == 1){
            pigSchema = pigSchema.getFields()[0].getSchema();
        }
        return pigSchema;
    } else {
        return null;
    }
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public FixedWidthLoader(String columnSpec) {
    try {
        columns = parseColumnSpec(columnSpec);
        String schemaStr = generateDefaultSchemaString();
        schema = new ResourceSchema(Utils.getSchemaFromString(schemaStr));
        fields = schema.getFields();
    } catch (ParserException e) {
        throw new IllegalArgumentException("Invalid schema format: " + e.getMessage());
    }
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
public static Schema getPigSchema(ResourceSchema rSchema) 
throws FrontendException {
    if(rSchema == null) {
        return null;
    }
    List<FieldSchema> fsList = new ArrayList<FieldSchema>();
    for(ResourceFieldSchema rfs : rSchema.getFields()) {
        FieldSchema fs = new FieldSchema(rfs.getName(), 
                rfs.getSchema() == null ? 
                        null : getPigSchema(rfs.getSchema()), rfs.getType());
        
        if(rfs.getType() == DataType.BAG) {
            if (fs.schema != null) { // allow partial schema
                if (fs.schema.size() == 1) {
                    FieldSchema innerFs = fs.schema.getField(0);
                    if (innerFs.type != DataType.TUPLE) {
                        ResourceFieldSchema.throwInvalidSchemaException();
                    }
                } else {
                    ResourceFieldSchema.throwInvalidSchemaException();
                }
            } 
        }
        fsList.add(fs);
    }
    return new Schema(fsList);
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
private String getReqiredColumnNamesString(ResourceSchema schema) {
    StringBuilder sb = new StringBuilder();
    for (ResourceFieldSchema field : schema.getFields()) {
        sb.append(field.getName()).append(",");
    }
    if(sb.charAt(sb.length() -1) == ',') {
        sb.deleteCharAt(sb.length() - 1);
    }
    return sb.toString();
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
/**
 * This method adds FieldSchema of 'input source tag/path' as the first
 * field. This will be called only when PigStorage is invoked with
 * '-tagFile' or '-tagPath' option and the schema file is present to be
 * loaded.
 *
 * @param schema
 * @param fieldName
 * @return ResourceSchema
 */
public static ResourceSchema getSchemaWithInputSourceTag(ResourceSchema schema, String fieldName) {
    ResourceFieldSchema[] fieldSchemas = schema.getFields();
    ResourceFieldSchema sourceTagSchema = new ResourceFieldSchema(new FieldSchema(fieldName, DataType.CHARARRAY));
    ResourceFieldSchema[] fieldSchemasWithSourceTag = new ResourceFieldSchema[fieldSchemas.length + 1];
    fieldSchemasWithSourceTag[0] = sourceTagSchema;
    for(int j = 0; j < fieldSchemas.length; j++) {
        fieldSchemasWithSourceTag[j + 1] = fieldSchemas[j];
    }
    return schema.setFields(fieldSchemasWithSourceTag);
}
 

import org.apache.pig.ResourceSchema; //导入方法依赖的package包/类
@Override
public void convert(Object from, BytesArray to) {

    // expect PigTuple holding a Tuple with only one field - chararray or bytearray
    Assert.isTrue(from instanceof PigTuple,
            String.format("Unexpected object type, expecting [%s], given [%s]", PigTuple.class, from.getClass()));

    PigTuple pt = (PigTuple) from;
    ResourceFieldSchema schema = pt.getSchema();

    // unwrap the tuple
    ResourceSchema tupleSchema = schema.getSchema();

    // empty tuple shortcut
    if (tupleSchema == null) {
        // write empty doc
        to.bytes("{}");
        return;
    }

    ResourceFieldSchema[] fields = tupleSchema.getFields();
    Assert.isTrue(fields.length == 1, "When using JSON input, only one field is expected");

    Object object;
    byte type;

    try {
        object = pt.getTuple().get(0);
        type = pt.getTuple().getType(0);
    } catch (Exception ex) {
        throw new EsHadoopIllegalStateException("Encountered exception while processing tuple", ex);
    }


    if (type == DataType.BIGCHARARRAY || type == DataType.CHARARRAY) {
        to.bytes(object.toString());
        return;
    }
    if (type == DataType.BYTEARRAY) {
        DataByteArray dba = (DataByteArray) object;
        to.bytes(dba.get(), dba.size());
        return;
    }

    throw new EsHadoopIllegalArgumentException(String.format("Cannot handle Pig type [%s]; expecting [%s,%s]", object.getClass(), String.class, DataByteArray.class));
}