Java ResourceSchema类代码示例

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    this.writer = writer;

    Properties props = getUDFProperties();
    String s = props.getProperty(ResourceSchema.class.getName());
    if (!StringUtils.hasText(s)) {
        log.warn("No resource schema found; using an empty one....");
        this.schema = new ResourceSchema();
    }
    else {
        this.schema = IOUtils.deserializeFromBase64(s);
    }
    this.pigTuple = new PigTuple(schema);
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
public JsonLoader(String schemaStr, String optStr) {
    schema = new ResourceSchema(getEscapedSchemaFromString(schemaStr));
    fields = schema.getFields();
    
    populateValidOptions();
    String[] optsArr = optStr.split(" ");
    
    try {
      configuredOptions_ = parser_.parse(validOptions_, optsArr);
    } catch (org.apache.commons.cli.ParseException e) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp( "[-inputFormat]", validOptions_ );
        throw new RuntimeException(e);
    }
    
    if (configuredOptions_.getOptionValue("inputFormat") != null) {
      this.inputFormatClassName = configuredOptions_.getOptionValue("inputFormat");
    }
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
public ResourceSchema getSchema(String location, Job job)
        throws IOException {

    if (schema != null) {
        // Send schema to backend
        // Schema should have been passed as an argument (-> constructor)
        // or provided in the default constructor

        UDFContext udfc = UDFContext.getUDFContext();
        Properties p = udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });
        p.setProperty(SCHEMA_SIGNATURE, schema.toString());

        return schema;
    } else {
        // Should never get here
        throw new IllegalArgumentException(
            "No schema found: default schema was never created and no user-specified schema was found."
        );
    }
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public void checkSchema(ResourceSchema s) throws IOException {
  if (myUDFContextSignature == null) {
    throw new IllegalStateException("No UDFContext Signature provided to this UDF! Cannot store field names!");
  }

  ResourceSchema.ResourceFieldSchema[] fields = s.getFields();
  if (fields == null || fields.length == 0) {
    throw new IOException("Input field names not available from schema during front-end processing! " +
        "FusionIndexPipelineStoreFunc must have field names!");
  }

  List<String> fieldNames = new ArrayList<String>(fields.length);
  for (int f = 0; f < fields.length; f++) {
    fieldNames.add(fields[f].getName());
  }

  // Save the fieldIndexToType Mapping in the UDFContext, keyed by our
  // UDFContext Signature so we don't step on other FusionIndexPipelineStoreFunc UDFs
  Properties udfProps =
      UDFContext.getUDFContext().getUDFProperties(getClass(), new String[]{myUDFContextSignature});
  udfProps.put(FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, fieldNames);

  log.info(String.format("Saved %s=%s into UDFContext using signature: %s",
      FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, String.valueOf(fieldNames), myUDFContextSignature));
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {
  HCatContext.INSTANCE.setConf(job.getConfiguration()).getConf().get()
    .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, true);
  Table table = phutil.getTable(location,
    hcatServerUri != null ? hcatServerUri : PigHCatUtil.getHCatServerUri(job),
    PigHCatUtil.getHCatServerPrincipal(job),

    // Pass job to initialize metastore conf overrides for embedded metastore case
    // (hive.metastore.uris = "").
    job);
  HCatSchema hcatTableSchema = HCatUtil.getTableSchemaWithPtnCols(table);
  try {
    PigHCatUtil.validateHCatTableSchemaFollowsPigRules(hcatTableSchema);
  } catch (IOException e) {
    throw new PigException(
      "Table schema incompatible for reading through HCatLoader :" + e.getMessage()
        + ";[Table schema was " + hcatTableSchema.toString() + "]"
      , PigHCatUtil.PIG_EXCEPTION_CODE, e);
  }
  storeInUDFContext(signature, HCatConstants.HCAT_TABLE_SCHEMA, hcatTableSchema);
  outputSchema = hcatTableSchema;
  return PigHCatUtil.getResourceSchema(hcatTableSchema);
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    // Save reader to use in getNext()
    this.reader = reader;

    splitIndex = split.getSplitIndex();

    // Get schema from front-end
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[] { udfContextSignature });

    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    requiredFields = (boolean[]) ObjectSerializer.deserialize(p.getProperty(REQUIRED_FIELDS_SIGNATURE));
    if (requiredFields != null) {
        numRequiredFields = 0;
        for (int i = 0; i < requiredFields.length; i++) {
            if (requiredFields[i])
                numRequiredFields++;
        }
    }
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    // Store the record writer reference so we can use it when it's time
    // to write tuples
    this.writer = writer;

    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    // Parse the schema from the string stored in the properties object.
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    // Build a Json factory
    jsonFactory = new JsonFactory();
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public final void checkSchema(final ResourceSchema rs) throws IOException {
  if (rs == null) {
    throw new IOException("checkSchema: called with null ResourceSchema");
  }
  Schema avroSchema = AvroStorageSchemaConversionUtilities
      .resourceSchemaToAvroSchema(rs,
          (schemaName == null || schemaName.length() == 0)
              ? "pig_output" : schemaName,
              schemaNameSpace,
              Maps.<String, List<Schema>> newHashMap(),
              doubleColonsToDoubleUnderscores);
  if (avroSchema == null) {
    throw new IOException("checkSchema: could not translate ResourceSchema to Avro Schema");
  }
  setOutputAvroSchema(avroSchema);
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
/**
 * Test invalid Resource Schema: multiple fields for a bag
 * @throws IOException 
 */
@Test(expected=FrontendException.class) 
public void testToPigSchemaWithInvalidSchema() throws IOException {
    ResourceFieldSchema[] level0 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("fld0").setType(DataType.CHARARRAY),
            new ResourceFieldSchema()
                .setName("fld1").setType(DataType.DOUBLE),        
            new ResourceFieldSchema()
                .setName("fld2").setType(DataType.INTEGER)
    };
    
    ResourceSchema rSchema0 = new ResourceSchema()
        .setFields(level0);
    
    ResourceFieldSchema[] level2 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t2").setType(DataType.BAG).setSchema(rSchema0)
    };
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation2() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    
    Schema origSchema = new Schema(
            new Schema.FieldSchema("t1", 
                    new Schema(
                            new Schema.FieldSchema("t0", 
                                    TypeCheckingTestUtil.genFlatSchema(
                                            aliases,types), 
                                            DataType.TUPLE)), DataType.BAG));
                    
    ResourceSchema rsSchema = new ResourceSchema(origSchema);

    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public void setStoreLocation(String location, Job job) throws IOException {
    if (location.startsWith("hbase://")){
        job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, location.substring(8));
    }else{
        job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, location);
    }

    String serializedSchema = getUDFProperties().getProperty(contextSignature + "_schema");
    if (serializedSchema!= null) {
        schema_ = (ResourceSchema) ObjectSerializer.deserialize(serializedSchema);
    }

    initialiseHBaseClassLoaderResources(job);
    m_conf = initializeLocalJobConfig(job);
    // Not setting a udf property and getting the hbase delegation token
    // only once like in setLocation as setStoreLocation gets different Job
    // objects for each call and the last Job passed is the one that is
    // launched. So we end up getting multiple hbase delegation tokens.
    addHBaseDelegationToken(m_conf, job);
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@Override
public void storeSchema(ResourceSchema schema, String location,
        Job job) throws IOException {
    FileSystem fs = FileSystem.get(job.getConfiguration());
    // verify that output is available prior to storeSchema call
    Path resultPath = new Path(location, "part-m-00000");
    if (!fs.exists(resultPath)) {
        FileStatus[] listing = fs.listStatus(new Path(location));
        for (FileStatus fstat : listing) {
            System.err.println(fstat.getPath());
        }
        // not creating the marker file below fails the test
        throw new IOException("" + resultPath + " not available in storeSchema");
    }

    // create a file to test that this method got called - if it gets called
    // multiple times, the create will throw an Exception
    fs.create(
            new Path(location + "_storeSchema_test"),
            false);
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
/**
 * Test that ResourceSchema is correctly created given a
 * pig.Schema and vice versa 
 */
@Test
public void testResourceFlatSchemaCreation() 
throws ExecException, SchemaMergeException, FrontendException {
    String [] aliases ={"f1", "f2"};
    byte[] types = {DataType.CHARARRAY, DataType.INTEGER};
    Schema origSchema = TypeCheckingTestUtil.genFlatSchema(
            aliases,types);
    ResourceSchema rsSchema = new ResourceSchema(origSchema);
    assertEquals("num fields", aliases.length, rsSchema.getFields().length);
    ResourceSchema.ResourceFieldSchema[] fields = rsSchema.getFields();
    for (int i=0; i<fields.length; i++) {
        assertEquals(fields[i].getName(), aliases[i]);
        assertEquals(fields[i].getType(), types[i]);
    }
    Schema genSchema = Schema.getPigSchema(rsSchema);
    assertTrue("generated schema equals original", 
            Schema.equals(genSchema, origSchema, true, false));
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void prepareToRead(RecordReader reader, PigSplit split)
throws IOException {
    this.reader = reader;
    
    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    // Parse the schema from the string stored in the properties object.
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    jsonFactory = new JsonFactory();
}
 

import org.apache.pig.ResourceSchema; //导入依赖的package包/类
/**
 * Test invalid Resource Schema: bag without tuple field
 * @throws IOException 
 */
@Test(expected=FrontendException.class) 
public void testToPigSchemaWithInvalidSchema2() throws IOException {
    ResourceFieldSchema[] level0 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("fld0").setType(DataType.CHARARRAY)
    };
    
    ResourceSchema rSchema0 = new ResourceSchema()
        .setFields(level0);
    
    ResourceFieldSchema[] level2 = new ResourceFieldSchema[] {
            new ResourceFieldSchema()
                .setName("t2").setType(DataType.BAG).setSchema(rSchema0)
    };
     
}