Java UDFContext类代码示例

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
public ResourceSchema getSchema(String location, Job job)
        throws IOException {

    if (schema != null) {
        // Send schema to backend
        // Schema should have been passed as an argument (-> constructor)
        // or provided in the default constructor

        UDFContext udfc = UDFContext.getUDFContext();
        Properties p = udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });
        p.setProperty(SCHEMA_SIGNATURE, schema.toString());

        return schema;
    } else {
        // Should never get here
        throw new IllegalArgumentException(
            "No schema found: default schema was never created and no user-specified schema was found."
        );
    }
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException {
    if (requiredFieldList == null)
        return null;

    if (!useDefaultSchema && requiredFieldList.getFields() != null)
    {
        requiredFields = new boolean[fields.length];

        for (RequiredField f : requiredFieldList.getFields()) {
            requiredFields[f.getIndex()] = true;
        }

        UDFContext udfc = UDFContext.getUDFContext();
        Properties p = udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });
        try {
            p.setProperty(REQUIRED_FIELDS_SIGNATURE, ObjectSerializer.serialize(requiredFields));
        } catch (Exception e) {
            throw new RuntimeException("Cannot serialize requiredFields for pushProjection");
        }
    }

    return new RequiredFieldResponse(true);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public void checkSchema(ResourceSchema s) throws IOException {
  if (myUDFContextSignature == null) {
    throw new IllegalStateException("No UDFContext Signature provided to this UDF! Cannot store field names!");
  }

  ResourceSchema.ResourceFieldSchema[] fields = s.getFields();
  if (fields == null || fields.length == 0) {
    throw new IOException("Input field names not available from schema during front-end processing! " +
        "FusionIndexPipelineStoreFunc must have field names!");
  }

  List<String> fieldNames = new ArrayList<String>(fields.length);
  for (int f = 0; f < fields.length; f++) {
    fieldNames.add(fields[f].getName());
  }

  // Save the fieldIndexToType Mapping in the UDFContext, keyed by our
  // UDFContext Signature so we don't step on other FusionIndexPipelineStoreFunc UDFs
  Properties udfProps =
      UDFContext.getUDFContext().getUDFProperties(getClass(), new String[]{myUDFContextSignature});
  udfProps.put(FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, fieldNames);

  log.info(String.format("Saved %s=%s into UDFContext using signature: %s",
      FIELD_NAMES_FROM_SCHEMA_PROPS_KEY, String.valueOf(fieldNames), myUDFContextSignature));
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
HCatSchema getHCatSchema(List<RequiredField> fields, String signature,
                         Class<?> classForUDFCLookup) throws IOException {
  if (fields == null) {
    return null;
  }

  Properties props = UDFContext.getUDFContext().getUDFProperties(
    classForUDFCLookup, new String[]{signature});
  HCatSchema hcatTableSchema = (HCatSchema) props.get(HCatConstants.HCAT_TABLE_SCHEMA);

  ArrayList<HCatFieldSchema> fcols = new ArrayList<HCatFieldSchema>();
  for (RequiredField rf : fields) {
    fcols.add(hcatTableSchema.getFields().get(rf.getIndex()));
  }
  return new HCatSchema(fcols);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void prepareToRead(RecordReader reader, PigSplit split)
throws IOException {
    this.reader = reader;
    
    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    // Parse the schema from the string stored in the properties object.
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    jsonFactory = new JsonFactory();
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
/**
 * Set up the run-time environment of the managed process.
 * 
 * @param pb
 *            {@link ProcessBuilder} used to exec the process
 */
private static void setupEnvironment(ProcessBuilder pb) {
    String separator = ":";
    Configuration conf = UDFContext.getUDFContext().getJobConf();
    Map<String, String> env = pb.environment();
    addJobConfToEnvironment(conf, env);

    // Add the current-working-directory to the $PATH
    File dir = pb.directory();
    String cwd = (dir != null) ? dir.getAbsolutePath() : System
            .getProperty("user.dir");

    String envPath = env.get(PATH);
    if (envPath == null) {
        envPath = cwd;
    } else {
        envPath = envPath + separator + cwd;
    }
    env.put(PATH, envPath);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    // Store the record writer reference so we can use it when it's time
    // to write tuples
    this.writer = writer;

    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    // Parse the schema from the string stored in the properties object.
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    // Build a Json factory
    jsonFactory = new JsonFactory();
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public RequiredFieldResponse pushProjection(
        RequiredFieldList requiredFieldList) throws FrontendException {
    if (requiredFieldList == null)
        return null;
    if (requiredFieldList.getFields() != null)
    {
        int schemaSize = ((StructTypeInfo)typeInfo).getAllStructFieldTypeInfos().size();
        mRequiredColumns = new boolean[schemaSize];
        for (RequiredField rf: requiredFieldList.getFields())
        {
            if (rf.getIndex()!=-1)
                mRequiredColumns[rf.getIndex()] = true;
        }
        Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
        try {
            p.setProperty(signature + RequiredColumnsSuffix, ObjectSerializer.serialize(mRequiredColumns));
        } catch (Exception e) {
            throw new RuntimeException("Cannot serialize mRequiredColumns");
        }
    }
    return new RequiredFieldResponse(true);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public ResourceSchema getSchema(String location,
        Job job) throws IOException {
    if (!dontLoadSchema) {
        schema = (new JsonMetadata()).getSchema(location, job, isSchemaOn);

        if (signature != null && schema != null) {
            if(tagFile) {
                schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_NAME");
            } else if(tagPath) {
                schema = Utils.getSchemaWithInputSourceTag(schema, "INPUT_FILE_PATH");
            }
            Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass(),
                    new String[] {signature});
            p.setProperty(signature + ".schema", schema.toString());
        }
    }
    return schema;
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    // Store writer to use in putNext()
    this.writer = writer;

    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[]{ udfContextSignature });
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));
    fields = schema.getFields();
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
/**
    * Reads the partition keys from the location i.e the base directory
    * 
    * @param location
    *            String must be the base directory for the partitions
    * @param conf
    * @param loaderClass
    * @throws IOException
    */
   public void setPartitionKeys(String location, Configuration conf,
    Class<? extends LoadFunc> loaderClass, String signature)
    throws IOException {

Set<String> partitionKeys = getPartitionKeys(location, conf);

if (partitionKeys != null) {
    StringBuilder buff = new StringBuilder();
    int i = 0;
    for (String key : partitionKeys) {
	if (i++ != 0) {
	    buff.append(",");
	}

	buff.append(key);
    }

    UDFContext.getUDFContext()
	    .getUDFProperties(loaderClass, new String[] { signature })
	    .setProperty(PARTITION_COLUMNS, buff.toString());
}

   }
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
private void instantiateFunc(FuncSpec fSpec) {
    this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
    this.setSignature(signature);
    Properties props = UDFContext.getUDFContext().getUDFProperties(func.getClass());
	Schema tmpS=(Schema)props.get("pig.evalfunc.inputschema."+signature);

	if(tmpS!=null)
		this.func.setInputSchema(tmpS);
    if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
        executor = new MonitoredUDFExecutor(func);
    }
    //the next couple of initializations do not work as intended for the following reasons
    //the reporter and pigLogger are member variables of PhysicalOperator
    //when instanitateFunc is invoked at deserialization time, both
    //reporter and pigLogger are null. They are set during map and reduce calls,
    //making the initializations here basically useless. Look at the processInput
    //method where these variables are re-initialized. At that point, the PhysicalOperator
    //is set up correctly with the reporter and pigLogger references
    this.func.setReporter(getReporter());
    this.func.setPigLogger(pigLogger);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage2() throws Exception {
    // Register a custom output size reader in configuration
    Configuration conf = new Configuration();
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
            DummyOutputSizeReader.class.getName());

    // ClientSystemProps is needed to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    Method getOutputSize = getJobStatsMethod("getOutputSize", POStore.class, Configuration.class);
    long outputSize = (Long) getOutputSize.invoke(
            null, createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);

    assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
            DummyOutputSizeReader.SIZE, outputSize);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage4() throws Exception {
    // Register a comma-separated list of readers in configuration, and
    // verify that the one that supports a non-file-based uri is used.
    Configuration conf = new Configuration();
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
            FileBasedOutputSizeReader.class.getName() + ","
                    + DummyOutputSizeReader.class.getName());

    // ClientSystemProps needs to be initialized to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    long outputSize = JobStats.getOutputSize(
            createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);

    assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
            DummyOutputSizeReader.SIZE, outputSize);
}
 

import org.apache.pig.impl.util.UDFContext; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void prepareToWrite(RecordWriter writer) throws IOException {
    this.writer = (DocumentDBRecordWriter) writer;
    
    // Parse the schema from the string stored in the properties object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[]{udfContextSignature});

    String strSchema = p.getProperty(PIG_OUTPUT_SCHEMA_UDF_CONTEXT);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    try {
        // Parse the schema from the string stored in the properties object.
        this.schema = new ResourceSchema(SchemaHelper.getSchemaFromString(strSchema));
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
    }

}