當前位置: 首頁>>代碼示例>>Java>>正文

Java PigSplit類代碼示例

本文整理匯總了Java中org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit類的典型用法代碼示例。如果您正苦於以下問題：Java PigSplit類的具體用法？Java PigSplit怎麽用？Java PigSplit使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


PigSplit類屬於org.apache.pig.backend.hadoop.executionengine.mapReduceLayer包，在下文中一共展示了PigSplit類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings("unchecked")
public void prepareToRead(RecordReader reader, PigSplit split)
throws IOException {
    this.reader = reader;
    
    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p =
        udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }

    // Parse the schema from the string stored in the properties object.
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    jsonFactory = new JsonFactory();
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:20，代碼來源:JsonLoader.java

示例2: initializeReader

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
private boolean initializeReader() throws IOException, 
InterruptedException {
    if(curSplitIndex > inpSplits.size() - 1) {
        // past the last split, we are done
        return false;
    }
    if(reader != null){
        reader.close();
    }
    InputSplit curSplit = inpSplits.get(curSplitIndex);
    TaskAttemptContext tAContext = HadoopShims.createTaskAttemptContext(conf, 
            new TaskAttemptID());
    reader = inputFormat.createRecordReader(curSplit, tAContext);
    reader.initialize(curSplit, tAContext);
    // create a dummy pigsplit - other than the actual split, the other
    // params are really not needed here where we are just reading the
    // input completely
    PigSplit pigSplit = new PigSplit(new InputSplit[] {curSplit}, -1, 
            new ArrayList<OperatorKey>(), -1);
    wrappedLoadFunc.prepareToRead(reader, pigSplit);
    return true;
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:23，代碼來源:ReadToEndLoader.java

示例3: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@Override
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    // Save reader to use in getNext()
    this.reader = reader;

    splitIndex = split.getSplitIndex();

    // Get schema from front-end
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[] { udfContextSignature });

    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema == null) {
        throw new IOException("Could not find schema in UDF context");
    }
    schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));

    requiredFields = (boolean[]) ObjectSerializer.deserialize(p.getProperty(REQUIRED_FIELDS_SIGNATURE));
    if (requiredFields != null) {
        numRequiredFields = 0;
        for (int i = 0; i < requiredFields.length; i++) {
            if (requiredFields[i])
                numRequiredFields++;
        }
    }
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:27，代碼來源:FixedWidthLoader.java

示例4: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings("rawtypes")
@Override
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    super.prepareToRead(reader, split);
    numRowsSampled = 0;
    avgTupleMemSz = 0;
    rowNum = 0;
    skipInterval = -1;
    memToSkipPerSample = 0;
    numRowSplTupleReturned = false;
    newSample = null;

    Configuration conf = split.getConf();
    sampleRate = conf.getInt(PigConfiguration.PIG_POISSON_SAMPLER_SAMPLE_RATE, DEFAULT_SAMPLE_RATE);
    heapPerc = conf.getFloat(PigConfiguration.PIG_SKEWEDJOIN_REDUCE_MEMUSAGE,
            PartitionSkewedKeys.DEFAULT_PERCENT_MEMUSAGE);
}
開發者ID:sigmoidanalytics，項目名稱:spork，代碼行數:18，代碼來源:PoissonSampleLoader.java

示例5: initializeReader

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
private boolean initializeReader() throws IOException, 
InterruptedException {
    if(curSplitIndex > inpSplits.size() - 1) {
        // past the last split, we are done
        return false;
    }
    if(reader != null){
        reader.close();
    }
    InputSplit curSplit = inpSplits.get(curSplitIndex);
    TaskAttemptContext tAContext = HadoopShims.createTaskAttemptContext(conf, 
            new TaskAttemptID());
    reader = inputFormat.createRecordReader(curSplit, tAContext);
    reader.initialize(curSplit, tAContext);
    // create a dummy pigsplit - other than the actual split, the other
    // params are really not needed here where we are just reading the
    // input completely
    PigSplit pigSplit = new PigSplit(new InputSplit[] {curSplit}, -1, 
            new ArrayList<OperatorKey>(), -1);
    // Set the conf object so that if the wrappedLoadFunc uses it,
    // it won't be null
    pigSplit.setConf(conf);
    wrappedLoadFunc.prepareToRead(reader, pigSplit);
    return true;
}
開發者ID:sigmoidanalytics，項目名稱:spork，代碼行數:26，代碼來源:ReadToEndLoader.java

示例6: attachInputs

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@Override
public void attachInputs(Map<String, LogicalInput> inputs,
        Configuration conf)
        throws ExecException {
    this.conf = conf;
    LogicalInput logInput = inputs.get(inputKey);
    if (logInput == null || !(logInput instanceof MRInput)) {
        throw new ExecException("POSimpleTezLoad only accepts MRInputs");
    }
    input = (MRInput) logInput;
    try {
        reader = input.getReader();
        // Set split index, MergeCoGroup need it. And this input is the only input of the
        // MergeCoGroup vertex.
        if (reader instanceof MRReader) {
            int splitIndex = ((PigSplit)((MRReader)reader).getSplit()).getSplitIndex();
            PigMapReduce.sJobContext.getConfiguration().setInt(PigImplConstants.PIG_SPLIT_INDEX, splitIndex);
        }
    } catch (IOException e) {
        throw new ExecException(e);
    }
}
開發者ID:sigmoidanalytics，項目名稱:spork，代碼行數:23，代碼來源:POSimpleTezLoad.java

示例7: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    this.reader = reader;
    aliasesTupleNames = StringUtils.tokenize(getUDFProperties().getProperty(
            InternalConfigurationOptions.INTERNAL_ES_TARGET_FIELDS));
}
開發者ID:xushjie1987，項目名稱:es-hadoop-v2.2.0，代碼行數:8，代碼來源:EsStorage.java

示例8: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
public void prepareToRead(RecordReader reader, PigSplit split)
{
    this.reader = reader;
    if (reader instanceof CqlRecordReader) {
        nativeProtocolVersion = ((CqlRecordReader) reader).getNativeProtocolVersion();
    }
}
開發者ID:vcostet，項目名稱:cassandra-kmean，代碼行數:8，代碼來源:CqlNativeStorage.java

示例9: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings("rawtypes")
@Override
public void prepareToRead(RecordReader reader, PigSplit split)
		throws IOException {
	this.reader = reader;
	this.tupleFactory = TupleFactory.getInstance();
}
開發者ID:news-sentiment，項目名稱:news-sentiment-pig，代碼行數:8，代碼來源:NewsPaperJsonLoader.java

示例10: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings("rawtypes")
@Override
public void prepareToRead(RecordReader reader, PigSplit split)
throws IOException {
    AvroStorageLog.funcCall("prepareToRead");
    this.reader = (PigAvroRecordReader) reader;
}
開發者ID:svemuri，項目名稱:CalcEngine，代碼行數:8，代碼來源:AvroStorage.java

示例11: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@Override
public void prepareToRead(RecordReader reader, PigSplit split) {
    in = reader;
    if (tagFile || tagPath) {
        sourcePath = ((FileSplit)split.getWrappedSplit()).getPath();
    }
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:8，代碼來源:PigStorage.java

示例12: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@SuppressWarnings("rawtypes")
@Override
public final void prepareToRead(final RecordReader r, final PigSplit s)
    throws IOException {
  reader = r;
  split = s;
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:8，代碼來源:AvroStorage.java

示例13: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@Override
public void prepareToRead(RecordReader reader, PigSplit split) throws IOException {
    super.prepareToRead(reader, split);
    numRowsSampled = 0;
    avgTupleMemSz = 0;
    rowNum = 0;
    skipInterval = -1;
    memToSkipPerSample = 0;
    numRowSplTupleReturned = false;
    newSample = null;

    Configuration conf = split.getConf();
    sampleRate = conf.getInt(SAMPLE_RATE, DEFAULT_SAMPLE_RATE);
    heapPerc = conf.getFloat(PERC_MEM_AVAIL, PartitionSkewedKeys.DEFAULT_PERCENT_MEMUSAGE);
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:16，代碼來源:PoissonSampleLoader.java

示例14: prepareToRead

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
@Override
public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit split) {
    in = reader;
    splitIndex = split.getSplitIndex();
    
    if (headerTreatment == Headers.DEFAULT) {
        headerTreatment = Headers.READ_INPUT_HEADER;
    }
}
開發者ID:sigmoidanalytics，項目名稱:spork，代碼行數:10，代碼來源:CSVExcelStorage.java

示例15: writeDebugHeader

import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; //導入依賴的package包/類
private void writeDebugHeader() {
    processError("===== Task Information Header =====" );

    processError("\nCommand: " + command);
    processError("\nStart time: " + new Date(System.currentTimeMillis()));
    if (job.getBoolean("mapred.task.is.map", false)) {
        MapContext context = (MapContext)PigMapReduce.sJobContext;
        PigSplit pigSplit = (PigSplit)context.getInputSplit();
        int numPaths = pigSplit.getNumPaths();
        processError("\nPigSplit contains " + numPaths + " wrappedSplits.");

        StringBuilder sb = new StringBuilder();
        for(int i = 0; i < numPaths; i++) {
          InputSplit wrappedSplit = pigSplit.getWrappedSplit(i);
          if (wrappedSplit instanceof FileSplit) {
              FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
              sb.append("\nInput-split: file=");
              sb.append(mapInputFileSplit.getPath());
              sb.append(" start-offset=");
              sb.append(Long.toString(mapInputFileSplit.getStart()));
              sb.append(" length=");
              sb.append(Long.toString(mapInputFileSplit.getLength()));
              processError(sb.toString());
              sb.setLength(0);
          }
        }
    }
    processError("\n=====          * * *          =====\n");
}
開發者ID:sigmoidanalytics，項目名稱:spork-streaming，代碼行數:30，代碼來源:HadoopExecutableManager.java


注：本文中的org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。