Java LineRecordReader类代码示例

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException
{
  key = new Text();
  value = new MapWritable();
  jsonParser = new JSONParser();

  lineReader = new LineRecordReader();
  lineReader.initialize(inputSplit, context);

  queryString = context.getConfiguration().get("query", "?q=*");

  // Load the data schemas
  FileSystem fs = FileSystem.get(context.getConfiguration());
  try
  {
    SystemConfiguration.setProperty("data.schemas", context.getConfiguration().get("data.schemas"));
    DataSchemaLoader.initialize(true, fs);
  } catch (Exception e)
  {
    e.printStackTrace();
  }
  String dataSchemaName = context.getConfiguration().get("dataSchemaName");
  dataSchema = DataSchemaRegistry.get(dataSchemaName);
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
/**
 * Actually instantiate the user's chosen RecordReader implementation.
 */
@SuppressWarnings("unchecked")
private void createChildReader() throws IOException, InterruptedException {
  LOG.debug("ChildSplit operates on: " + split.getPath(index));

  Configuration conf = context.getConfiguration();

  // Determine the file format we're reading.
  Class rrClass;
  if (ExportJobBase.isSequenceFiles(conf, split.getPath(index))) {
    rrClass = SequenceFileRecordReader.class;
  } else {
    rrClass = LineRecordReader.class;
  }

  // Create the appropriate record reader.
  this.rr = (RecordReader<LongWritable, Object>)
      ReflectionUtils.newInstance(rrClass, conf);
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
/**
 * Called once at initialization to initialize the RecordReader.
 *
 * @param genericSplit the split that defines the range of records to read.
 * @param context the information about the task.
 * @throws IOException on IO Error.
 */
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    try {
      LOG.debug("initialize('{}', '{}')",
          HadoopToStringUtil.toString(genericSplit), HadoopToStringUtil.toString(context));
    } catch (InterruptedException ie) {
      LOG.debug("InterruptedException during HadoopToStringUtil.toString", ie);
    }
  }
  Preconditions.checkArgument(genericSplit instanceof FileSplit,
      "InputSplit genericSplit should be an instance of FileSplit.");
  // Get FileSplit.
  FileSplit fileSplit = (FileSplit) genericSplit;
  // Create the JsonParser.
  jsonParser = new JsonParser();
  // Initialize the LineRecordReader.
  lineReader = new LineRecordReader();
  lineReader.initialize(fileSplit, context);
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(final InputSplit inputSplit,
					   final TaskAttemptContext taskAttemptContext)
		throws IOException, InterruptedException {
	this.lrr = new LineRecordReader();
	this.lrr.initialize(inputSplit, taskAttemptContext);

}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
		throws IOException, InterruptedException {
	int halfOfBufferSize = pair.value.capacity() / 2;
	maxLineSize = context.getConfiguration().getInt(
			LineRecordReader.MAX_LINE_LENGTH, halfOfBufferSize);
	if (maxLineSize > halfOfBufferSize) {
		context.getConfiguration().setInt(LineRecordReader.MAX_LINE_LENGTH,
				halfOfBufferSize);
		maxLineSize = halfOfBufferSize;
	}
	r.initialize(split, context);
	FileSplit fs = (FileSplit) split;
	start = fs.getStart();
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
public ScriptRecordReader(final VertexQueryFilter vertexQuery, final TaskAttemptContext context) throws IOException {
    this.lineRecordReader = new LineRecordReader();
    this.vertexQuery = vertexQuery;
    this.configuration = DEFAULT_COMPAT.getContextConfiguration(context);
    this.faunusConf = ModifiableHadoopConfiguration.of(configuration);
    final FileSystem fs = FileSystem.get(configuration);
    try {
        this.engine.eval(new InputStreamReader(fs.open(new Path(faunusConf.getInputConf(ROOT_NS).get(SCRIPT_FILE)))));
    } catch (Exception e) {
        throw new IOException(e.getMessage());
    }
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
  throws IOException, InterruptedException {
  lineRecordReader = new LineRecordReader();
  lineRecordReader.initialize(inputSplit, taskAttemptContext);
  currentKey = new ImmutableBytesWritable();
  parser = new JSONParser();
  skipBadLines = taskAttemptContext.getConfiguration().getBoolean(
    SKIP_LINES_CONF_KEY, true);
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
@SuppressWarnings("squid:S2095") // recordReader is closed explictly in the close() method
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
    InterruptedException
{
  if (split instanceof FileSplit)
  {
    FileSplit fsplit = (FileSplit) split;
    delimitedParser = getDelimitedParser(fsplit.getPath().toString(),
        context.getConfiguration());
    recordReader = new LineRecordReader();
    recordReader.initialize(fsplit, context);
    // Skip the first
    if (delimitedParser.getSkipFirstLine())
    {
      // Only skip the first line of the first split. The other
      // splits are somewhere in the middle of the original file,
      // so their first lines should not be skipped.
      if (fsplit.getStart() != 0)
      {
        nextKeyValue();
      }
    }
  }
  else
  {
    throw new IOException("input split is not a FileSplit");
  }
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public RecordReader<LongWritable, Text> createRecordReader(
		InputSplit split,
		TaskAttemptContext context )
		throws IOException,
		InterruptedException {
	return new LineRecordReader();
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
private void initializeNextReader() throws IOException {

			rdr = new LineRecordReader();
			rdr.initialize(
					new FileSplit(split.getPath(currentSplit),

					split.getOffset(currentSplit), split
							.getLength(currentSplit), null), context);

			++currentSplit;
		}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
		throws IOException, InterruptedException {

	rdr = new LineRecordReader();
	rdr.initialize(split, context);
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext attempt)
		throws IOException, InterruptedException {
	lineReader = new LineRecordReader();
	lineReader.initialize(inputSplit, attempt);		
			
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
public TsvRecordReader(Configuration conf, int[] keyFields) throws IOException
{
	in = new LineRecordReader();
	if (keyFields.length == 0)
	{
		cutter = null;
		builder = null;
	}
	else
	{
		cutter = new CutText( conf.get(DELIM_CONF, DELIM_DEFALT), keyFields);
		builder = new StringBuilder(1000);
	}
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException
{
	lineReader = new LineRecordReader();
	lineReader.initialize(genericSplit, context);

	split = (FileSplit)genericSplit;
	value = null;
}
 

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader; //导入依赖的package包/类
public ScriptRecordReader() {
    this.lineRecordReader = new LineRecordReader();
}