Java Reader类代码示例

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void attachInputs(Map<String, LogicalInput> inputs,
        Configuration conf)
        throws ExecException {
    this.conf = conf;
    LogicalInput input = inputs.get(inputKey);
    if (input == null) {
        throw new ExecException("Input from vertex " + inputKey + " is missing");
    }

    try {
        Reader r = input.getReader();
        if (r instanceof KeyValueReader) {
            reader = (KeyValueReader) r;
        } else {
            shuffleInput = true;
            shuffleReader = (KeyValuesReader) r;
            hasNext = shuffleReader.next();
        }
        LOG.info("Attached input from vertex " + inputKey + " : input=" + input + ", reader=" + r);
    } catch (Exception e) {
        throw new ExecException(e);
    }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void attachInputs(Map<String, LogicalInput> inputs,
        Configuration conf) throws ExecException {
    LogicalInput input = inputs.get(inputKey);
    if (input == null) {
        throw new ExecException("Input from vertex " + inputKey + " is missing");
    }
    try {
        Reader r = input.getReader();
        if (r instanceof KeyValueReader) {
            reader = (KeyValueReader) r;
        } else {
            shuffleInput = true;
            shuffleReader = (KeyValuesReader) r;
        }
        LOG.info("Attached input from vertex " + inputKey + " : input=" + input + ", reader=" + r);
    } catch (Exception e) {
        throw new ExecException(e);
    }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 1);
  Preconditions.checkState(getOutputs().size() == 1);
  LogicalInput input = getInputs().values().iterator().next();
  Reader rawReader = input.getReader();
  Preconditions.checkState(rawReader instanceof KeyValueReader);
  LogicalOutput output = getOutputs().values().iterator().next();

  KeyValueReader reader = (KeyValueReader) rawReader;
  KeyValueWriter writer = (KeyValueWriter) output.getWriter();

  while (reader.next()) {
    Object val = reader.getCurrentValue();
    writer.write(val, NullWritable.get());
  }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 2);
  Preconditions.checkState(getOutputs().size() == 1);
  // Get the input data for the 2 sides of the join from the 2 inputs
  LogicalInput logicalInput1 = getInputs().get(input1);
  LogicalInput logicalInput2 = getInputs().get(input2);
  Reader inputReader1 = logicalInput1.getReader();
  Reader inputReader2 = logicalInput2.getReader();
  Preconditions.checkState(inputReader1 instanceof KeyValuesReader);
  Preconditions.checkState(inputReader2 instanceof KeyValuesReader);
  LogicalOutput lo = getOutputs().get(joinOutput);
  Preconditions.checkState(lo.getWriter() instanceof KeyValueWriter);
  KeyValueWriter writer = (KeyValueWriter) lo.getWriter();

  join((KeyValuesReader) inputReader1, (KeyValuesReader) inputReader2,
      writer);
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 1);
  Preconditions.checkState(getOutputs().size() == 1);
  // not looking up inputs and outputs by name because there is just one
  // instance and this processor is used in many different DAGs with
  // different names for inputs and outputs
  LogicalInput input = getInputs().values().iterator().next();
  Reader rawReader = input.getReader();
  Preconditions.checkState(rawReader instanceof KeyValueReader);
  LogicalOutput output = getOutputs().values().iterator().next();

  KeyValueReader reader = (KeyValueReader) rawReader;
  KeyValueWriter writer = (KeyValueWriter) output.getWriter();

  while (reader.next()) {
    Object val = reader.getCurrentValue();
    // The data value itself is the join key. Simply write it out as the
    // key.
    // The output value is null.
    writer.write(val, NullWritable.get());
  }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 2);
  Preconditions.checkState(getOutputs().size() == 0);
  LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
  LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
  Reader lhsReaderRaw = lhsInput.getReader();
  Reader rhsReaderRaw = rhsInput.getReader();
  Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
  Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
  KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
  KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;

  TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
      MISSING_KEY_COUNTER_NAME);

  while (lhsReader.next()) {
    if (rhsReader.next()) {
      if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
        LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
        lhsMissingKeyCounter.increment(1);
      }
    } else {
      lhsMissingKeyCounter.increment(1);
      LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
      break;
    }
  }
  if (rhsReader.next()) {
    lhsMissingKeyCounter.increment(1);
    LOG.info("ExtraKey in rhs: " + lhsReader.getClass());
  }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public void run() throws Exception {
  Preconditions.checkState(getInputs().size() == 2);
  Preconditions.checkState(getOutputs().size() == 0);
  LogicalInput lhsInput = getInputs().get(LHS_INPUT_NAME);
  LogicalInput rhsInput = getInputs().get(RHS_INPUT_NAME);
  Reader lhsReaderRaw = lhsInput.getReader();
  Reader rhsReaderRaw = rhsInput.getReader();
  Preconditions.checkState(lhsReaderRaw instanceof KeyValuesReader);
  Preconditions.checkState(rhsReaderRaw instanceof KeyValuesReader);
  KeyValuesReader lhsReader = (KeyValuesReader) lhsReaderRaw;
  KeyValuesReader rhsReader = (KeyValuesReader) rhsReaderRaw;
  boolean rhsReaderEnd = false;

  TezCounter lhsMissingKeyCounter = getContext().getCounters().findCounter(COUNTER_GROUP_NAME,
      MISSING_KEY_COUNTER_NAME);

  while (lhsReader.next()) {
    if (rhsReader.next()) {
      if (!lhsReader.getCurrentKey().equals(rhsReader.getCurrentKey())) {
        LOG.info("MismatchedKeys: " + "lhs=" + lhsReader.getCurrentKey() + ", rhs=" + rhsReader.getCurrentKey());
        lhsMissingKeyCounter.increment(1);
      }
    } else {
      lhsMissingKeyCounter.increment(1);
      LOG.info("ExtraKey in lhs: " + lhsReader.getClass());
      rhsReaderEnd = true;
      break;
    }
  }
  if (!rhsReaderEnd && rhsReader.next()) {
    lhsMissingKeyCounter.increment(1);
    LOG.info("ExtraKey in rhs: " + rhsReader.getClass());
  }
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  return new KeyValueReader() {
    String[] keys = new String[numRecordPerSrc];

    int i = -1;

    @Override
    public boolean next() throws IOException {
      if (i == -1) {
        for (int j = 0; j < numRecordPerSrc; j++) {
          keys[j] = ""+j;
        }
      }
      i++;
      return i < keys.length;
    }

    @Override
    public Object getCurrentKey() throws IOException {
      return keys[i];
    }

    @Override
    public Object getCurrentValue() throws IOException {
      return keys[i];
    }
  };
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  if (this.exLocation == ExceptionLocation.INPUT_HANDLE_EVENTS) {
    synchronized (condition) {
      // wait for exception thrown from handleEvents. Otherwise,
      // processor may exit before the exception from handleEvents is
      // caught.
      condition.wait();
    }
  }
  if (this.exLocation == ExceptionLocation.INPUT_GET_READER) {
    throw new Exception(this.exLocation.name());
  }
  return null;
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  return null;
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  return new ConcatenatedMergedKeyValueReader();
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  return new ConcatenatedMergedKeyValuesReader();
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  throw new UnsupportedOperationException("getReader not supported. use getKeyValueReaders");
}
 

import org.apache.tez.runtime.api.Reader; //导入依赖的package包/类
@Override
public Reader getReader() throws Exception {
  return reader;
}