Java OutputCollector.collect方法代碼示例

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
/** Combines values for a given key.  
 * @param key the key is expected to be a Text object, whose prefix indicates
 * the type of aggregation to aggregate the values. 
 * @param values the values to combine
 * @param output to collect combined values
 */
public void reduce(Text key, Iterator<Text> values,
                   OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
  String keyStr = key.toString();
  int pos = keyStr.indexOf(ValueAggregatorDescriptor.TYPE_SEPARATOR);
  String type = keyStr.substring(0, pos);
  ValueAggregator aggregator = ValueAggregatorBaseDescriptor
    .generateValueAggregator(type);
  while (values.hasNext()) {
    aggregator.addNextValue(values.next());
  }
  Iterator outputs = aggregator.getCombinerOutput().iterator();

  while (outputs.hasNext()) {
    Object v = outputs.next();
    if (v instanceof Text) {
      output.collect(key, (Text)v);
    } else {
      output.collect(key, new Text(v.toString()));
    }
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
/** Run a FileOperation */
public void map(Text key, FileOperation value,
    OutputCollector<WritableComparable<?>, Text> out, Reporter reporter
    ) throws IOException {
  try {
    value.run(jobconf);
    ++succeedcount;
    reporter.incrCounter(Counter.SUCCEED, 1);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);

    String s = "FAIL: " + value + ", " + StringUtils.stringifyException(e);
    out.collect(null, new Text(s));
    LOG.info(s);
  } finally {
    reporter.setStatus(getCountString());
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
@Override
public void map(ImmutableBytesWritable row, Result result,
    OutputCollector<ImmutableBytesWritable, Put> outCollector,
    Reporter reporter) throws IOException {
  String rowKey = Bytes.toString(result.getRow());
  final ImmutableBytesWritable pKey = new ImmutableBytesWritable(
      Bytes.toBytes(PRESIDENT_PATTERN));
  final ImmutableBytesWritable aKey = new ImmutableBytesWritable(
      Bytes.toBytes(ACTOR_PATTERN));
  ImmutableBytesWritable outKey = null;

  if (rowKey.startsWith(PRESIDENT_PATTERN)) {
    outKey = pKey;
  } else if (rowKey.startsWith(ACTOR_PATTERN)) {
    outKey = aKey;
  } else {
    throw new AssertionError("unexpected rowKey");
  }

  String name = Bytes.toString(result.getValue(COLUMN_FAMILY,
      COLUMN_QUALIFIER));
  outCollector.collect(outKey, new Put(Bytes.toBytes("rowKey2")).add(
      COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(name)));
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
@Override
public void reduce(Text token, Iterator<StringIntPair> values,
    OutputCollector<Key, BusyAirportModel> output, Reporter reporter)
    throws IOException {
  String topAirport = null;
  int max = 0;

  while (values.hasNext()) {
    StringIntPair v = values.next();
    if (v.getSecond() > max) {
      max = v.getSecond();
      topAirport = v.getFirst();
    }
  }
  BusyAirportModel busy = new BusyAirportModel(topAirport, max);
  output.collect(new Key(), busy);
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
@Override
public void map(Object key, ResultSet rs,
    OutputCollector<Text, IntWritable> output,
    Reporter reporter) throws IOException {

  String origAirport;
  String destAirport;
  String combo;

  try {
    origAirport = rs.getString("ORIG_AIRPORT");
    destAirport = rs.getString("DEST_AIRPORT");
    combo = origAirport.compareTo(
        destAirport) < 0 ? origAirport + destAirport : destAirport + origAirport;
    reusableText.set(combo);
    output.collect(reusableText, countOne);
  } catch (SQLException e) {
    e.printStackTrace();
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void map(Object key, Object value,
                OutputCollector output, Reporter reporter) throws IOException {
  if (this.reporter == null) {
    this.reporter = reporter;
  }
  addLongValue("totalCount", 1);
  TaggedMapOutput aRecord = generateTaggedMapOutput(value);
  if (aRecord == null) {
    addLongValue("discardedCount", 1);
    return;
  }
  Text groupKey = generateGroupKey(aRecord);
  if (groupKey == null) {
    addLongValue("nullGroupKeyCount", 1);
    return;
  }
  output.collect(groupKey, aRecord);
  addLongValue("collectedCount", 1);
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
/**
 * Counts the number of inlinks and outlinks for each url and sets a default
 * score of 0.0 for each url (node) in the webgraph.
 */
public void reduce(Text key, Iterator<LinkDatum> values,
    OutputCollector<Text, Node> output, Reporter reporter)
    throws IOException {

  Node node = new Node();
  int numInlinks = 0;
  int numOutlinks = 0;

  // loop through counting number of in and out links
  while (values.hasNext()) {
    LinkDatum next = values.next();
    if (next.getLinkType() == LinkDatum.INLINK) {
      numInlinks++;
    } else if (next.getLinkType() == LinkDatum.OUTLINK) {
      numOutlinks++;
    }
  }

  // set the in and outlinks and a default score of 0
  node.setNumInlinks(numInlinks);
  node.setNumOutlinks(numOutlinks);
  node.setInlinkScore(0.0f);
  output.collect(key, node);
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void map(LongWritable key, Text val,
    OutputCollector<Text, NullWritable> out, Reporter r) throws IOException {

  LOG.info("Mapper input line: " + val.toString());

  try {
    // Use the user's record class to parse the line back in.
    userRecord.parse(val);
  } catch (RecordParser.ParseError pe) {
    LOG.error("Got parse error: " + pe.toString());
    throw new IOException(pe);
  }

  LOG.info("Mapper output line: " + userRecord.toString());

  out.collect(new Text(userRecord.toString()), NullWritable.get());

  if (!userRecord.toString(false).equals(val.toString())) {
    // Could not format record w/o end-of-record delimiter.
    throw new IOException("Returned string w/o EOR has value ["
        + userRecord.toString(false) + "] when ["
        + val.toString() + "] was expected.");
  }

  if (!userRecord.toString().equals(val.toString() + "\n")) {
    // misparsed.
    throw new IOException("Returned string has value ["
        + userRecord.toString() + "] when ["
        + val.toString() + "\n] was expected.");
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void reduce(Object arg0, Iterator arg1, OutputCollector arg2, Reporter arg3) throws IOException {
  int count = 0;
  while (arg1.hasNext()) {
    count += 1;
    arg1.next();
  }
  arg2.collect(arg0, new Text("" + count));
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void map(LongWritable key, HarEntry value,
    OutputCollector<IntWritable, Text> out,
    Reporter reporter) throws IOException {
  Path relPath = new Path(value.path);
  int hash = HarFileSystem.getHarHash(relPath);
  String towrite = null;
  Path srcPath = realPath(relPath, rootPath);
  long startPos = partStream.getPos();
  FileSystem srcFs = srcPath.getFileSystem(conf);
  FileStatus srcStatus = srcFs.getFileStatus(srcPath);
  String propStr = encodeProperties(srcStatus);
  if (value.isDir()) { 
    towrite = encodeName(relPath.toString())
              + " dir " + propStr + " 0 0 ";
    StringBuffer sbuff = new StringBuffer();
    sbuff.append(towrite);
    for (String child: value.children) {
      sbuff.append(encodeName(child) + " ");
    }
    towrite = sbuff.toString();
    //reading directories is also progress
    reporter.progress();
  }
  else {
    FSDataInputStream input = srcFs.open(srcStatus.getPath());
    reporter.setStatus("Copying file " + srcStatus.getPath() + 
        " to archive.");
    copyData(srcStatus.getPath(), input, partStream, reporter);
    towrite = encodeName(relPath.toString())
              + " file " + partname + " " + startPos
              + " " + srcStatus.getLen() + " " + propStr + " ";
  }
  out.collect(new IntWritable(hash), new Text(towrite));
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
/**
 * Runs the given operation and reports on its results
 * 
 * @param op
 *          the operation to run
 * @param reporter
 *          the status reporter to notify
 * @param output
 *          the output to write to
 * @throws IOException
 */
private void runOperation(Operation op, Reporter reporter,
    OutputCollector<Text, Text> output, long opNum) throws IOException {
  if (op == null) {
    return;
  }
  logAndSetStatus(reporter, "Running operation #" + opNum + " (" + op + ")");
  List<OperationOutput> opOut = op.run(filesystem);
  logAndSetStatus(reporter, "Finished operation #" + opNum + " (" + op + ")");
  if (opOut != null && !opOut.isEmpty()) {
    for (OperationOutput outData : opOut) {
      output.collect(outData.getKey(), outData.getOutputValue());
    }
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void map(K key, Text value,
                OutputCollector<Text, LongWritable> output,
                Reporter reporter)
  throws IOException {
  String text = value.toString();
  Matcher matcher = pattern.matcher(text);
  while (matcher.find()) {
    output.collect(new Text(matcher.group(group)), new LongWritable(1));
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
@Override
public void reduce(Text key, Iterator<IntWritable> values,
    OutputCollector<Text, IntWritable> output, Reporter reporter)
    throws IOException {
  int sum = 0;
  while (values.hasNext()) {
    sum += values.next().get();
  }
  if (key.equals(OPEN_EXECTIME)){
    executionTime[OPEN] = sum;
  } else if (key.equals(NUMOPS_OPEN)){
    numOfOps[OPEN] = sum;
  } else if (key.equals(LIST_EXECTIME)){
    executionTime[LIST] = sum;
  } else if (key.equals(NUMOPS_LIST)){
    numOfOps[LIST] = sum;
  } else if (key.equals(DELETE_EXECTIME)){
    executionTime[DELETE] = sum;
  } else if (key.equals(NUMOPS_DELETE)){
    numOfOps[DELETE] = sum;
  } else if (key.equals(CREATE_EXECTIME)){
    executionTime[CREATE] = sum;
  } else if (key.equals(NUMOPS_CREATE)){
    numOfOps[CREATE] = sum;
  } else if (key.equals(WRITE_CLOSE_EXECTIME)){
    System.out.println(WRITE_CLOSE_EXECTIME + " = " + sum);
    executionTime[WRITE_CLOSE]= sum;
  } else if (key.equals(NUMOPS_WRITE_CLOSE)){
    numOfOps[WRITE_CLOSE] = sum;
  } else if (key.equals(TOTALOPS)){
    totalOps = sum;
  } else if (key.equals(ELAPSED_TIME)){
    totalTime = sum;
  }
  result.set(sum);
  output.collect(key, result);
  // System.out.println("Key = " + key + " Sum is =" + sum);
  // printResults(System.out);
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void map(LongWritable key, Text value, 
                OutputCollector<Text, IntWritable> output, 
                Reporter reporter) throws IOException {
  String line = value.toString();
  StringTokenizer itr = new StringTokenizer(line);
  while (itr.hasMoreTokens()) {
    word.set(itr.nextToken());
    output.collect(word, one);
  }
}
 

import org.apache.hadoop.mapred.OutputCollector; //導入方法依賴的package包/類
public void reduce(Text key, Iterator<IntWritable> values,
                   OutputCollector<Text, IntWritable> output, 
                   Reporter reporter) throws IOException {
  int sum = 0;
  while (values.hasNext()) {
    sum += values.next().get();
  }
  output.collect(key, new IntWritable(sum));
}