Java Reporter.incrCounter方法代码示例

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
/** Run a FileOperation */
public void map(Text key, FileOperation value,
    OutputCollector<WritableComparable<?>, Text> out, Reporter reporter
    ) throws IOException {
  try {
    value.run(jobconf);
    ++succeedcount;
    reporter.incrCounter(Counter.SUCCEED, 1);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);

    String s = "FAIL: " + value + ", " + StringUtils.stringifyException(e);
    out.collect(null, new Text(s));
    LOG.info(s);
  } finally {
    reporter.setStatus(getCountString());
  }
}
 

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
/**
 * Increment number of files copied and bytes copied and then report status
 */
void updateCopyStatus(FileStatus srcstat, Reporter reporter) {
  copycount++;
  reporter.incrCounter(Counter.BYTESCOPIED, srcstat.getLen());
  reporter.incrCounter(Counter.COPY, 1);
  updateStatus(reporter);
}
 

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
/**
 * Skip copying this file if already exists at the destination.
 * Updates counters and copy status if skipping this file.
 * @return true    if copy of this file can be skipped
 */
private boolean skipCopyFile(FileStatus srcstat, Path absdst,
                        OutputCollector<WritableComparable<?>, Text> outc,
                        Reporter reporter) throws IOException {
  if (destFileSys.exists(absdst) && !overwrite
      && !needsUpdate(srcstat, destFileSys, absdst)) {
    outc.collect(null, new Text("SKIP: " + srcstat.getPath()));
    ++skipcount;
    reporter.incrCounter(Counter.SKIP, 1);
    updateStatus(reporter);
    return true;
  }
  return false;
}
 

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
/**
 * Copies single file to the path specified by tmpfile.
 * @param srcstat  src path and metadata
 * @param tmpfile  temporary file to which copy is to be done
 * @param absdst   actual destination path to which copy is to be done
 * @param reporter
 * @return Number of bytes copied
 */
private long doCopyFile(FileStatus srcstat, Path tmpfile, Path absdst,
                        Reporter reporter) throws IOException {
  long bytesCopied = 0L;
  Path srcPath = srcstat.getPath();
  // open src file
  try (FSDataInputStream in = srcPath.getFileSystem(job).open(srcPath)) {
    reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
    // open tmp file
    try (FSDataOutputStream out = create(tmpfile, reporter, srcstat)) {
      LOG.info("Copying file " + srcPath + " of size " +
               srcstat.getLen() + " bytes...");
    
      // copy file
      for(int bytesRead; (bytesRead = in.read(buffer)) >= 0; ) {
        out.write(buffer, 0, bytesRead);
        bytesCopied += bytesRead;
        reporter.setStatus(
            String.format("%.2f ", bytesCopied*100.0/srcstat.getLen())
            + absdst + " [ " +
            TraditionalBinaryPrefix.long2String(bytesCopied, "", 1) + " / "
            + TraditionalBinaryPrefix.long2String(srcstat.getLen(), "", 1)
            + " ]");
      }
    }
  }
  return bytesCopied;
}
 

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
public void map(ImmutableBytesWritable row, Result values,
    OutputCollector<ImmutableBytesWritable, Result> output,
    Reporter reporter)
throws IOException {
    // Count every row containing data, whether it's in qualifiers or values
    reporter.incrCounter(Counters.ROWS, 1);
}
 

import org.apache.hadoop.mapred.Reporter; //导入方法依赖的package包/类
/** Map method. Copies one file from source file system to destination.
 * @param key src len
 * @param value FilePair (FileStatus src, Path dst)
 * @param out Log of failed copies
 * @param reporter
 */
public void map(LongWritable key,
                FilePair value,
                OutputCollector<WritableComparable<?>, Text> out,
                Reporter reporter) throws IOException {
  final FileStatus srcstat = value.input;
  final Path relativedst = new Path(value.output);
  try {
    copyWithRetries(srcstat, relativedst, out, reporter);
  } catch (IOException e) {
    ++failcount;
    reporter.incrCounter(Counter.FAIL, 1);
    updateStatus(reporter);
    final String sfailure = "FAIL " + relativedst + " : " +
                      StringUtils.stringifyException(e);
    out.collect(null, new Text(sfailure));
    LOG.info(sfailure);
    if (e instanceof FileNotFoundException) {
      final String s = "Possible Cause for failure: Either the filesystem "
                       + srcstat.getPath().getFileSystem(job)
                       + " is not accessible or the file is deleted";
      LOG.error(s);
      out.collect(null, new Text(s));
    }

    try {
      for (int i = 0; i < 3; ++i) {
        try {
          final Path tmp = new Path(job.get(TMP_DIR_LABEL), relativedst);
          if (destFileSys.delete(tmp, true))
            break;
        } catch (Throwable ex) {
          // ignore, we are just cleaning up
          LOG.debug("Ignoring cleanup exception", ex);
        }
        // update status, so we don't get timed out
        updateStatus(reporter);
        Thread.sleep(3 * 1000);
      }
    } catch (InterruptedException inte) {
      throw (IOException)new IOException().initCause(inte);
    }
  } finally {
    updateStatus(reporter);
  }
}