Java HBaseTestingUtility.cleanupDataTestDirOnTestFS方法代码示例

import org.apache.hadoop.hbase.HBaseTestingUtility; //导入方法依赖的package包/类
protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
    String[] args, int valueMultiplier) throws Exception {
  TableName table = TableName.valueOf(args[args.length - 1]);
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util
      .getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  try {
    // Job will fail if observer rejects entries without TTL
    assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));
  } finally {
    // Clean up
    if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
      LOG.debug("Deleting test subdirectory");
      util.cleanupDataTestDirOnTestFS(table.getNameAsString());
    }
  }

  return tool;
}
 

import org.apache.hadoop.hbase.HBaseTestingUtility; //导入方法依赖的package包/类
/**
 * Run an ImportTsv job and perform basic validation on the results. Returns
 * the ImportTsv <code>Tool</code> instance so that other tests can inspect it
 * for further validation as necessary. This method is static to insure
 * non-reliance on instance's util/conf facilities.
 * 
 * @param args
 *          Any arguments to pass BEFORE inputFile path is appended.
 * @param dataAvailable
 * @return The Tool instance used to run the test.
 */
private Tool doMROnTableTest(HBaseTestingUtility util, String family, String data, String[] args,
    int valueMultiplier, boolean dataAvailable) throws Exception {
  String table = args[args.length - 1];
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  validateTable(conf, TableName.valueOf(table), family, valueMultiplier, dataAvailable);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table);
  }
  return tool;
}
 

import org.apache.hadoop.hbase.HBaseTestingUtility; //导入方法依赖的package包/类
/**
 * Run an ImportTsv job and perform basic validation on the results.
 * Returns the ImportTsv <code>Tool</code> instance so that other tests can
 * inspect it for further validation as necessary. This method is static to
 * insure non-reliance on instance's util/conf facilities.
 * @param args Any arguments to pass BEFORE inputFile path is appended.
 * @return The Tool instance used to run the test.
 */
protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
    String[] args, int valueMultiplier, int expectedKVCount)
throws Exception {
  String table = args[args.length - 1];
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util.getDataTestDirOnTestFS(table), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  if (data == null) {
    data = "KEY\u001bVALUE1\u001bVALUE2\n";
  }
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  // Perform basic validation. If the input args did not include
  // ImportTsv.BULK_OUTPUT_CONF_KEY then validate data in the table.
  // Otherwise, validate presence of hfiles.
  boolean createdHFiles = false;
  String outputPath = null;
  for (String arg : argv) {
    if (arg.contains(ImportTsv.BULK_OUTPUT_CONF_KEY)) {
      createdHFiles = true;
      // split '-Dfoo=bar' on '=' and keep 'bar'
      outputPath = arg.split("=")[1];
      break;
    }
  }

  if (createdHFiles)
    validateHFiles(fs, outputPath, family, expectedKVCount);
  else
    validateTable(conf, TableName.valueOf(table), family, valueMultiplier);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table);
  }
  return tool;
}
 

import org.apache.hadoop.hbase.HBaseTestingUtility; //导入方法依赖的package包/类
/**
 * Run an ImportTsv job and perform basic validation on the results. Returns
 * the ImportTsv <code>Tool</code> instance so that other tests can inspect it
 * for further validation as necessary. This method is static to insure
 * non-reliance on instance's util/conf facilities.
 *
 * @param args
 *          Any arguments to pass BEFORE inputFile path is appended.
 * @param expectedKVCount Expected KV count. pass -1 to skip the kvcount check
 *
 * @return The Tool instance used to run the test.
 */
protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
    String[] args, int valueMultiplier, int expectedKVCount) throws Exception {
  TableName table = TableName.valueOf(args[args.length - 1]);
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util
      .getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  if (data == null) {
    data = "KEY\u001bVALUE1\u001bVALUE2\n";
  }
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));

  // Perform basic validation. If the input args did not include
  // ImportTsv.BULK_OUTPUT_CONF_KEY then validate data in the table.
  // Otherwise, validate presence of hfiles.
  boolean createdHFiles = false;
  String outputPath = null;
  for (String arg : argv) {
    if (arg.contains(ImportTsv.BULK_OUTPUT_CONF_KEY)) {
      createdHFiles = true;
      // split '-Dfoo=bar' on '=' and keep 'bar'
      outputPath = arg.split("=")[1];
      break;
    }
  }
  LOG.debug("validating the table " + createdHFiles);
  if (createdHFiles)
   validateHFiles(fs, outputPath, family, expectedKVCount);
  else
    validateTable(conf, table, family, valueMultiplier);

  if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
    LOG.debug("Deleting test subdirectory");
    util.cleanupDataTestDirOnTestFS(table.getNameAsString());
  }
  return tool;
}