Java Tool.run方法代码示例

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testSplitRamdom() throws Exception {
  Tool tool = new InputSampler<Object,Object>(new Configuration());
  int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
      // Use 0.999 probability to reduce the flakiness of the test because
      // the test will fail if the number of samples is less than (number of reduces + 1).
      "-splitRandom", "0.999f", "20", "100",
      input1, input2, output });
  assertEquals(0, result);
  Object[] partitions = readPartitions(output);
  // must be 3 split points since NUM_REDUCES = 4:
  assertEquals(3, partitions.length);
  // check that the partition array is sorted:
  Object[] sortedPartitions = Arrays.copyOf(partitions, partitions.length);
  Arrays.sort(sortedPartitions, new LongWritable.Comparator());
  assertArrayEquals(sortedPartitions, partitions);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testRun_l2t10() throws Exception {
  // gen rowkeys file for later test
  Configuration conf = TEST_UTIL.getConfiguration();
  String outputPath = "/run_b2t3";
  Tool tool = new GetRandomRowsByRegions(conf);
  int status = tool.run(new String[] { "-b", "2", "-t", "3", VERTEX_TABLE, outputPath });
  Assert.assertEquals(0, status);

  // merge content
  File tf = mergeResults(conf, outputPath, "rowkeys-1");

  // run test
  File tPath = tf.getParentFile();
  tPath = new File(tPath, "performanceTestResults_" + System.currentTimeMillis());
  FileUtils.forceMkdir(tPath);

  tool = new HGraphClientPerformanceTest(conf);
  status =
      tool.run(new String[] { "-l", "2", "-t", "10", VERTEX_TABLE, EDGE_TABLE,
          tf.getAbsolutePath(), tPath.getAbsolutePath() });
  Assert.assertEquals(0, status);

  // verify test results
  outputTestResults(tPath);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testRun_ml2t10() throws Exception {
  // gen rowkeys file for later test
  Configuration conf = TEST_UTIL.getConfiguration();
  String outputPath = "/run_ml2t10";
  Tool tool = new GetRandomRowsByRegions(conf);
  int status = tool.run(new String[] { "-b", "2", "-t", "3", VERTEX_TABLE, outputPath });
  Assert.assertEquals(0, status);

  // merge content
  File tf = mergeResults(conf, outputPath, "rowkeys-2");

  // run test
  File tPath = tf.getParentFile();
  tPath = new File(tPath, "performanceTestResults_" + System.currentTimeMillis());
  FileUtils.forceMkdir(tPath);

  tool = new HGraphClientPerformanceTest(conf);
  status =
      tool.run(new String[] { "-m", "-l", "2", "-t", "10", VERTEX_TABLE, EDGE_TABLE,
          tf.getAbsolutePath(), tPath.getAbsolutePath() });
  Assert.assertEquals(0, status);

  // verify test results
  outputTestResults(tPath);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testRun_i2000l2t10() throws Exception {
  // gen rowkeys file for later test
  Configuration conf = TEST_UTIL.getConfiguration();
  String outputPath = "/run_i2000l2t10";
  Tool tool = new GetRandomRowsByRegions(conf);
  int status = tool.run(new String[] { "-b", "2", "-t", "3", VERTEX_TABLE, outputPath });
  Assert.assertEquals(0, status);

  // merge content
  File tf = mergeResults(conf, outputPath, "rowkeys-2");

  // run test
  File tPath = tf.getParentFile();
  tPath = new File(tPath, "performanceTestResults_" + System.currentTimeMillis());
  FileUtils.forceMkdir(tPath);

  tool = new HGraphClientPerformanceTest(conf);
  status =
      tool.run(new String[] { "-i", "2000", "-l", "2", "-t", "10", VERTEX_TABLE, EDGE_TABLE,
          tf.getAbsolutePath(), tPath.getAbsolutePath() });
  Assert.assertEquals(0, status);

  // verify test results
  outputTestResults(tPath);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testSplitSample() throws Exception {
  Tool tool = new InputSampler<Object,Object>(new Configuration());
  int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
      "-splitSample", "10", "100",
      input1, input2, output });
  assertEquals(0, result);

  Object[] partitions = readPartitions(output);
  assertArrayEquals(
      new LongWritable[] { new LongWritable(2L), new LongWritable(7L), new LongWritable(20L),},
      partitions);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testSplitInterval() throws Exception {
  Tool tool = new InputSampler<Object,Object>(new Configuration());
  int result = tool.run(new String[] { "-r", Integer.toString(NUM_REDUCES),
      "-splitInterval", "0.5f", "0",
      input1, input2, output });
  assertEquals(0, result);
  Object[] partitions = readPartitions(output);
  assertArrayEquals(new LongWritable[] { new LongWritable(7L), new LongWritable(9L),
    new LongWritable(35L),}, partitions);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
private void runBalancerCli(Configuration conf,
    long totalUsedSpace, long totalCapacity) throws Exception {
  waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);

  final String[] args = { "-policy", "datanode" };
  final Tool tool = new Cli();    
  tool.setConf(conf);
  final int r = tool.run(args); // start rebalancing
  
  assertEquals("Tools should exit 0 on success", 0, r);
  waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
  LOG.info("Rebalancing with default ctor.");
  waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
private void runBalancerCli(Configuration conf, long totalUsedSpace,
    long totalCapacity) throws Exception {
  waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);

  final String[] args = {"-policy", "datanode"};
  final Tool tool = new Cli();
  tool.setConf(conf);
  final int r = tool.run(args); // start rebalancing
  
  assertEquals("Tools should exit 0 on success", 0, r);
  waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
  LOG.info("Rebalancing with default ctor.");
  waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
public static int runMRJob(Tool tool, String[] args) throws Exception {
    Configuration conf = tool.getConf();
    if (conf == null) {
        conf = new Configuration();
    }

    GenericOptionsParser parser = getParser(conf, args);
    //set the configuration back, so that Tool can configure itself
    tool.setConf(conf);

    //get the args w/o generic hadoop args
    String[] toolArgs = parser.getRemainingArgs();
    return tool.run(toolArgs);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
private static int execute0(
        Configuration conf,
        ClassLoader classLoader,
        ClassDescription clientClass,
        String executionId,
        Map<String, String> batchArguments) throws Exception {
    try (ClassLoaderContext context = new ClassLoaderContext(classLoader)) {
        Configuration copy = new Configuration(conf);
        copy.setClassLoader(classLoader);
        Tool tool = resolveClient(copy, clientClass);
        configure(copy, executionId, batchArguments);
        return tool.run(EMPTY_ARGUMENTS);
    }
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
protected void walToHFiles(List<String> dirPaths, TableName tableName) throws IOException {
  Tool player = new WALPlayer();

  // Player reads all files in arbitrary directory structure and creates
  // a Map task for each file. We use ';' as separator
  // because WAL file names contains ','
  String dirs = StringUtils.join(dirPaths, ';');
  String jobname = "Incremental_Backup-" + backupId + "-" + tableName.getNameAsString();

  Path bulkOutputPath = getBulkOutputDirForTable(tableName);
  conf.set(WALPlayer.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString());
  conf.set(WALPlayer.INPUT_FILES_SEPARATOR_KEY, ";");
  conf.set(JOB_NAME_CONF_KEY, jobname);
  String[] playerArgs = { dirs, tableName.getNameAsString() };

  try {
    player.setConf(conf);
    int result = player.run(playerArgs);
    if(result != 0) {
      throw new IOException("WAL Player failed");
    }
    conf.unset(WALPlayer.INPUT_FILES_SEPARATOR_KEY);
    conf.unset(JOB_NAME_CONF_KEY);
  } catch (IOException e) {
    throw e;
  } catch (Exception ee) {
    throw new IOException("Can not convert from directory " + dirs
        + " (check Hadoop, HBase and WALPlayer M/R job logs) ", ee);
  }
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
void testJoin(Tool bookAndAuthorJoin, boolean mapOnly) throws Exception {
  File authorsFile = getTestTempFile("authors.tsv");
  File booksFile = getTestTempFile("books.tsv");
  File outputDir = getTestTempDir("output");
  outputDir.delete();

  writeLines(authorsFile, readLines("/assignment1/authors.tsv"));
  writeLines(booksFile, readLines("/assignment1/books.tsv"));

  Configuration conf = new Configuration();

  bookAndAuthorJoin.setConf(conf);
  bookAndAuthorJoin.run(new String[] { "--authors", authorsFile.getAbsolutePath(),
      "--books", booksFile.getAbsolutePath(), "--output", outputDir.getAbsolutePath() });

  String outputFilename = mapOnly ? "part-m-00000" : "part-r-00000";
  
  Multimap<String, Book> booksByAuthors = readBooksByAuthors(new File(outputDir, outputFilename));

  assertTrue(booksByAuthors.containsKey("Charles Bukowski"));
  assertTrue(booksByAuthors.get("Charles Bukowski")
      .contains(new Book("Confessions of a Man Insane Enough to Live with Beasts", 1965)));
  assertTrue(booksByAuthors.get("Charles Bukowski")
      .contains(new Book("Hot Water Music", 1983)));

  assertTrue(booksByAuthors.containsKey("Fyodor Dostoyevsky"));
  assertTrue(booksByAuthors.get("Fyodor Dostoyevsky").contains(new Book("Crime and Punishment", 1866)));
  assertTrue(booksByAuthors.get("Fyodor Dostoyevsky").contains(new Book("The Brothers Karamazov", 1880)));

}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testRun_b4() throws IOException, Exception {
  // init test table
  String tableName = "test.vertex-02";
  String outputPath = "/run_b4";
  createTestTable(tableName, "00030", "00060");

  // start test
  Configuration conf = TEST_UTIL.getConfiguration();
  Tool driver = new CalculateInputSplitMapper(conf);
  int code = driver.run(new String[] { "-b", "4", tableName, outputPath });
  Assert.assertEquals(0, code);

  // get test results
  Path path = new Path(outputPath);
  FileSystem fs = path.getFileSystem(conf);

  // FileStatus[] files = fs.listStatus(path);
  // for (int a = 0; a < files.length; a++) {
  // System.out.println(files[a].getPath());
  // }
  InputStream is = fs.open(new Path(path, "part-r-00000"));
  LineIterator it = IOUtils.lineIterator(is, "UTF-8");
  System.out.println("print out test results");
  while (it.hasNext()) {
    System.out.println(it.next());
  }
  LineIterator.closeQuietly(it);
  IOUtils.closeQuietly(is);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
@Test
public void testRun() throws Exception {
  importData(new String[] { "-Dimporttsv.columns=HBASE_ROW_KEY," + CF[0] + ":[email protected]",
      "-Dimporttsv.separator=|" }, TABLE, CF,
    "org/trend/hgraph/mapreduce/pagerank/vertex-test-01.data");
  printTable(TABLE);
  
  Tool tool = new ResetPageRankUpdateFlag(TEST_UTIL.getConfiguration());
  int status = tool.run(new String[] { TABLE });
  Assert.assertEquals(0, status);

  printTable(TABLE);
}
 

import org.apache.hadoop.util.Tool; //导入方法依赖的package包/类
public int run(Tool tool, String[] args) throws Exception
{
	Configuration conf = new Configuration();
   tool.setConf(conf);

   return tool.run(args);
 }