Java LoadIncrementalHFiles.doBulkLoad方法代码示例

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void bulkLoadHFile(
    TableName tableName,
    byte[] family,
    byte[] qualifier,
    byte[][][] hfileRanges,
    int numRowsPerRange) throws Exception {

  Path familyDir = new Path(loadPath, Bytes.toString(family));
  fs.mkdirs(familyDir);
  int hfileIdx = 0;
  for (byte[][] range : hfileRanges) {
    byte[] from = range[0];
    byte[] to = range[1];
    createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
        family, qualifier, from, to, numRowsPerRange);
  }
  //set global read so RegionServer can move it
  setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));

  try (Connection conn = ConnectionFactory.createConnection(conf);
       HTable table = (HTable)conn.getTable(tableName)) {
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    loader.doBulkLoad(loadPath, table);
  }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
public void doBulkLoadSinglePut(boolean verbose, HTable table) throws IOException, ClassNotFoundException, InterruptedException, Exception {
    ClassTools.preLoad(LoadIncrementalHFiles.class);

    // setup the bulkload temp folder
    HDFSPath bulkLoadPath = new HDFSPath(
            getConfiguration(),
            "/tmp/" + UUID.randomUUID().toString());
    if (bulkLoadPath.existsDir()) {
        bulkLoadPath.trash();
    }

    // setup the job
    setMapOutputKeyClass(ImmutableBytesWritable.class);
    setMapOutputValueClass(Put.class);
    HFileOutputFormat2.configureIncrementalLoad(this, table);
    HFileOutputFormat2.setOutputPath(this, bulkLoadPath);
    if (waitForCompletion(verbose)) {
        // Load generated HFiles into table
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
        loader.doBulkLoad(bulkLoadPath, table);
    } else {
        log.info("loading failed.");
    }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
  
  if (args.length == 0) {
    System.out.println("ImportToLargerTableMain {originalHFilePath} {largeTableName}");
    return;
  }
  
  String output = args[0];
  String hTableName = args[1];
  
  Configuration config = HBaseConfiguration.create();
  HBaseConfiguration.addHbaseResources(config);
  
  HTable hTable = new HTable(config, hTableName);
  
  FileSystem hdfs = FileSystem.get(config);
  
  //Must all HBase to have write access to HFiles
  HFileUtils.changePermissionR(output, hdfs);
  
  LoadIncrementalHFiles load = new LoadIncrementalHFiles(config);
  load.doBulkLoad(new Path(output), hTable);
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
@Override
protected void postJobCompletion(Job job) {

	// If job is successful, load it into HBase
	try {
		if (job.isSuccessful()) {
			LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
					getConf());
			loader.doBulkLoad(outputDir, htable);
			System.out.println("MapReduce and bulk load successful");
		} else {
			System.err
					.println("MapReduce job failed.  Skipping bulk load.");
		}
	} catch (Exception e) {
		e.printStackTrace();
	}
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
@Override
public void call(BulkImportPartition importPartition) throws Exception {
    Configuration conf = HConfiguration.unwrapDelegate();
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    FileSystem fs = FileSystem.get(URI.create(bulkImportDirectory), conf);
    Long conglomerateId = importPartition.getConglomerateId();
    PartitionFactory tableFactory= SIDriver.driver().getTableFactory();
    try(Partition partition=tableFactory.getTable(Long.toString(conglomerateId))){
        Path path = new Path(importPartition.getFilePath()).getParent();
        if (fs.exists(path)) {
            loader.doBulkLoad(path,(HTable) ((ClientPartition)partition).unwrapDelegate());
            fs.delete(path, true);
        } else {
            LOG.warn("Path doesn't exist, nothing to load into this partition? " + path);
        }
        if (LOG.isDebugEnabled()) {
            SpliceLogUtils.debug(LOG, "Loaded file %s", path.toString());
        }
    }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
/**
 * Perform the loading of Hfiles.
 */
@Override
protected void completeImport(Job job) throws IOException, ImportException {
  super.completeImport(job);

  FileSystem fileSystem = FileSystem.get(job.getConfiguration());

  // Make the bulk load files source directory accessible to the world
  // so that the hbase user can deal with it
  Path bulkLoadDir = getContext().getDestination();
  setPermission(fileSystem, fileSystem.getFileStatus(bulkLoadDir),
    FsPermission.createImmutable((short) 00777));

  HTable hTable = new HTable(job.getConfiguration(), options.getHBaseTable());

  // Load generated HFiles into table
  try {
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
      job.getConfiguration());
    loader.doBulkLoad(bulkLoadDir, hTable);
  }
  catch (Exception e) {
    String errorMessage = String.format("Unrecoverable error while " +
      "performing the bulk load of files in [%s]",
      bulkLoadDir.toString());
    throw new ImportException(errorMessage, e);
  }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
@Test
public void testBulkLoad() throws Exception {
  TableName tableName = TableName.valueOf("testBulkLoad");
  long l = System.currentTimeMillis();
  HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
  createTable(admin, tableName);
  Scan scan = createScan();
  final HTable table = init(admin, l, scan, tableName);
  // use bulkload
  final Path hfilePath = writeToHFile(l, "/temp/testBulkLoad/", "/temp/testBulkLoad/col/file",
    false);
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true);
  final LoadIncrementalHFiles bulkload = new LoadIncrementalHFiles(conf);
  bulkload.doBulkLoad(hfilePath, table);
  ResultScanner scanner = table.getScanner(scan);
  Result result = scanner.next();
  result = scanAfterBulkLoad(scanner, result, "version2");
  Put put0 = new Put(Bytes.toBytes("row1"));
  put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes
      .toBytes("version3")));
  table.put(put0);
  admin.flush(tableName);
  scanner = table.getScanner(scan);
  result = scanner.next();
  while (result != null) {
    List<KeyValue> kvs = result.getColumn(Bytes.toBytes("col"), Bytes.toBytes("q"));
    for (KeyValue _kv : kvs) {
      if (Bytes.toString(_kv.getRow()).equals("row1")) {
        System.out.println(Bytes.toString(_kv.getRow()));
        System.out.println(Bytes.toString(_kv.getQualifier()));
        System.out.println(Bytes.toString(_kv.getValue()));
        Assert.assertEquals("version3", Bytes.toString(_kv.getValue()));
      }
    }
    result = scanner.next();
  }
  scanner.close();
  table.close();
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void LoadHFile2HBase(Configuration conf,String tableName,String hfile) throws Exception{
	conf.set("hbase.metrics.showTableName", "false");
	LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
	HBaseAdmin admin = new HBaseAdmin(conf);
	HTable table = new HTable(conf, tableName);

	loader.doBulkLoad(new Path(hfile), table);
	table.flushCommits();
	table.close();
	admin.close();
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void LoadHFile2HBase(Configuration conf, TableName tableName, String hfile) throws Exception {
    conf.set("hbase.metrics.showTableName", "false");
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    Admin admin = conn.getAdmin();
    Table table = conn.getTable(tableName);
    RegionLocator rl = conn.getRegionLocator(tableName);
    loader.doBulkLoad(new Path(hfile), admin, table, rl);
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
/**
 * Submits the job and waits for completion.
 * @param job job
 * @param outputPath output path
 * @throws Exception
 */
private void configureRunnableJobUsingBulkLoad(Job job, Path outputPath, TableName outputTableName,
                                               boolean skipDependencyJars) throws Exception {
    job.setMapperClass(getBulkMapperClass());
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(KeyValue.class);
    final Configuration configuration = job.getConfiguration();
    try (Connection conn = ConnectionFactory.createConnection(configuration);
         Admin admin = conn.getAdmin();
         Table table = conn.getTable(outputTableName);
         RegionLocator regionLocator = conn.getRegionLocator(outputTableName)) {
        HFileOutputFormat2.configureIncrementalLoad(job, table, regionLocator);
        if (skipDependencyJars) {
            job.getConfiguration().unset("tmpjars");
        }
        boolean status = job.waitForCompletion(true);
        if (!status) {
            LOG.error("IndexTool job failed!");
            throw new Exception("IndexTool job failed: " + job.toString());
        }

        LOG.info("Loading HFiles from {}", outputPath);
        LoadIncrementalHFiles loader = new LoadIncrementalHFiles(configuration);
        loader.doBulkLoad(outputPath, admin, table, regionLocator);
    }
    FileSystem.get(configuration).delete(outputPath, true);
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void bulkLoadHFile(
    TableName tableName,
    byte[] family,
    byte[] qualifier,
    byte[][][] hfileRanges,
    int numRowsPerRange) throws Exception {

  Path familyDir = new Path(loadPath, Bytes.toString(family));
  fs.mkdirs(familyDir);
  int hfileIdx = 0;
  for (byte[][] range : hfileRanges) {
    byte[] from = range[0];
    byte[] to = range[1];
    createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
        family, qualifier, from, to, numRowsPerRange);
  }
  //set global read so RegionServer can move it
  setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));

  try (HTable table = (HTable)TEST_UTIL.getConnection().getTable(tableName)) {
    try (Admin admin = TEST_UTIL.getHBaseAdmin()) {
      TEST_UTIL.waitTableEnabled(admin, tableName.getName());
      LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
      loader.doBulkLoad(loadPath, table);
    }
  }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void bulkLoadHFile(
    TableName tableName,
    byte[] family,
    byte[] qualifier,
    byte[][][] hfileRanges,
    int numRowsPerRange) throws Exception {

  Path familyDir = new Path(loadPath, Bytes.toString(family));
  fs.mkdirs(familyDir);
  int hfileIdx = 0;
  for (byte[][] range : hfileRanges) {
    byte[] from = range[0];
    byte[] to = range[1];
    createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
        family, qualifier, from, to, numRowsPerRange);
  }
  //set global read so RegionServer can move it
  setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));

  HTable table = new HTable(conf, tableName);
  try {
    HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
    TEST_UTIL.waitTableEnabled(admin, tableName.getName());
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    loader.doBulkLoad(loadPath, table);
  } finally {
    table.close();
  }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
private void bulkLoadHFile(
    byte[] tableName,
    byte[] family,
    byte[] qualifier,
    byte[][][] hfileRanges,
    int numRowsPerRange) throws Exception {

  Path familyDir = new Path(loadPath, Bytes.toString(family));
  fs.mkdirs(familyDir);
  int hfileIdx = 0;
  for (byte[][] range : hfileRanges) {
    byte[] from = range[0];
    byte[] to = range[1];
    createHFile(new Path(familyDir, "hfile_"+(hfileIdx++)),
        family, qualifier, from, to, numRowsPerRange);
  }
  //set global read so RegionServer can move it
  setPermission(loadPath, FsPermission.valueOf("-rwxrwxrwx"));

  HTable table = new HTable(conf, tableName);
  try {
    TEST_UTIL.waitTableAvailable(tableName, 30000);
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
    loader.doBulkLoad(loadPath, table);
  } finally {
    table.close();
  }
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
public void completeImport() throws Exception {
  LoadIncrementalHFiles loader = new LoadIncrementalHFiles(getConfiguration());
  HTable table = new HTable(getConfiguration(), _tableName);
  loader.doBulkLoad(_hfilePath, table);

  FileSystem fs = _hfilePath.getFileSystem(getConfiguration());
  fs.delete(_hfilePath, true);
}
 

import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; //导入方法依赖的package包/类
@Test
public void testBulkLoadNativeHFile() throws Exception {
  TableName tableName = TableName.valueOf("testBulkLoadNativeHFile");
  long l = System.currentTimeMillis();
  HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
  createTable(admin, tableName);
  Scan scan = createScan();
  final HTable table = init(admin, l, scan, tableName);
  // use bulkload
  final Path hfilePath = writeToHFile(l, "/temp/testBulkLoadNativeHFile/",
    "/temp/testBulkLoadNativeHFile/col/file", true);
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true);
  final LoadIncrementalHFiles bulkload = new LoadIncrementalHFiles(conf);
  bulkload.doBulkLoad(hfilePath, table);
  ResultScanner scanner = table.getScanner(scan);
  Result result = scanner.next();
  // We had 'version0', 'version1' for 'row1,col:q' in the table.
  // Bulk load added 'version2'  scanner should be able to see 'version2'
  result = scanAfterBulkLoad(scanner, result, "version2");
  Put put0 = new Put(Bytes.toBytes("row1"));
  put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes
      .toBytes("version3")));
  table.put(put0);
  admin.flush(tableName);
  scanner = table.getScanner(scan);
  result = scanner.next();
  while (result != null) {
    List<KeyValue> kvs = result.getColumn(Bytes.toBytes("col"), Bytes.toBytes("q"));
    for (KeyValue _kv : kvs) {
      if (Bytes.toString(_kv.getRow()).equals("row1")) {
        System.out.println(Bytes.toString(_kv.getRow()));
        System.out.println(Bytes.toString(_kv.getQualifier()));
        System.out.println(Bytes.toString(_kv.getValue()));
        Assert.assertEquals("version3", Bytes.toString(_kv.getValue()));
      }
    }
    result = scanner.next();
  }
  scanner.close();
  table.close();
}