Java DistCp类代码示例

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
public void oldtestCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS_URI, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {LOCAL_FS_STR+TEST_ROOT_DIR+"/srcdat",
                      LOCAL_FS_STR+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {LOCAL_FS_STR+TEST_ROOT_DIR+"/srcdat",
                      LOCAL_FS_STR+TEST_ROOT_DIR+"/src2/srcdat",
                      LOCAL_FS_STR+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
@Test
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** Test copying from a source directory that doesn't exist */
@Test
public void testSrcFileNotFound() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
  deldir(localfs, TEST_ROOT_DIR+"/srcdatdoesnotexist");

  int result = ToolRunner.run(new DistCp(new Configuration()),
      new String[] {"file:///"+TEST_ROOT_DIR+"/srcdatdoesnotexist",
          "file:///"+TEST_ROOT_DIR+"/destdat"});

  assertEquals("Should have failed with a -1, indicating invalid arguments",
      -1, result);

  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdatdoesnotexist");
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
public void init(int numPartition) throws Exception {
  this.numPartition = numPartition;
  sourceFiles = HFileGenerator.generateHFiles(fs, conf, tempFolder,
      options.loadOptions.getPartitioner(), numPartition, numPartition * 1000);
  blockSize = 0;
  for (Path path : sourceFiles) {
    long fileSize = new File(path.toString()).length();
    if (fileSize > blockSize) {
      blockSize = fileSize;
    }
  }
  zkManager = mock(ZooKeeperManager.class);
  distCp = mock(DistCp.class);
  job = mock(Job.class);
  when(zkManager.getClusterInfo()).thenReturn(new ClusterInfo(NAME_NODE, REPLICA_FACTOR));
  when(distCp.execute()).thenReturn(job);
  when(job.waitForCompletion(anyBoolean())).then(new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
      Thread.sleep(1000);
      return true;
    }
  });
  doNothing().when(zkManager).lockFileSet(anyString(), any(FileSetInfo.class));
  doNothing().when(zkManager).unlockFileSet(anyString());
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
@Override
protected DistCp getDistCp(Configuration conf, DistCpOptions options) {
  assertEquals(Constants.MAPRED_MAP_MAX_ATTEMPTS,
      Integer.parseInt(conf.get("mapred.map.max.attempts")));
  assertEquals(Constants.CHECKSUM_BYTES,
      Integer.parseInt(conf.get("io.bytes.per.checksum")));
  long blockSizeExpected = blockSize;
  if (blockSizeExpected % Constants.CHECKSUM_BYTES != 0) {
    blockSizeExpected = (blockSize / Constants.CHECKSUM_BYTES + 1) * Constants.CHECKSUM_BYTES;
  }
  assertEquals(blockSizeExpected, Long.parseLong(conf.get("dfs.block.size")));
  assertEquals(REPLICA_FACTOR, Integer.parseInt(conf.get("dfs.replication")));
  assertEquals(sourceFiles, options.getSourcePaths());
  assertTrue(options.shouldSkipCRC());
  assertTrue(options.shouldSyncFolder());
  assertTrue(options.getTargetPath().toString().startsWith("hdfs://" + NAME_NODE + HDFS_DIR));
  if (numPartition == 1) {
    assertTrue(options.getTargetPath().toString()
        .endsWith(TerrapinUtil.formatPartitionName(0)));
  }
  return distCp;
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
public void distCopy(Path sourceFile, Path destFile) {
    try {
        Configuration conf = new Configuration();
        DistCp distCp = new DistCp(conf, new DistCpOptions(sourceFile, destFile));
        distCp.run(new String[] { sourceFile.toString(), destFile.toString() });
    } catch (Exception e) {
        log.error("Couldn't execute distCp from {} to {}", sourceFile.toString(), destFile.toString());
        e.printStackTrace();
    }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
public void testCopyDuplication() throws Exception {
  final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
  try {    
    MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
    ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
    assertTrue("Source and destination directories do not match.",
               checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));

    assertEquals(DistCp.DuplicationException.ERROR_CODE,
        ToolRunner.run(new DistCp(new Configuration()),
        new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
                      "file:///"+TEST_ROOT_DIR+"/destdat",}));
  }
  finally {
    deldir(localfs, TEST_ROOT_DIR+"/destdat");
    deldir(localfs, TEST_ROOT_DIR+"/srcdat");
    deldir(localfs, TEST_ROOT_DIR+"/src2");
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster(conf, 2, true, null);
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCp(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/**
 * Run distcp in a separate thread, but kill the thread if runtime exceeds timeout.
 *
 * @param distCp directory copier object
 * @param options the command line arguments to pass to DistCp
 * @param timeout the maximum number of miliseconds that DistCp should run
 * @param pollInterval how frequently to check if DistCp is done
 * @return the value returned by DistCp
 *
 * @throws InterruptedException if this thread is interrupted while waiting for DistCp
 */
private int runDistCp(final DistCp distCp, final List<String> options, long timeout,
    long pollInterval) throws DistCpException {

  // Kick off distcp in a separate thread so we can implement a timeout
  final Container<Integer> retVal = new Container<Integer>();
  Thread distCpRunner = new Thread() {
    @Override
    public void run() {
      int ret = distCp.run(options.toArray(new String[] {}));
      retVal.set(Integer.valueOf(ret));
    }
  };
  distCpRunner.setDaemon(true);
  distCpRunner.setName(Thread.currentThread().getName() + "-distcp-" + distCpRunner.getId());
  distCpRunner.start();

  long startTime = System.currentTimeMillis();
  while (true) {
    if (System.currentTimeMillis() - startTime > timeout) {
      LOG.debug(String.format("DistCp exceeded timeout of %sms", timeout));
      distCpRunner.interrupt();
      break;
    }

    if (retVal.get() != null) {
      break;
    }

    try {
      Thread.sleep(pollInterval);
    } catch (InterruptedException e) {
      throw new DistCpException(e);
    }
  }

  return retVal.get() == null ? -1 : retVal.get();
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** copy files from local file system to local file system */
public void testCopyFromLocalToLocal() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS_URI, conf);
  MyFile[] files = createFiles(LOCAL_FS_URI, TEST_ROOT_DIR+"/srcdat");
  ToolRunner.run(new DistCp(new Configuration()),
                         new String[] {LOCAL_FS_STR+TEST_ROOT_DIR+"/srcdat",
                                       LOCAL_FS_STR+TEST_ROOT_DIR+"/destdat"});
  assertTrue("Source and destination directories do not match.",
             checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** copy files from local file system to local file system
 *  using relative short path name to verify
 *  DistCp supports relative short path name */
public void testCopyFromLocalToLocalUsingRelativePathName() throws Exception {
  Configuration conf = new Configuration();
  FileSystem localfs = FileSystem.get(LOCAL_FS_URI, conf);
  MyFile[] files = createFiles(LOCAL_FS_URI, TEST_ROOT_DIR+"/srcdat");

  // get DistCp source and destination relative path
  Path src = new Path(TEST_ROOT_DIR+"/srcdat");
  Path srcRoot = new Path(".");
  FileStatus srcfilestat = localfs.getFileStatus(src);
  FileStatus srcRootfilestat = localfs.getFileStatus(srcRoot);
  // srcStr is file:/{hadoop-src-root}/build/test/data/srcdat
  String srcStr = srcfilestat.getPath().toString();
  // srcRootStr is file:/{hadoop-src-root}
  String srcRootStr = srcRootfilestat.getPath().toString();
  // +1 to remove /, srcRelativePath is build/test/data/srcdat
  String srcRelativePath = srcStr.substring(srcRootStr.length() + 1);
  // replace "srcdat" with "destdat" to get destRelativePath,
  // destRelativePath is build/test/data/destdat
  String destRelativePath = srcRelativePath.substring(0,
      srcRelativePath.length() - "srcdat".length()) + "destdat";

  // run DistCp with relative path source and destination parameters
  ToolRunner.run(new DistCp(new Configuration()),
      new String[] {srcRelativePath, destRelativePath});

  // check result
  assertTrue("Source and destination directories do not match.",
      checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));

  // clean up
  deldir(localfs, TEST_ROOT_DIR+"/destdat");
  deldir(localfs, TEST_ROOT_DIR+"/srcdat");
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/**
 * copy files from dfs file system to dfs file system Pass option to use
 * -skiptmp flag
 */
private void testCopyFromDfsToDfs(boolean skipTmp) throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster(conf, 2, true, null);
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      List<String> argList = new ArrayList<String>();
      addToArgList(argList, "-log", namenode + "/logs");
      addSrcDstToArgList(argList, skipTmp, namenode + "/destdat", namenode
          + "/srcdat");
      ToolRunner.run(new DistCp(conf),
          argList.toArray(new String[argList.size()]));
      assertTrue("Source and destination directories do not match.",
          checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode + "/logs"), conf);
      assertTrue("Log directory does not exist.",
          fs.exists(new Path(namenode + "/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** copy files from local file system to dfs file system */
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster(conf, 1, true, null);
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS_URI, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCp(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       LOCAL_FS_STR+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS_URI, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS_URI, conf);
    cluster = new MiniDFSCluster(conf, 1, true, null);
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCp(conf), new String[] {
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       LOCAL_FS_STR+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, TEST_ROOT_DIR+"/destdat");
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
 

import org.apache.hadoop.tools.DistCp; //导入依赖的package包/类
/**
 * verify that -delete option works for other {@link FileSystem}
 * implementations. See MAPREDUCE-1285 */
public void testDeleteLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS_URI, conf);
    cluster = new MiniDFSCluster(conf, 1, true, null);
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      String destdir = TEST_ROOT_DIR + "/destdat";
      MyFile[] localFiles = createFiles(localfs, destdir);
      ToolRunner.run(new DistCp(conf), new String[] {
                                       "-delete",
                                       "-update",
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       LOCAL_FS_STR+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, destdir, files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, destdir);
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}