Java TestDistCpUtils.createFile方法代码示例

import org.apache.hadoop.tools.util.TestDistCpUtils; //导入方法依赖的package包/类
@Test(timeout=10000)
public void testDuplicates() {
  FileSystem fs = null;
  try {
    fs = FileSystem.get(getConf());
    List<Path> srcPaths = new ArrayList<Path>();
    srcPaths.add(new Path("/tmp/in/*/*"));
    TestDistCpUtils.createFile(fs, "/tmp/in/src1/1.txt");
    TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt");
    Path target = new Path("/tmp/out");
    Path listingFile = new Path("/tmp/list");
    DistCpOptions options = new DistCpOptions(srcPaths, target);
    CopyListing listing = CopyListing.getCopyListing(getConf(), CREDENTIALS, options);
    try {
      listing.buildListing(listingFile, options);
      Assert.fail("Duplicates not detected");
    } catch (DuplicateFileException ignore) {
    }
  } catch (IOException e) {
    LOG.error("Exception encountered in test", e);
    Assert.fail("Test failed " + e.getMessage());
  } finally {
    TestDistCpUtils.delete(fs, "/tmp");
  }
}
 

import org.apache.hadoop.tools.util.TestDistCpUtils; //导入方法依赖的package包/类
@Test(timeout=10000)
public void testBuildListingForSingleFile() {
  FileSystem fs = null;
  String testRootString = "/singleFileListing";
  Path testRoot = new Path(testRootString);
  SequenceFile.Reader reader = null;
  try {
    fs = FileSystem.get(getConf());
    if (fs.exists(testRoot))
      TestDistCpUtils.delete(fs, testRootString);

    Path sourceFile = new Path(testRoot, "/source/foo/bar/source.txt");
    Path decoyFile  = new Path(testRoot, "/target/moo/source.txt");
    Path targetFile = new Path(testRoot, "/target/moo/target.txt");

    TestDistCpUtils.createFile(fs, sourceFile.toString());
    TestDistCpUtils.createFile(fs, decoyFile.toString());
    TestDistCpUtils.createFile(fs, targetFile.toString());

    List<Path> srcPaths = new ArrayList<Path>();
    srcPaths.add(sourceFile);

    DistCpOptions options = new DistCpOptions(srcPaths, targetFile);
    CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);

    final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
    listing.buildListing(listFile, options);

    reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile));

    CopyListingFileStatus fileStatus = new CopyListingFileStatus();
    Text relativePath = new Text();
    Assert.assertTrue(reader.next(relativePath, fileStatus));
    Assert.assertTrue(relativePath.toString().equals(""));
  }
  catch (Exception e) {
    Assert.fail("Unexpected exception encountered.");
    LOG.error("Unexpected exception: ", e);
  }
  finally {
    TestDistCpUtils.delete(fs, testRootString);
    IOUtils.closeStream(reader);
  }
}
 

import org.apache.hadoop.tools.util.TestDistCpUtils; //导入方法依赖的package包/类
@Test
public void testDeleteMissingFlatInterleavedFiles() {
  TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
  JobContext jobContext = new JobContextImpl(taskAttemptContext.getConfiguration(),
      taskAttemptContext.getTaskAttemptID().getJobID());
  Configuration conf = jobContext.getConfiguration();


  String sourceBase;
  String targetBase;
  FileSystem fs = null;
  try {
    OutputCommitter committer = new CopyCommitter(null, taskAttemptContext);
    fs = FileSystem.get(conf);
    sourceBase = "/tmp1/" + String.valueOf(rand.nextLong());
    targetBase = "/tmp1/" + String.valueOf(rand.nextLong());
    TestDistCpUtils.createFile(fs, sourceBase + "/1");
    TestDistCpUtils.createFile(fs, sourceBase + "/3");
    TestDistCpUtils.createFile(fs, sourceBase + "/4");
    TestDistCpUtils.createFile(fs, sourceBase + "/5");
    TestDistCpUtils.createFile(fs, sourceBase + "/7");
    TestDistCpUtils.createFile(fs, sourceBase + "/8");
    TestDistCpUtils.createFile(fs, sourceBase + "/9");

    TestDistCpUtils.createFile(fs, targetBase + "/2");
    TestDistCpUtils.createFile(fs, targetBase + "/4");
    TestDistCpUtils.createFile(fs, targetBase + "/5");
    TestDistCpUtils.createFile(fs, targetBase + "/7");
    TestDistCpUtils.createFile(fs, targetBase + "/9");
    TestDistCpUtils.createFile(fs, targetBase + "/A");

    DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), 
        new Path("/out"));
    options.setSyncFolder(true);
    options.setDeleteMissing(true);
    options.appendToConf(conf);

    CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
    Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
    listing.buildListing(listingFile, options);

    conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase);
    conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase);

    committer.commitJob(jobContext);
    if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
      Assert.fail("Source and target folders are not in sync");
    }
    Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);

    //Test for idempotent commit
    committer.commitJob(jobContext);
    if (!TestDistCpUtils.checkIfFoldersAreInSync(fs, targetBase, sourceBase)) {
      Assert.fail("Source and target folders are not in sync");
    }
    Assert.assertEquals(fs.listStatus(new Path(targetBase)).length, 4);
  } catch (IOException e) {
    LOG.error("Exception encountered while testing for delete missing", e);
    Assert.fail("Delete missing failure");
  } finally {
    TestDistCpUtils.delete(fs, "/tmp1");
    conf.set(DistCpConstants.CONF_LABEL_DELETE_MISSING, "false");
  }

}