Java DistCpConstants类代码示例

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
private Path getTmpFile(Path target, Mapper.Context context) {
  Path targetWorkPath = new Path(context.getConfiguration().
      get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));

  Path root = target.equals(targetWorkPath)? targetWorkPath.getParent() : targetWorkPath;
  LOG.info("Creating temp file: " +
      new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()));
  return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString());
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
private void assertDefaultValues(DistCpOptions distCpOptions) {
  assertThat(distCpOptions, is(not(nullValue())));
  assertThat(distCpOptions.preserveAttributes().hasNext(), is(false));
  assertThat(distCpOptions.shouldPreserveRawXattrs(), is(false));
  assertThat(distCpOptions.shouldAppend(), is(false));
  assertThat(distCpOptions.shouldAtomicCommit(), is(false));
  assertThat(distCpOptions.getAtomicWorkPath(), is(nullValue()));
  assertThat(distCpOptions.shouldBlock(), is(true));
  assertThat(distCpOptions.getCopyStrategy(), is(DistCpConstants.UNIFORMSIZE));
  assertThat(distCpOptions.shouldDeleteMissing(), is(false));
  assertThat(distCpOptions.shouldIgnoreFailures(), is(false));
  assertThat(distCpOptions.getLogPath(), is(nullValue()));
  assertThat(distCpOptions.getMapBandwidth(), is(DistCpConstants.DEFAULT_BANDWIDTH_MB));
  assertThat(distCpOptions.getMaxMaps(), is(DistCpConstants.DEFAULT_MAPS));
  assertThat(distCpOptions.shouldOverwrite(), is(false));
  assertThat(distCpOptions.shouldSkipCRC(), is(false));
  assertThat(distCpOptions.getSslConfigurationFile(), is(nullValue()));
  assertThat(distCpOptions.shouldSyncFolder(), is(false));
  assertThat(distCpOptions.getTargetPathExists(), is(true));
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
/**
 * Implementation of the Mapper::setup() method. This extracts the DistCp-
 * options specified in the Job's configuration, to set up the Job.
 * @param context Mapper's context.
 * @throws IOException On IO failure.
 * @throws InterruptedException If the job is interrupted.
 */
@Override
public void setup(Context context) throws IOException, InterruptedException {
  conf = context.getConfiguration();

  syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false);
  ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
  skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false);
  overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false);
  append = conf.getBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), false);
  preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch.
      PRESERVE_STATUS.getConfigLabel()));

  targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  Path targetFinalPath = new Path(conf.get(
          DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  targetFS = targetFinalPath.getFileSystem(conf);

  if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) {
    overWrite = true; // When target is an existing file, overwrite it.
  }

  if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) {
    initializeSSLConf(context);
  }
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
private static Configuration getConfiguration() throws IOException {
  Configuration configuration = getConfigurationForCluster();
  final FileSystem fs = cluster.getFileSystem();
  Path workPath = new Path(TARGET_PATH)
          .makeQualified(fs.getUri(), fs.getWorkingDirectory());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
          workPath.toString());
  configuration.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
          workPath.toString());
  configuration.setBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(),
          false);
  configuration.setBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(),
          true);
  configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
          "br");
  return configuration;
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
@Test
public void testSetCommitDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getCommitDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setCommitDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getCommitDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Commit Directory");
  }
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
@Test
public void testSetWorkingDirectory() {
  try {
    Job job = Job.getInstance(new Configuration());
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    job.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, "");
    Assert.assertEquals(null, CopyOutputFormat.getWorkingDirectory(job));

    Path directory = new Path("/tmp/test");
    CopyOutputFormat.setWorkingDirectory(job, directory);
    Assert.assertEquals(directory, CopyOutputFormat.getWorkingDirectory(job));
    Assert.assertEquals(directory.toString(), job.getConfiguration().
        get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  } catch (IOException e) {
    LOG.error("Exception encountered while running test", e);
    Assert.fail("Failed while testing for set Working Directory");
  }
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
@Test
public void testDynamicInputChunkContext() throws IOException {
  Configuration configuration = new Configuration();
  configuration.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH,
      "/tmp/test/file1.seq");
  DynamicInputFormat firstInputFormat = new DynamicInputFormat();
  DynamicInputFormat secondInputFormat = new DynamicInputFormat();
  DynamicInputChunkContext firstContext =
      firstInputFormat.getChunkContext(configuration);
  DynamicInputChunkContext secondContext =
      firstInputFormat.getChunkContext(configuration);
  DynamicInputChunkContext thirdContext =
      secondInputFormat.getChunkContext(configuration);
  DynamicInputChunkContext fourthContext =
      secondInputFormat.getChunkContext(configuration);
  Assert.assertTrue("Chunk contexts from the same DynamicInputFormat " +
      "object should be the same.",firstContext.equals(secondContext));
  Assert.assertTrue("Chunk contexts from the same DynamicInputFormat " +
      "object should be the same.",thirdContext.equals(fourthContext));
  Assert.assertTrue("Contexts from different DynamicInputFormat " +
      "objects should be different.",!firstContext.equals(thirdContext));
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
/**
 * Implementation of the Mapper::setup() method. This extracts the DistCp-
 * options specified in the Job's configuration, to set up the Job.
 * @param context Mapper's context.
 * @throws IOException On IO failure.
 * @throws InterruptedException If the job is interrupted.
 */
@Override
public void setup(Context context) throws IOException, InterruptedException {
  conf = context.getConfiguration();

  syncFolders = conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false);
  ignoreFailures = conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
  skipCrc = conf.getBoolean(DistCpOptionSwitch.SKIP_CRC.getConfigLabel(), false);
  overWrite = conf.getBoolean(DistCpOptionSwitch.OVERWRITE.getConfigLabel(), false);
  preserve = DistCpUtils.unpackAttributes(conf.get(DistCpOptionSwitch.
      PRESERVE_STATUS.getConfigLabel()));

  targetWorkPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));
  Path targetFinalPath = new Path(conf.get(
          DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
  targetFS = targetFinalPath.getFileSystem(conf);

  if (targetFS.exists(targetFinalPath) && targetFS.isFile(targetFinalPath)) {
    overWrite = true; // When target is an existing file, overwrite it.
  }

  if (conf.get(DistCpConstants.CONF_LABEL_SSL_CONF) != null) {
    initializeSSLConf(context);
  }
}
 

import org.apache.hadoop.tools.DistCpConstants; //导入依赖的package包/类
private static ThrottledInputStream getInputStream(Path path,
    Configuration conf) throws IOException {
  try {
    FileSystem fs = path.getFileSystem(conf);
    long bandwidthMB = conf.getInt(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
            DistCpConstants.DEFAULT_BANDWIDTH_MB);
    FSDataInputStream in = fs.open(path);
    return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024);
  }
  catch (IOException e) {
    throw new CopyReadException(e);
  }
}