当前位置: 首页>>代码示例>>Java>>正文

Java DistCpOptions类代码示例

本文整理汇总了Java中org.apache.hadoop.tools.DistCpOptions类的典型用法代码示例。如果您正苦于以下问题：Java DistCpOptions类的具体用法？Java DistCpOptions怎么用？Java DistCpOptions使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。


DistCpOptions类属于org.apache.hadoop.tools包，在下文中一共展示了DistCpOptions类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: doBuildListing

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Override
public void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException {
  try (Writer writer = newWriter(pathToListFile)) {

    Path sourceRootPath = getRootPath(getConf());

    for (Path sourcePath : options.getSourcePaths()) {

      FileSystem fileSystem = sourcePath.getFileSystem(getConf());
      FileStatus directory = fileSystem.getFileStatus(sourcePath);

      Map<String, CopyListingFileStatus> children = new FileStatusTreeTraverser(fileSystem)
          .preOrderTraversal(directory)
          .transform(new CopyListingFileStatusFunction(fileSystem, options))
          .uniqueIndex(new RelativePathFunction(sourceRootPath));

      for (Entry<String, CopyListingFileStatus> entry : children.entrySet()) {
        LOG.debug("Adding '{}' with relative path '{}'", entry.getValue().getPath(), entry.getKey());
        writer.append(new Text(entry.getKey()), entry.getValue());
        writer.sync();
      }
    }
  }
}
开发者ID:HotelsDotCom，项目名称:circus-train，代码行数:25，代码来源:CircusTrainCopyListing.java

示例2: distCopy

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
public void distCopy(Path sourceFile, Path destFile) {
    try {
        Configuration conf = new Configuration();
        DistCp distCp = new DistCp(conf, new DistCpOptions(sourceFile, destFile));
        distCp.run(new String[] { sourceFile.toString(), destFile.toString() });
    } catch (Exception e) {
        log.error("Couldn't execute distCp from {} to {}", sourceFile.toString(), destFile.toString());
        e.printStackTrace();
    }
}
开发者ID:redBorder，项目名称:camus-sync，代码行数:11，代码来源:HdfsServer.java

示例3: assertDefaultValues

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
private void assertDefaultValues(DistCpOptions distCpOptions) {
  assertThat(distCpOptions, is(not(nullValue())));
  assertThat(distCpOptions.preserveAttributes().hasNext(), is(false));
  assertThat(distCpOptions.shouldPreserveRawXattrs(), is(false));
  assertThat(distCpOptions.shouldAppend(), is(false));
  assertThat(distCpOptions.shouldAtomicCommit(), is(false));
  assertThat(distCpOptions.getAtomicWorkPath(), is(nullValue()));
  assertThat(distCpOptions.shouldBlock(), is(true));
  assertThat(distCpOptions.getCopyStrategy(), is(DistCpConstants.UNIFORMSIZE));
  assertThat(distCpOptions.shouldDeleteMissing(), is(false));
  assertThat(distCpOptions.shouldIgnoreFailures(), is(false));
  assertThat(distCpOptions.getLogPath(), is(nullValue()));
  assertThat(distCpOptions.getMapBandwidth(), is(DistCpConstants.DEFAULT_BANDWIDTH_MB));
  assertThat(distCpOptions.getMaxMaps(), is(DistCpConstants.DEFAULT_MAPS));
  assertThat(distCpOptions.shouldOverwrite(), is(false));
  assertThat(distCpOptions.shouldSkipCRC(), is(false));
  assertThat(distCpOptions.getSslConfigurationFile(), is(nullValue()));
  assertThat(distCpOptions.shouldSyncFolder(), is(false));
  assertThat(distCpOptions.getTargetPathExists(), is(true));
}
开发者ID:HotelsDotCom，项目名称:circus-train，代码行数:21，代码来源:DistCpOptionsParserTest.java

示例4: copyFileWithRetry

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}
开发者ID:naver，项目名称:hadoop，代码行数:18，代码来源:CopyMapper.java

示例5: copyFileWithRetry

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
  totalBytesCopied += bytesCopied;
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:19，代码来源:CopyMapper.java

示例6: copyFileWithRetry

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
private void copyFileWithRetry(String description, FileStatus sourceFileStatus,
             Path target, Context context,
             EnumSet<DistCpOptions.FileAttribute> fileAttributes) throws IOException {

  long bytesCopied;
  try {
    bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description)
                     .execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:18，代码来源:CopyMapper.java

示例7: getDistCp

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Override
protected DistCp getDistCp(Configuration conf, DistCpOptions options) {
  assertEquals(Constants.MAPRED_MAP_MAX_ATTEMPTS,
      Integer.parseInt(conf.get("mapred.map.max.attempts")));
  assertEquals(Constants.CHECKSUM_BYTES,
      Integer.parseInt(conf.get("io.bytes.per.checksum")));
  long blockSizeExpected = blockSize;
  if (blockSizeExpected % Constants.CHECKSUM_BYTES != 0) {
    blockSizeExpected = (blockSize / Constants.CHECKSUM_BYTES + 1) * Constants.CHECKSUM_BYTES;
  }
  assertEquals(blockSizeExpected, Long.parseLong(conf.get("dfs.block.size")));
  assertEquals(REPLICA_FACTOR, Integer.parseInt(conf.get("dfs.replication")));
  assertEquals(sourceFiles, options.getSourcePaths());
  assertTrue(options.shouldSkipCRC());
  assertTrue(options.shouldSyncFolder());
  assertTrue(options.getTargetPath().toString().startsWith("hdfs://" + NAME_NODE + HDFS_DIR));
  if (numPartition == 1) {
    assertTrue(options.getTargetPath().toString()
        .endsWith(TerrapinUtil.formatPartitionName(0)));
  }
  return distCp;
}
开发者ID:pinterest-attic，项目名称:terrapin，代码行数:23，代码来源:BaseUploaderTest.java

示例8: getSourcePaths

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private List<Path> getSourcePaths(Field fieldInputOptions) throws IOException{
  Object options;
  try {
    options = fieldInputOptions.get(this);
    if (options instanceof DistCpOptions) {
      return ((DistCpOptions) options).getSourcePaths();
    } else {
      // Hadoop 3
      Class<?> classContext = Class.forName("org.apache.hadoop.tools.DistCpContext");
      Method methodGetSourcePaths = classContext.getDeclaredMethod("getSourcePaths");
      methodGetSourcePaths.setAccessible(true);

      return (List<Path>) methodGetSourcePaths.invoke(options);
    }
  } catch (IllegalArgumentException | IllegalAccessException |
            ClassNotFoundException | NoSuchMethodException |
            SecurityException | InvocationTargetException e) {
    throw new IOException(e);
  }

}
开发者ID:apache，项目名称:hbase，代码行数:23，代码来源:MapReduceBackupCopyJob.java

示例9: testRunStarPattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunStarPattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("ivory.include.path", "*/3/*");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 3);
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:17，代码来源:FilteredCopyListingTest.java

示例10: testRunQuestionPattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunQuestionPattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("ivory.include.path", "*/3/?");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 2);
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:17，代码来源:FilteredCopyListingTest.java

示例11: testRunRangePattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunRangePattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("ivory.include.path", "*/3/[47]");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 2);
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:17，代码来源:FilteredCopyListingTest.java

示例12: testRunSpecificPattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunSpecificPattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("ivory.include.path", "*/3/40");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 1);
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:17，代码来源:FilteredCopyListingTest.java

示例13: testRunListPattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunListPattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("ivory.include.path", "*/3/{4,7}");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 2);
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:17，代码来源:FilteredCopyListingTest.java

示例14: testArguments

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testArguments() throws Exception {
    /*
     * <arg>-update</arg>
     * <arg>-blocking</arg><arg>true</arg> <arg>-maxMaps</arg><arg>20</arg>
     * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg>
     * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg>
     */
    FeedReplicator replicator = new FeedReplicator();
    DistCpOptions options = replicator.getDistCpOptions(new String[] { "-update",
            "-blocking", "true", "-maxMaps", "5", "-sourcePaths",
            "hdfs://localhost:8020/tmp/", "-targetPath",
            "hdfs://localhost1:8020/tmp/" });

    List<Path> srcPaths = new ArrayList<Path>();
    srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
    Assert.assertEquals(options.getSourcePaths(), srcPaths);
    Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
}
开发者ID:sriksun，项目名称:ivry-security，代码行数:20，代码来源:FeedReplicatorTest.java

示例15: testRunStarPattern

import org.apache.hadoop.tools.DistCpOptions; //导入依赖的package包/类
@Test
public void testRunStarPattern() throws Exception {
    final URI uri = FileSystem.getLocal(new Configuration()).getUri();
    final String pathString = uri.toString();
    Path fileSystemPath = new Path(pathString);
    Path source = new Path(fileSystemPath.toString() + "///tmp/source");
    Path target = new Path(fileSystemPath.toString() + "///tmp/target");
    Path listingPath = new Path(fileSystemPath.toString() + "///tmp/META/fileList.seq");
    DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);

    Configuration configuration = new Configuration();
    configuration.set("falcon.include.path", "*/3/*");
    new FilteredCopyListing(configuration, CREDENTIALS).buildListing(listingPath, options);

    verifyContents(listingPath, 3);
}
开发者ID:shaikidris，项目名称:incubator-falcon，代码行数:17，代码来源:FilteredCopyListingTest.java


注：本文中的org.apache.hadoop.tools.DistCpOptions类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。