Java FileSystem.closeAll方法代码示例

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
      "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  JobConf job = new JobConf(conf);
  TextInputFormat fileInputFormat = new TextInputFormat();
  fileInputFormat.configure(job);
  InputSplit[] splits = fileInputFormat.getSplits(job, 1);
  Assert.assertEquals("Input splits are not correct", 2, splits.length);
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testListLocatedStatus() throws Exception {
  Configuration conf = getConfiguration();
  conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
  conf.setBoolean("fs.test.impl.disable.cache", false);
  conf.set(FileInputFormat.INPUT_DIR, "test:///a1/a2");
  MockFileSystem mockFs =
      (MockFileSystem) new Path("test:///").getFileSystem(conf);
  Assert.assertEquals("listLocatedStatus already called",
      0, mockFs.numListLocatedStatusCalls);
  Job job = Job.getInstance(conf);
  FileInputFormat<?, ?> fileInputFormat = new TextInputFormat();
  List<InputSplit> splits = fileInputFormat.getSplits(job);
  Assert.assertEquals("Input splits are not correct", 2, splits.size());
  Assert.assertEquals("listLocatedStatuss calls",
      1, mockFs.numListLocatedStatusCalls);
  FileSystem.closeAll();
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@Test
public void testFileSystemCloseAll() throws Exception {
  Configuration conf = getTestConfiguration();
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
  URI address = FileSystem.getDefaultUri(conf);

  try {
    FileSystem.closeAll();

    conf = getTestConfiguration();
    FileSystem.setDefaultUri(conf, address);
    FileSystem.get(conf);
    FileSystem.get(conf);
    FileSystem.closeAll();
  }
  finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@Override
@Before
public void setUp() throws IOException {
  super.setUp();
  // Without this closeAll() call, setting of FileSystem block size is
  // not effective and will be old block size set in earlier test.
  FileSystem.closeAll();
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@Before
public void closeFilesystems() throws IOException {
  // prevent the same mockfs instance from being reused due to FS cache
  FileSystem.closeAll();
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
private void testConcurrentCommitTaskWithSubDir(int version)
    throws Exception {
  final Job job = Job.getInstance();
  FileOutputFormat.setOutputPath(job, outDir);
  final Configuration conf = job.getConfiguration();
  conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
  conf.setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
      version);

  conf.setClass("fs.file.impl", RLFS.class, FileSystem.class);
  FileSystem.closeAll();

  final JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
  final FileOutputCommitter amCommitter =
      new FileOutputCommitter(outDir, jContext);
  amCommitter.setupJob(jContext);

  final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
  taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
  taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);

  final TextOutputFormat[] tof = new TextOutputFormat[2];
  for (int i = 0; i < tof.length; i++) {
    tof[i] = new TextOutputFormat() {
      @Override
      public Path getDefaultWorkFile(TaskAttemptContext context,
          String extension) throws IOException {
        final FileOutputCommitter foc = (FileOutputCommitter)
            getOutputCommitter(context);
        return new Path(new Path(foc.getWorkPath(), SUB_DIR),
            getUniqueFile(context, getOutputName(context), extension));
      }
    };
  }

  final ExecutorService executor = Executors.newFixedThreadPool(2);
  try {
    for (int i = 0; i < taCtx.length; i++) {
      final int taskIdx = i;
      executor.submit(new Callable<Void>() {
        @Override
        public Void call() throws IOException, InterruptedException {
          final OutputCommitter outputCommitter =
              tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
          outputCommitter.setupTask(taCtx[taskIdx]);
          final RecordWriter rw =
              tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
          writeOutput(rw, taCtx[taskIdx]);
          outputCommitter.commitTask(taCtx[taskIdx]);
          return null;
        }
      });
    }
  } finally {
    executor.shutdown();
    while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
      LOG.info("Awaiting thread termination!");
    }
  }

  amCommitter.commitJob(jContext);
  final RawLocalFileSystem lfs = new RawLocalFileSystem();
  lfs.setConf(conf);
  assertFalse("Must not end up with sub_dir/sub_dir",
      lfs.exists(new Path(OUT_SUB_DIR, SUB_DIR)));

  // validate output
  validateContent(OUT_SUB_DIR);
  FileUtil.fullyDelete(new File(outDir.toString()));
}
 

import org.apache.hadoop.fs.FileSystem; //导入方法依赖的package包/类
@After
public void resetFileSystems() throws IOException {
  FileSystem.closeAll();
}