Java TestFileCreation.createFile方法代码示例

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
@Test
public void testBlockCreation() throws IOException {
  Path file1 = new Path(BASE_DIR, "file1.dat");
  FSDataOutputStream out = TestFileCreation.createFile(hdfs, file1, 3);

  for(int idx = 0; idx < NUM_BLOCKS; idx++) {
    // write one block
    writeFile(file1, out, BLOCK_SIZE);
    // verify consistency
    verifyFileBlocks(file1.toString(), true);
  }

  // close file
  out.close();
  // verify consistency
  verifyFileBlocks(file1.toString(), false);
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
@Test
public void testDeleteUnclosed() throws IOException, InterruptedException {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRIES_ON_FAILURE_KEY, 0);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    assertTrue(fs.mkdir(new Path("/foo"), FsPermission.getDefault()));
    TestFileCreation.createFile(fs, new Path("/foo/bar"), 1);
    assertTrue(fs.delete(new Path("/foo/bar"), true));
    assertFalse(fs.exists(new Path("/foo/bar")));
    assertFalse("Not All subtree locks were removed after operation ", subTreeLocksExists());
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
@Test
public void testBlockCreation() throws IOException {
  Path file1 = new Path(BASE_DIR, "file1.dat");
  FSDataOutputStream out = TestFileCreation.createFile(hdfs, file1, 3);

  for (int idx = 0; idx < NUM_BLOCKS; idx++) {
    // write one block
    writeFile(file1, out, BLOCK_SIZE);
    // verify consistency
    verifyFileBlocks(file1.toString(), true);
  }

  // close file
  out.close();
  // verify consistency
  verifyFileBlocks(file1.toString(), false);
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
/**
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test
public void testGetBlockLocations() throws IOException {
  final NamenodeProtocols namenode = cluster.getNameNodeRpc();
  final Path p = new Path(BASE_DIR, "file2.dat");
  final String src = p.toString();
  final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3);

  // write a half block
  int len = BLOCK_SIZE >>> 1;
  writeFile(p, out, len);

  for(int i = 1; i < NUM_BLOCKS; ) {
    // verify consistency
    final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len);
    final List<LocatedBlock> blocks = lb.getLocatedBlocks();
    assertEquals(i, blocks.size());
    final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
    assertTrue(b instanceof BlockInfoContiguousUnderConstruction);

    if (++i < NUM_BLOCKS) {
      // write one more block
      writeFile(p, out, BLOCK_SIZE);
      len += BLOCK_SIZE;
    }
  }
  // close file
  out.close();
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
/**
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test
public void testGetBlockLocations() throws IOException {
  final NamenodeProtocols namenode = cluster.getNameNodeRpc();
  final BlockManager blockManager = cluster.getNamesystem().getBlockManager();
  final Path p = new Path(BASE_DIR, "file2.dat");
  final String src = p.toString();
  final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3);

  // write a half block
  int len = BLOCK_SIZE >>> 1;
  writeFile(p, out, len);

  for(int i = 1; i < NUM_BLOCKS; ) {
    // verify consistency
    final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len);
    final List<LocatedBlock> blocks = lb.getLocatedBlocks();
    assertEquals(i, blocks.size());
    final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
    assertFalse(blockManager.getStoredBlock(b).isComplete());

    if (++i < NUM_BLOCKS) {
      // write one more block
      writeFile(p, out, BLOCK_SIZE);
      len += BLOCK_SIZE;
    }
  }
  // close file
  out.close();
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
/**
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test
public void testGetBlockLocations() throws IOException {
  final NamenodeProtocols namenode = cluster.getNameNodeRpc();
  final Path p = new Path(BASE_DIR, "file2.dat");
  final String src = p.toString();
  final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3);

  // write a half block
  int len = BLOCK_SIZE >>> 1;
  writeFile(p, out, len);

  for(int i = 1; i < NUM_BLOCKS; ) {
    // verify consistency
    final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len);
    final List<LocatedBlock> blocks = lb.getLocatedBlocks();
    assertEquals(i, blocks.size());
    final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
    assertTrue(b instanceof BlockInfoUnderConstruction);

    if (++i < NUM_BLOCKS) {
      // write one more block
      writeFile(p, out, BLOCK_SIZE);
      len += BLOCK_SIZE;
    }
  }
  // close file
  out.close();
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
/**
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test
public void testGetBlockLocations() throws IOException {
  final NamenodeProtocols namenode = cluster.getNameNodeRpc();
  final Path p = new Path(BASE_DIR, "file2.dat");
  final String src = p.toString();
  final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3);

  // write a half block
  int len = BLOCK_SIZE >>> 1;
  writeFile(p, out, len);

  for (int i = 1; i < NUM_BLOCKS; ) {
    // verify consistency
    final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len);
    final List<LocatedBlock> blocks = lb.getLocatedBlocks();
    assertEquals(i, blocks.size());
    final Block b = blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
    assertTrue(b instanceof BlockInfoUnderConstruction);

    if (++i < NUM_BLOCKS) {
      // write one more block
      writeFile(p, out, BLOCK_SIZE);
      len += BLOCK_SIZE;
    }
  }
  // close file
  out.close();
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
@Test
public void testBasicPutGet()
    throws IOException, URISyntaxException,
    ServiceException, NoSuchAlgorithmException {
  S3HdfsPath s3HdfsPath = testUtil.setUpS3HdfsPath("rewrite", "readme.txt");

  // Create file and blank metadata in HDFS (but not s3)
  Path path = new Path(s3HdfsPath.getFullHdfsObjPath());
  FSDataOutputStream out =
      TestFileCreation.createFile(hdfs, path, 3);
  TestFileCreation.writeFile(out, 128);
  out.close();

  Path pathMeta = new Path(s3HdfsPath.getFullHdfsMetaPath());
  FSDataOutputStream outMeta = hdfs.create(pathMeta);
  outMeta.close();

  // Get the object
  S3Bucket bucket = new S3Bucket(s3HdfsPath.getBucketName());
  String objectKey = s3HdfsPath.getObjectName();

  S3Object returnedObject1 = s3Service.getObject(bucket.getName(), objectKey);
  System.out.println("RETURNED_OBJECT_1");
  System.out.println(returnedObject1); // returned has dataInputStream!

  // Verify the object
  assertEquals(bucket.getName(), returnedObject1.getBucketName());
  assertEquals(objectKey, returnedObject1.getKey());

  // verify returned data
  testUtil.compareS3ObjectWithHdfsFile(returnedObject1.getDataInputStream(),
      path);

  // List objects
  S3Object[] ls = s3Service.listObjects(bucket.getName());
  assertEquals("Should be one object", 1, ls.length);
  System.out.println("LISTED_OBJECTS_1");
  System.out.println(ls[0]);
}
 

import org.apache.hadoop.hdfs.TestFileCreation; //导入方法依赖的package包/类
/**
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test
public void testGetBlockLocations() throws IOException {
  final NameNode namenode = cluster.getNameNode();
  final Path p = new Path(BASE_DIR, "file2.dat");
  final String src = p.toString();
  final FSDataOutputStream out = TestFileCreation.createFile(hdfs, p, 3);

  // write a half block
  int len = BLOCK_SIZE >>> 1;
  writeFile(p, out, len);

  for(int i = 1; i < NUM_BLOCKS; ) {
    // verify consistency
    final LocatedBlocks lb = namenode.getBlockLocations(src, 0, len);
    final List<LocatedBlock> blocks = lb.getLocatedBlocks();
    assertEquals(i, blocks.size());
    final Block b = blocks.get(blocks.size() - 1).getBlock();
    assertTrue(b instanceof BlockInfoUnderConstruction);

    if (++i < NUM_BLOCKS) {
      // write one more block
      writeFile(p, out, BLOCK_SIZE);
      len += BLOCK_SIZE;
    }
  }
  // close file
  out.close();
}