Java BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR属性代码示例

private void importData(String targetDir, SqoopOptions.FileLayout fileLayout) {
  SqoopOptions options;
  options = getSqoopOptions(newConf());
  options.setTableName(TABLE_NAME);
  options.setNumMappers(1);
  options.setFileLayout(fileLayout);
  options.setDeleteMode(true);

  Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
  options.setTargetDir(new Path(warehouse, targetDir).toString());

  ImportTool importTool = new ImportTool();
  Sqoop importer = new Sqoop(importTool, options.getConf(), options);
  int ret = Sqoop.runSqoop(importer, new String[0]);
  if (0 != ret) {
    fail("Initial import failed with exit code " + ret);
  }
}
 

/**
 * Delete all files in a directory for a table.
 */
public void clearDir(String tableName) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    fs.delete(tableDir, true);
  } catch (Exception e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

public void runMergeTest(SqoopOptions.FileLayout fileLayout) throws Exception {
  createTable(initRecords);

  // Create a jar to use for the merging process; we'll load it
  // into the current thread CL for when this runs. This needs
  // to contain a different class name than used for the imports
  // due to classloaderstack issues in the same JVM.
  final String MERGE_CLASS_NAME = "ClassForMerging";
  SqoopOptions options = getSqoopOptions(newConf());
  options.setTableName(TABLE_NAME);
  options.setClassName(MERGE_CLASS_NAME);

  CodeGenTool codeGen = new CodeGenTool();
  Sqoop codeGenerator = new Sqoop(codeGen, options.getConf(), options);
  int ret = Sqoop.runSqoop(codeGenerator, new String[0]);
  if (0 != ret) {
    fail("Nonzero exit from codegen: " + ret);
  }

  List<String> jars = codeGen.getGeneratedJarFiles();
  String jarFileName = jars.get(0);

  // Now do the imports.
  importData(OLD_PATH, fileLayout);

  // Check that we got records that meet our expected values.
  checkData(OLD_PATH, initRecords, fileLayout);

  Thread.sleep(25);

  // Modify the data in the warehouse.
  createTable(newRecords);

  Thread.sleep(25);

  // Do another import, into the "new" dir.
  importData(NEW_PATH, fileLayout);

  checkData(NEW_PATH, newRecords, fileLayout);

  // Now merge the results!
  ClassLoaderStack.addJarFile(jarFileName, MERGE_CLASS_NAME);
  Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
  options = getSqoopOptions(newConf());
  options.setMergeOldPath(new Path(warehouse, OLD_PATH).toString());
  options.setMergeNewPath(new Path(warehouse, NEW_PATH).toString());
  options.setMergeKeyCol("ID");
  options.setTargetDir(new Path(warehouse, FINAL_PATH).toString());
  options.setClassName(MERGE_CLASS_NAME);
  options.setExistingJarName(jarFileName);

  MergeTool mergeTool = new MergeTool();
  Sqoop merger = new Sqoop(mergeTool, options.getConf(), options);
  ret = Sqoop.runSqoop(merger, new String[0]);
  if (0 != ret) {
    fail("Merge failed with exit code " + ret);
  }

  checkData(FINAL_PATH, mergedRecords, fileLayout);
}
 

/**
 * Look at a directory that should contain files full of an imported 'id'
 * column. Assert that all numbers in [0, expectedNums) are present
 * in order.
 */
public void assertDirOfNumbers(String tableName, int expectedNums) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    FileStatus [] stats = fs.listStatus(tableDir);
    String [] fileNames = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
      fileNames[i] = stats[i].getPath().toString();
    }

    Arrays.sort(fileNames);

    // Read all the files in sorted order, adding the value lines to the list.
    List<String> receivedNums = new ArrayList<String>();
    for (String fileName : fileNames) {
      if (fileName.startsWith("_") || fileName.startsWith(".")) {
        continue;
      }

      BufferedReader r = new BufferedReader(
          new InputStreamReader(fs.open(new Path(fileName))));
      try {
        while (true) {
          String s = r.readLine();
          if (null == s) {
            break;
          }

          receivedNums.add(s.trim());
        }
      } finally {
        r.close();
      }
    }

    assertEquals(expectedNums, receivedNums.size());

    // Compare the received values with the expected set.
    for (int i = 0; i < expectedNums; i++) {
      assertEquals((int) i, (int) Integer.valueOf(receivedNums.get(i)));
    }
  } catch (Exception e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

/**
 * Look at a directory that should contain files full of an imported 'id'
 * column and 'last_modified' column. Assert that all numbers in [0, expectedNums) are present
 * in order.
 */
public void assertDirOfNumbersAndTimestamps(String tableName, int expectedNums) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    FileStatus [] stats = fs.listStatus(tableDir);
    String [] fileNames = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
      fileNames[i] = stats[i].getPath().toString();
    }

    Arrays.sort(fileNames);

    // Read all the files in sorted order, adding the value lines to the list.
    List<String> receivedNums = new ArrayList<String>();
    for (String fileName : fileNames) {
      if (fileName.startsWith("_") || fileName.startsWith(".")) {
        continue;
      }

      BufferedReader r = new BufferedReader(
          new InputStreamReader(fs.open(new Path(fileName))));
      try {
        while (true) {
          String s = r.readLine();
          if (null == s) {
            break;
          }

          receivedNums.add(s.trim());
        }
      } finally {
        r.close();
      }
    }

    assertEquals(expectedNums, receivedNums.size());

    // Compare the received values with the expected set.
    for (int i = 0; i < expectedNums; i++) {
      assertEquals((int) i, (int) Integer.valueOf(receivedNums.get(i).split(",")[0]));
    }
  } catch (Exception e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

/**
 * Assert that a directory contains a file with exactly one line
 * in it, containing the prescribed number 'val'.
 */
public void assertFirstSpecificNumber(String tableName, int val) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    FileStatus [] stats = fs.listStatus(tableDir);
    String [] filePaths = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
      filePaths[i] = stats[i].getPath().toString();
    }

    // Read the first file that is not a hidden file.
    boolean foundVal = false;
    for (String filePath : filePaths) {
      String fileName = new Path(filePath).getName();
      if (fileName.startsWith("_") || fileName.startsWith(".")) {
        continue;
      }

      if (foundVal) {
        // Make sure we don't have two or more "real" files in the dir.
        fail("Got an extra data-containing file in this directory.");
      }

      BufferedReader r = new BufferedReader(
          new InputStreamReader(fs.open(new Path(filePath))));
      try {
        String s = r.readLine();
        if (null == s) {
          fail("Unexpected empty file " + filePath + ".");
        }
        assertEquals(val, (int) Integer.valueOf(s.trim()));

        String nextLine = r.readLine();
        if (nextLine != null) {
          fail("Expected only one result, but got another line: " + nextLine);
        }

        // Successfully got the value we were looking for.
        foundVal = true;
      } finally {
        r.close();
      }
    }
  } catch (IOException e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

/**
 * Assert that a directory contains a file with exactly one line
 * in it, containing the prescribed number 'val'.
 */
public void assertSpecificNumber(String tableName, int val) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    FileStatus [] stats = fs.listStatus(tableDir);
    String [] filePaths = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
      filePaths[i] = stats[i].getPath().toString();
    }

    // Read the first file that is not a hidden file.
    boolean foundVal = false;
    for (String filePath : filePaths) {
      String fileName = new Path(filePath).getName();
      if (fileName.startsWith("_") || fileName.startsWith(".")) {
        continue;
      }

      if (foundVal) {
        // Make sure we don't have two or more "real" files in the dir.
        fail("Got an extra data-containing file in this directory.");
      }

      BufferedReader r = new BufferedReader(
          new InputStreamReader(fs.open(new Path(filePath))));
      try {
        String s = r.readLine();
        if (val == (int) Integer.valueOf(s.trim().split(",")[0])) {
          if (foundVal) {
            fail("Expected only one result, but got another line: " + s);
          }
          foundVal = true;
        }
      } finally {
        r.close();
      }
    }
  } catch (IOException e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

/**
 * Assert that a directory contains a file with exactly one line
 * in it, containing the prescribed number 'val'.
 */
public void assertSpecificNumber(String tableName, int val) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    FileStatus [] stats = fs.listStatus(tableDir);
    String [] filePaths = new String[stats.length];
    for (int i = 0; i < stats.length; i++) {
      filePaths[i] = stats[i].getPath().toString();
    }

    // Read the first file that is not a hidden file.
    boolean foundVal = false;
    for (String filePath : filePaths) {
      String fileName = new Path(filePath).getName();
      if (fileName.startsWith("_") || fileName.startsWith(".")) {
        continue;
      }

      if (foundVal) {
        // Make sure we don't have two or more "real" files in the dir.
        fail("Got an extra data-containing file in this directory.");
      }

      BufferedReader r = new BufferedReader(
          new InputStreamReader(fs.open(new Path(filePath))));
      try {
        String s = r.readLine();
        if (null == s) {
          fail("Unexpected empty file " + filePath + ".");
        }
        assertEquals(val, (int) Integer.valueOf(s.trim()));

        String nextLine = r.readLine();
        if (nextLine != null) {
          fail("Expected only one result, but got another line: " + nextLine);
        }

        // Successfully got the value we were looking for.
        foundVal = true;
      } finally {
        r.close();
      }
    }
  } catch (IOException e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}