Java BaseSqoopTestCase类代码示例

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
protected void createExportFile(ColumnGenerator... extraCols)
  throws IOException {
  String ext = ".txt";

  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part0" + ext);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);

  BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
  for (int i = 0; i < 3; i++) {
    String line = getRecordLine(i, extraCols);
    w.write(line);
    LOG.debug("Create Export file - Writing line : " + line);
  }
  w.close();
  os.close();
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
private void importData(String targetDir, SqoopOptions.FileLayout fileLayout) {
  SqoopOptions options;
  options = getSqoopOptions(newConf());
  options.setTableName(TABLE_NAME);
  options.setNumMappers(1);
  options.setFileLayout(fileLayout);
  options.setDeleteMode(true);

  Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
  options.setTargetDir(new Path(warehouse, targetDir).toString());

  ImportTool importTool = new ImportTool();
  Sqoop importer = new Sqoop(importTool, options.getConf(), options);
  int ret = Sqoop.runSqoop(importer, new String[0]);
  if (0 != ret) {
    fail("Initial import failed with exit code " + ret);
  }
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
protected void createExportFile(ColumnGenerator...extraCols)
  throws IOException, SQLException {
  String ext = ".txt";

  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part0" + ext);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);


  BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
  for (int i = 0; i < 3; i++) {
   String line = getRecordLine(i, extraCols);
   w.write(line);
   LOG.debug("Create Export file - Writing line : " + line);
  }
  w.close();
  os.close();
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
public Configuration newConf() {
  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  conf.set("mapred.job.tracker", "local");
  return conf;
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Delete all files in a directory for a table.
 */
public void clearDir(String tableName) {
  try {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    Path warehouse = new Path(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
    Path tableDir = new Path(warehouse, tableName);
    fs.delete(tableDir, true);
  } catch (Exception e) {
    fail("Got unexpected exception: " + StringUtils.stringifyException(e));
  }
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Return a list of arguments to import the specified table.
 */
private List<String> getArgListForTable(String tableName, boolean commonArgs,
    boolean isAppend, boolean appendTimestamp) {
  List<String> args = new ArrayList<String>();
  if (commonArgs) {
    CommonArgs.addHadoopFlags(args);
  }
  args.add("--connect");
  args.add(SOURCE_DB_URL);
  args.add("--table");
  args.add(tableName);
  args.add("--warehouse-dir");
  args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
  if (isAppend) {
    args.add("--incremental");
    args.add("append");
    if (!appendTimestamp) {
      args.add("--check-column");
      args.add("ID");
    } else {
      args.add("--check-column");
      args.add("LAST_MODIFIED");
    }
  } else {
    args.add("--incremental");
    args.add("lastmodified");
    args.add("--check-column");
    args.add("LAST_MODIFIED");
  }
  args.add("--columns");
  args.add("ID");
  args.add("-m");
  args.add("1");

  return args;
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Return list of arguments to import by query.
 * @return
 */
private List<String> getArgListForQuery(String query, String directoryName,
  boolean commonArgs, boolean isAppend, boolean appendTimestamp) {
  List<String> args = new ArrayList<String>();
  if (commonArgs) {
    CommonArgs.addHadoopFlags(args);
  }
  args.add("--connect");
  args.add(SOURCE_DB_URL);
  args.add("--query");
  args.add(query);
  args.add("--class-name");
  args.add(directoryName);
  args.add("--target-dir");
  args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR
    + System.getProperty("file.separator") + directoryName);
  if (isAppend) {
    args.add("--incremental");
    args.add("append");
    if (!appendTimestamp) {
      args.add("--check-column");
      args.add("ID");
    } else {
      args.add("--check-column");
      args.add("LAST_MODIFIED");
    }
  } else {
    args.add("--incremental");
    args.add("lastmodified");
    args.add("--check-column");
    args.add("LAST_MODIFIED");
  }
  args.add("-m");
  args.add("1");

  return args;
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
private DataFileReader<GenericRecord> read(Path filename) throws IOException {
  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FsInput fsInput = new FsInput(filename, conf);
  DatumReader<GenericRecord> datumReader =
    new GenericDatumReader<GenericRecord>();
  return new DataFileReader<GenericRecord>(fsInput, datumReader);
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Create a data file that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export)
 * @param numRecords how many records to write to the file.
 * @param gzip is true if the file should be gzipped.
 */
protected void createTextFile(int fileNum, int numRecords, boolean gzip,
    ColumnGenerator... extraCols) throws IOException {
  int startId = fileNum * numRecords;

  String ext = ".txt";
  if (gzip) {
    ext = ext + ".gz";
  }
  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part" + fileNum + ext);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);
  if (gzip) {
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(filePath);
    os = codec.createOutputStream(os);
  }
  BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
  for (int i = 0; i < numRecords; i++) {
    w.write(getRecordLine(startId + i, extraCols));
  }
  w.close();
  os.close();

  if (gzip) {
    verifyCompressedFile(filePath, numRecords);
  }
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
private void verifyCompressedFile(Path f, int expectedNumLines)
    throws IOException {
  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  InputStream is = fs.open(f);
  CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
  CompressionCodec codec = ccf.getCodec(f);
  LOG.info("gzip check codec is " + codec);
  Decompressor decompressor = CodecPool.getDecompressor(codec);
  if (null == decompressor) {
    LOG.info("Verifying gzip sanity with null decompressor");
  } else {
    LOG.info("Verifying gzip sanity with decompressor: "
        + decompressor.toString());
  }
  is = codec.createInputStream(is, decompressor);
  BufferedReader r = new BufferedReader(new InputStreamReader(is));
  int numLines = 0;
  while (true) {
    String ln = r.readLine();
    if (ln == null) {
      break;
    }
    numLines++;
  }

  r.close();
  assertEquals("Did not read back correct number of lines",
      expectedNumLines, numLines);
  LOG.info("gzip sanity check returned " + numLines + " lines; ok.");
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Create a data file in SequenceFile format that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export).
 * @param numRecords how many records to write to the file.
 * @param className the table class name to instantiate and populate
 *          for each record.
 */
private void createSequenceFile(int fileNum, int numRecords, String className)
    throws IOException {

  try {
    // Instantiate the value record object via reflection.
    Class cls = Class.forName(className, true,
        Thread.currentThread().getContextClassLoader());
    SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(
        cls, new Configuration());

    // Create the SequenceFile.
    Configuration conf = new Configuration();
    if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
      conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
    }
    FileSystem fs = FileSystem.get(conf);
    Path tablePath = getTablePath();
    Path filePath = new Path(tablePath, "part" + fileNum);
    fs.mkdirs(tablePath);
    SequenceFile.Writer w = SequenceFile.createWriter(
        fs, conf, filePath, LongWritable.class, cls);

    // Now write the data.
    int startId = fileNum * numRecords;
    for (int i = 0; i < numRecords; i++) {
      record.parse(getRecordLine(startId + i));
      w.append(new LongWritable(startId + i), record);
    }

    w.close();
  } catch (ClassNotFoundException cnfe) {
    throw new IOException(cnfe);
  } catch (RecordParser.ParseError pe) {
    throw new IOException(pe);
  }
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/** Removing an existing table directory from the filesystem. */
private void removeTablePath() throws IOException {
  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.delete(getTablePath(), true);
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
public void setUp() throws IOException, InterruptedException {
  conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  String tmpDir = System.getProperty("test.build.data", "/tmp/");
  this.outDir = new Path(System.getProperty("java.io.tmpdir"));
  FileSystem fs = FileSystem.get(conf);
  if (fs.exists(outDir)) {
    fs.delete(outDir, true);
  }
  fs.mkdirs(outDir);

  loader = new LargeObjectLoader(conf, outDir);
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
/**
 * Create a data file that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export)
 * @param numRecords how many records to write to the file.
 */
protected void createAvroFile(int fileNum, int numRecords,
    ColumnGenerator... extraCols) throws IOException {

  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part" + fileNum);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);

  Schema schema = buildAvroSchema(extraCols);
  DatumWriter<GenericRecord> datumWriter =
    new GenericDatumWriter<GenericRecord>();
  DataFileWriter<GenericRecord> dataFileWriter =
    new DataFileWriter<GenericRecord>(datumWriter);
  dataFileWriter.create(schema, os);

  for (int i = 0; i < numRecords; i++) {
    GenericRecord record = new GenericData.Record(schema);
    record.put("id", i);
    record.put("msg", getMsgPrefix() + i);
    addExtraColumns(record, i, extraCols);
    dataFileWriter.append(record);
  }

  dataFileWriter.close();
  os.close();
}
 

import com.cloudera.sqoop.testutil.BaseSqoopTestCase; //导入依赖的package包/类
public void initUtils() throws IOException, MetaException {
  if (initialized) {
    return;
  }
  conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  fs = FileSystem.get(conf);
  fs.initialize(fs.getWorkingDirectory().toUri(), conf);
  storageInfo = null;
  SqoopHCatUtilities.setTestMode(true);
}