Java DatasetReader类代码示例

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testOverrideTypeMapping() throws IOException {
  String [] types = { "INT" };
  String [] vals = { "10" };
  createTableWithColTypes(types, vals);

  String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
  runImport(getOutputArgv(true, extraArgs));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "DATA_COL0", Type.STRING);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testNonIdentCharactersInColumnName() throws IOException {
  String [] names = { "test_p-a+r/quet" };
  String [] types = { "INT" };
  String [] vals = { "2015" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "TEST_P_A_R_QUET", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("TEST_P_A_R_QUET", 2015, record1.get("TEST_P_A_R_QUET"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testNullableParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { null };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertNull(record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testQueryImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputQueryArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void testIncrementalParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));
  runImport(getOutputArgv(true, new String[]{"--append"}));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
                @DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {

  DatasetReader<StandardEvent> reader = input.newReader();
  DatasetWriter<StandardEvent> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
                @DataOut(name="target_users") View<GenericRecord> output) {

  DatasetReader<GenericRecord> reader = input.newReader();
  DatasetWriter<GenericRecord> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get the epoch timestamps for every snapshot time of an URL in HBase.
 * 
 * @param url
 *          The URL of the page to get snapshot timestamps for
 * @return The list of timestamps
 */
public List<Long> getSnapshotTimestamps(String url) {
  url = normalizeUrl(url);
  List<Long> snapshotTimestamps = new ArrayList<Long>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    while (reader.hasNext()) {
      snapshotTimestamps.add(reader.next().getFetchedAt());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return snapshotTimestamps;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get the most recent WebPageSnapshotModel from HBase
 * 
 * @param url
 *          The URL to get the snapshotted page from HBase
 * @return The WebPageSnapshotModel, or null if there are no fetches for this
 *         URL
 */
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
  url = normalizeUrl(url);
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    // we don't know the exact timestamp in the key, but we know since keys
    // are in timestamp descending order that the first row for an URL will be
    // the most recent.
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    if (reader.hasNext()) {
      return reader.next();
    } else {
      return null;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
/**
 * Get WebPageSnapshotModels for an URL from HBase since the since param.
 * 
 * @param url
 *          The URL of the page to fetch
 * @param since
 *          The models to fetch since
 * @return The list of models that have been fetched for an URL since the
 *         since param.
 */
private List<WebPageSnapshotModel> getWebPageSnapshotsSince(String url,
    long since) {
  url = normalizeUrl(url);
  List<WebPageSnapshotModel> models = new ArrayList<WebPageSnapshotModel>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", since).newReader();
    while (reader.hasNext()) {
      models.add(reader.next());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return models;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hdfs:/tmp/data/users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.newReader();
    for (GenericRecord user : reader) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hive?dataset=users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.newReader();
    for (GenericRecord user : users.newReader()) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the users dataset
  Dataset<Record> users = Datasets.load(
      "dataset:hdfs:/tmp/data/users", Record.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Record> reader = null;
  try {
    reader = users.with("favoriteColor", "green").newReader();
    for (GenericRecord user : reader) {
      System.out.println(user);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}
 

import org.kitesdk.data.DatasetReader; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Load the products dataset
  Dataset<Product> products = Datasets.load(
      "dataset:hdfs:/tmp/data/products", Product.class);

  // Get a reader for the dataset and read all the users
  DatasetReader<Product> reader = null;
  try {
    reader = products.newReader();
    for (Product product : reader) {
      System.out.println(product);
    }

  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  return 0;
}