Java DatasetReader.next方法代码示例

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testOverrideTypeMapping() throws IOException {
  String [] types = { "INT" };
  String [] vals = { "10" };
  createTableWithColTypes(types, vals);

  String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
  runImport(getOutputArgv(true, extraArgs));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "DATA_COL0", Type.STRING);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testNonIdentCharactersInColumnName() throws IOException {
  String [] names = { "test_p-a+r/quet" };
  String [] types = { "INT" };
  String [] vals = { "2015" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "TEST_P_A_R_QUET", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("TEST_P_A_R_QUET", 2015, record1.get("TEST_P_A_R_QUET"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testNullableParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { null };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertNull(record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testQueryImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputQueryArgv(true, null));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testIncrementalParquetImport() throws IOException, SQLException {
  String [] types = { "INT" };
  String [] vals = { "1" };
  createTableWithColTypes(types, vals);

  runImport(getOutputArgv(true, null));
  runImport(getOutputArgv(true, new String[]{"--append"}));

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    record1 = reader.next();
    assertEquals(1, record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Get the most recent WebPageSnapshotModel from HBase
 * 
 * @param url
 *          The URL to get the snapshotted page from HBase
 * @return The WebPageSnapshotModel, or null if there are no fetches for this
 *         URL
 */
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
  url = normalizeUrl(url);
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    // we don't know the exact timestamp in the key, but we know since keys
    // are in timestamp descending order that the first row for an URL will be
    // the most recent.
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    if (reader.hasNext()) {
      return reader.next();
    } else {
      return null;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testMultiTableImportAsParquetFormat() throws IOException {
  String [] argv = getArgv(new String[]{"--as-parquetfile"}, null);
  runImport(new ImportAllTablesTool(), argv);

  Path warehousePath = new Path(this.getWarehouseDir());
  int i = 0;
  for (String tableName : this.tableNames) {
    Path tablePath = new Path(warehousePath, tableName);
    Dataset dataset = Datasets.load("dataset:file:" + tablePath);

    // dequeue the expected value for this table. This
    // list has the same order as the tableNames list.
    String expectedVal = Integer.toString(i++) + ","
        + this.expectedStrings.get(0);
    this.expectedStrings.remove(0);

    DatasetReader<GenericRecord> reader = dataset.newReader();
    try {
      GenericRecord record = reader.next();
      String line = record.get(0) + "," + record.get(1);
      assertEquals("Table " + tableName + " expected a different string",
          expectedVal, line);
      assertFalse(reader.hasNext());
    } finally {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
@Test
public void testJobConfiguration() {

  Map<String,String> settings = ImmutableMap.of("test.app.level.setting", "appvalue",
      "kite.job.write-config-job.test.job.level.setting", "jobvalue",
      "kite.job.write-config-job.output.kv-output.test.output.level.setting", "outputvalue");

  AppContext context = new AppContext(settings, getConfiguration());

  TestScheduler scheduler = TestScheduler.load(WriteConfigOutputApp.class, context);

  Instant nominalTime = new DateTime(2015, 5, 15, 12, 0, 0, 0, DateTimeZone.UTC).toInstant();

  scheduler.runScheduledJobs(nominalTime);

  Dataset<KeyValues> ds = Datasets.load(WriteConfigOutputApp.OUTPUT_DATASET, KeyValues.class);

  DatasetReader<KeyValues> reader = ds.newReader();

  try {

    KeyValues kv = reader.next();

    Assert.assertEquals(ImmutableMap.of(
            "test.app.level.setting", "appvalue",
            "test.job.level.setting", "jobvalue",
            "output.kv-output.test.output.level.setting", "outputvalue"),
        kv.getJobsettings());

    Assert.assertEquals(ImmutableMap.of(
            "test.app.level.setting", "appvalue",
            "test.job.level.setting", "jobvalue",
            "test.output.level.setting", "outputvalue"),
        kv.getOutputsettings());

  } finally {
    Closeables.closeQuietly(reader);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
private void runParquetImportTest(String codec) throws IOException {
  String[] types = {"BIT", "INTEGER", "BIGINT", "REAL", "DOUBLE", "VARCHAR(6)",
      "VARBINARY(2)",};
  String[] vals = {"true", "100", "200", "1.0", "2.0", "'s'", "'0102'", };
  createTableWithColTypes(types, vals);

  String [] extraArgs = { "--compression-codec", codec};
  runImport(getOutputArgv(true, extraArgs));

  assertEquals(CompressionType.forName(codec), getCompressionType());

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "DATA_COL0", Type.BOOLEAN);
  checkField(fields.get(1), "DATA_COL1", Type.INT);
  checkField(fields.get(2), "DATA_COL2", Type.LONG);
  checkField(fields.get(3), "DATA_COL3", Type.FLOAT);
  checkField(fields.get(4), "DATA_COL4", Type.DOUBLE);
  checkField(fields.get(5), "DATA_COL5", Type.STRING);
  checkField(fields.get(6), "DATA_COL6", Type.BYTES);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    GenericRecord record1 = reader.next();
    assertNotNull(record1);
    assertEquals("DATA_COL0", true, record1.get("DATA_COL0"));
    assertEquals("DATA_COL1", 100, record1.get("DATA_COL1"));
    assertEquals("DATA_COL2", 200L, record1.get("DATA_COL2"));
    assertEquals("DATA_COL3", 1.0f, record1.get("DATA_COL3"));
    assertEquals("DATA_COL4", 2.0, record1.get("DATA_COL4"));
    assertEquals("DATA_COL5", "s", record1.get("DATA_COL5"));
    Object object = record1.get("DATA_COL6");
    assertTrue(object instanceof ByteBuffer);
    ByteBuffer b = ((ByteBuffer) object);
    assertEquals((byte) 1, b.get(0));
    assertEquals((byte) 2, b.get(1));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Loads the expected number of items when they are available in the dataset.
 */
private static List<ExampleEvent> loadWhenAvailable(View<ExampleEvent> view,
                                                    int expected,
                                                    int attempts) throws InterruptedException {

  for (int attempt = 0; attempt < attempts; ++attempt) {

    List<ExampleEvent> items = Lists.newArrayList();

    DatasetReader<ExampleEvent> reader = view.newReader();
    int count = 0;

    try {

      while (count < expected && reader.hasNext()) {

        ExampleEvent event = reader.next();

        items.add(event);

        count++;
      }

    } finally {
      reader.close();
    }

    if (count == expected) {
      return items;
    }

    Thread.sleep(1000);

  }

  return null;
}