本文整理汇总了Java中org.kitesdk.data.DatasetReader.next方法的典型用法代码示例。如果您正苦于以下问题:Java DatasetReader.next方法的具体用法?Java DatasetReader.next怎么用?Java DatasetReader.next使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.kitesdk.data.DatasetReader
的用法示例。
在下文中一共展示了DatasetReader.next方法的11个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testOverrideTypeMapping
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testOverrideTypeMapping() throws IOException {
String [] types = { "INT" };
String [] vals = { "10" };
createTableWithColTypes(types, vals);
String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
runImport(getOutputArgv(true, extraArgs));
Schema schema = getSchema();
assertEquals(Type.RECORD, schema.getType());
List<Field> fields = schema.getFields();
assertEquals(types.length, fields.size());
checkField(fields.get(0), "DATA_COL0", Type.STRING);
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例2: testFirstUnderscoreInColumnName
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testFirstUnderscoreInColumnName() throws IOException {
String [] names = { "_NAME" };
String [] types = { "INT" };
String [] vals = { "1987" };
createTableWithColTypesAndNames(names, types, vals);
runImport(getOutputArgv(true, null));
Schema schema = getSchema();
assertEquals(Type.RECORD, schema.getType());
List<Field> fields = schema.getFields();
assertEquals(types.length, fields.size());
checkField(fields.get(0), "__NAME", Type.INT);
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals("__NAME", 1987, record1.get("__NAME"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例3: testNonIdentCharactersInColumnName
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testNonIdentCharactersInColumnName() throws IOException {
String [] names = { "test_p-a+r/quet" };
String [] types = { "INT" };
String [] vals = { "2015" };
createTableWithColTypesAndNames(names, types, vals);
runImport(getOutputArgv(true, null));
Schema schema = getSchema();
assertEquals(Type.RECORD, schema.getType());
List<Field> fields = schema.getFields();
assertEquals(types.length, fields.size());
checkField(fields.get(0), "TEST_P_A_R_QUET", Type.INT);
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals("TEST_P_A_R_QUET", 2015, record1.get("TEST_P_A_R_QUET"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例4: testNullableParquetImport
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testNullableParquetImport() throws IOException, SQLException {
String [] types = { "INT" };
String [] vals = { null };
createTableWithColTypes(types, vals);
runImport(getOutputArgv(true, null));
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertNull(record1.get("DATA_COL0"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例5: testQueryImport
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testQueryImport() throws IOException, SQLException {
String [] types = { "INT" };
String [] vals = { "1" };
createTableWithColTypes(types, vals);
runImport(getOutputQueryArgv(true, null));
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals(1, record1.get("DATA_COL0"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例6: testIncrementalParquetImport
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testIncrementalParquetImport() throws IOException, SQLException {
String [] types = { "INT" };
String [] vals = { "1" };
createTableWithColTypes(types, vals);
runImport(getOutputArgv(true, null));
runImport(getOutputArgv(true, new String[]{"--append"}));
DatasetReader<GenericRecord> reader = getReader();
try {
assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals(1, record1.get("DATA_COL0"));
record1 = reader.next();
assertEquals(1, record1.get("DATA_COL0"));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例7: getMostRecentWebPageSnapshot
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Get the most recent WebPageSnapshotModel from HBase
*
* @param url
* The URL to get the snapshotted page from HBase
* @return The WebPageSnapshotModel, or null if there are no fetches for this
* URL
*/
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
url = normalizeUrl(url);
DatasetReader<WebPageSnapshotModel> reader = null;
try {
// we don't know the exact timestamp in the key, but we know since keys
// are in timestamp descending order that the first row for an URL will be
// the most recent.
reader = webPageSnapshotModels.from("url", url)
.from("fetchedAtRevTs", 0L).to("url", url)
.to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
if (reader.hasNext()) {
return reader.next();
} else {
return null;
}
} finally {
if (reader != null) {
reader.close();
}
}
}
示例8: testMultiTableImportAsParquetFormat
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void testMultiTableImportAsParquetFormat() throws IOException {
String [] argv = getArgv(new String[]{"--as-parquetfile"}, null);
runImport(new ImportAllTablesTool(), argv);
Path warehousePath = new Path(this.getWarehouseDir());
int i = 0;
for (String tableName : this.tableNames) {
Path tablePath = new Path(warehousePath, tableName);
Dataset dataset = Datasets.load("dataset:file:" + tablePath);
// dequeue the expected value for this table. This
// list has the same order as the tableNames list.
String expectedVal = Integer.toString(i++) + ","
+ this.expectedStrings.get(0);
this.expectedStrings.remove(0);
DatasetReader<GenericRecord> reader = dataset.newReader();
try {
GenericRecord record = reader.next();
String line = record.get(0) + "," + record.get(1);
assertEquals("Table " + tableName + " expected a different string",
expectedVal, line);
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
}
示例9: testJobConfiguration
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
@Test
public void testJobConfiguration() {
Map<String,String> settings = ImmutableMap.of("test.app.level.setting", "appvalue",
"kite.job.write-config-job.test.job.level.setting", "jobvalue",
"kite.job.write-config-job.output.kv-output.test.output.level.setting", "outputvalue");
AppContext context = new AppContext(settings, getConfiguration());
TestScheduler scheduler = TestScheduler.load(WriteConfigOutputApp.class, context);
Instant nominalTime = new DateTime(2015, 5, 15, 12, 0, 0, 0, DateTimeZone.UTC).toInstant();
scheduler.runScheduledJobs(nominalTime);
Dataset<KeyValues> ds = Datasets.load(WriteConfigOutputApp.OUTPUT_DATASET, KeyValues.class);
DatasetReader<KeyValues> reader = ds.newReader();
try {
KeyValues kv = reader.next();
Assert.assertEquals(ImmutableMap.of(
"test.app.level.setting", "appvalue",
"test.job.level.setting", "jobvalue",
"output.kv-output.test.output.level.setting", "outputvalue"),
kv.getJobsettings());
Assert.assertEquals(ImmutableMap.of(
"test.app.level.setting", "appvalue",
"test.job.level.setting", "jobvalue",
"test.output.level.setting", "outputvalue"),
kv.getOutputsettings());
} finally {
Closeables.closeQuietly(reader);
}
}
示例10: runParquetImportTest
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
private void runParquetImportTest(String codec) throws IOException {
String[] types = {"BIT", "INTEGER", "BIGINT", "REAL", "DOUBLE", "VARCHAR(6)",
"VARBINARY(2)",};
String[] vals = {"true", "100", "200", "1.0", "2.0", "'s'", "'0102'", };
createTableWithColTypes(types, vals);
String [] extraArgs = { "--compression-codec", codec};
runImport(getOutputArgv(true, extraArgs));
assertEquals(CompressionType.forName(codec), getCompressionType());
Schema schema = getSchema();
assertEquals(Type.RECORD, schema.getType());
List<Field> fields = schema.getFields();
assertEquals(types.length, fields.size());
checkField(fields.get(0), "DATA_COL0", Type.BOOLEAN);
checkField(fields.get(1), "DATA_COL1", Type.INT);
checkField(fields.get(2), "DATA_COL2", Type.LONG);
checkField(fields.get(3), "DATA_COL3", Type.FLOAT);
checkField(fields.get(4), "DATA_COL4", Type.DOUBLE);
checkField(fields.get(5), "DATA_COL5", Type.STRING);
checkField(fields.get(6), "DATA_COL6", Type.BYTES);
DatasetReader<GenericRecord> reader = getReader();
try {
GenericRecord record1 = reader.next();
assertNotNull(record1);
assertEquals("DATA_COL0", true, record1.get("DATA_COL0"));
assertEquals("DATA_COL1", 100, record1.get("DATA_COL1"));
assertEquals("DATA_COL2", 200L, record1.get("DATA_COL2"));
assertEquals("DATA_COL3", 1.0f, record1.get("DATA_COL3"));
assertEquals("DATA_COL4", 2.0, record1.get("DATA_COL4"));
assertEquals("DATA_COL5", "s", record1.get("DATA_COL5"));
Object object = record1.get("DATA_COL6");
assertTrue(object instanceof ByteBuffer);
ByteBuffer b = ((ByteBuffer) object);
assertEquals((byte) 1, b.get(0));
assertEquals((byte) 2, b.get(1));
assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
示例11: loadWhenAvailable
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Loads the expected number of items when they are available in the dataset.
*/
private static List<ExampleEvent> loadWhenAvailable(View<ExampleEvent> view,
int expected,
int attempts) throws InterruptedException {
for (int attempt = 0; attempt < attempts; ++attempt) {
List<ExampleEvent> items = Lists.newArrayList();
DatasetReader<ExampleEvent> reader = view.newReader();
int count = 0;
try {
while (count < expected && reader.hasNext()) {
ExampleEvent event = reader.next();
items.add(event);
count++;
}
} finally {
reader.close();
}
if (count == expected) {
return items;
}
Thread.sleep(1000);
}
return null;
}