本文整理汇总了Java中org.kitesdk.data.DatasetReader.hasNext方法的典型用法代码示例。如果您正苦于以下问题:Java DatasetReader.hasNext方法的具体用法?Java DatasetReader.hasNext怎么用?Java DatasetReader.hasNext使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.kitesdk.data.DatasetReader
的用法示例。
在下文中一共展示了DatasetReader.hasNext方法的9个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: run
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
@DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {
DatasetReader<StandardEvent> reader = input.newReader();
DatasetWriter<StandardEvent> writer = output.newWriter();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
Closeables.closeQuietly(writer);
}
}
示例2: run
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
@DataOut(name="target_users") View<GenericRecord> output) {
DatasetReader<GenericRecord> reader = input.newReader();
DatasetWriter<GenericRecord> writer = output.newWriter();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
Closeables.closeQuietly(writer);
}
}
示例3: getSnapshotTimestamps
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Get the epoch timestamps for every snapshot time of an URL in HBase.
*
* @param url
* The URL of the page to get snapshot timestamps for
* @return The list of timestamps
*/
public List<Long> getSnapshotTimestamps(String url) {
url = normalizeUrl(url);
List<Long> snapshotTimestamps = new ArrayList<Long>();
DatasetReader<WebPageSnapshotModel> reader = null;
try {
reader = webPageSnapshotModels.from("url", url)
.from("fetchedAtRevTs", 0L).to("url", url)
.to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
while (reader.hasNext()) {
snapshotTimestamps.add(reader.next().getFetchedAt());
}
} finally {
if (reader != null) {
reader.close();
}
}
return snapshotTimestamps;
}
示例4: getMostRecentWebPageSnapshot
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Get the most recent WebPageSnapshotModel from HBase
*
* @param url
* The URL to get the snapshotted page from HBase
* @return The WebPageSnapshotModel, or null if there are no fetches for this
* URL
*/
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
url = normalizeUrl(url);
DatasetReader<WebPageSnapshotModel> reader = null;
try {
// we don't know the exact timestamp in the key, but we know since keys
// are in timestamp descending order that the first row for an URL will be
// the most recent.
reader = webPageSnapshotModels.from("url", url)
.from("fetchedAtRevTs", 0L).to("url", url)
.to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
if (reader.hasNext()) {
return reader.next();
} else {
return null;
}
} finally {
if (reader != null) {
reader.close();
}
}
}
示例5: getWebPageSnapshotsSince
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Get WebPageSnapshotModels for an URL from HBase since the since param.
*
* @param url
* The URL of the page to fetch
* @param since
* The models to fetch since
* @return The list of models that have been fetched for an URL since the
* since param.
*/
private List<WebPageSnapshotModel> getWebPageSnapshotsSince(String url,
long since) {
url = normalizeUrl(url);
List<WebPageSnapshotModel> models = new ArrayList<WebPageSnapshotModel>();
DatasetReader<WebPageSnapshotModel> reader = null;
try {
reader = webPageSnapshotModels.from("url", url)
.from("fetchedAtRevTs", 0L).to("url", url)
.to("fetchedAtRevTs", since).newReader();
while (reader.hasNext()) {
models.add(reader.next());
}
} finally {
if (reader != null) {
reader.close();
}
}
return models;
}
示例6: verifyHiveDataset
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
private void verifyHiveDataset(String tableName, Object[][] valsArray) {
String datasetUri = String.format("dataset:hive:default/%s",
tableName.toLowerCase());
assertTrue(Datasets.exists(datasetUri));
Dataset dataset = Datasets.load(datasetUri);
assertFalse(dataset.isEmpty());
DatasetReader<GenericRecord> reader = dataset.newReader();
try {
List<String> expectations = new ArrayList<String>();
if (valsArray != null) {
for (Object[] vals : valsArray) {
expectations.add(Arrays.toString(vals));
}
}
while (reader.hasNext() && expectations.size() > 0) {
String actual = Arrays.toString(
convertGenericRecordToArray(reader.next()));
assertTrue("Expect record: " + actual, expectations.remove(actual));
}
assertFalse(reader.hasNext());
assertEquals(0, expectations.size());
} finally {
reader.close();
}
}
示例7: loadWhenAvailable
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Loads the expected number of items when they are available in the dataset.
* @param expected
* @return
*/
private <T> List<T> loadWhenAvailable(View<T> view, int expected) {
for (int attempt = 0; attempt < 20; ++attempt) {
List<T> items = Lists.newArrayList();
DatasetReader<T> reader = view.newReader();
int count = 0;
try {
for (; count < expected; ++count) {
if (!reader.hasNext())
continue;
items.add(reader.next());
}
} finally {
reader.close();
}
if (count == expected) {
return items;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
Assert.fail("Unable to load the expected items.");
return null;
}
示例8: runJob
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
@Override
public void runJob(Map params, JobContext jobContext, Instant nominalTime) {
View output = (View) params.get("output");
DatasetWriter<GenericRecord> writer = output.newWriter();
// Simply write all of the input datasets to the output.
for (Map.Entry<String,View> param: ((Map<String,View>) params).entrySet()) {
if (!param.getKey().equals("output")) {
DatasetReader<GenericRecord> reader = param.getValue().newReader();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
}
}
}
Closeables.closeQuietly(writer);
}
示例9: loadWhenAvailable
import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
* Loads the expected number of items when they are available in the dataset.
*/
private static List<ExampleEvent> loadWhenAvailable(View<ExampleEvent> view,
int expected,
int attempts) throws InterruptedException {
for (int attempt = 0; attempt < attempts; ++attempt) {
List<ExampleEvent> items = Lists.newArrayList();
DatasetReader<ExampleEvent> reader = view.newReader();
int count = 0;
try {
while (count < expected && reader.hasNext()) {
ExampleEvent event = reader.next();
items.add(event);
count++;
}
} finally {
reader.close();
}
if (count == expected) {
return items;
}
Thread.sleep(1000);
}
return null;
}