Java DatasetReader.hasNext方法代码示例

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
                @DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {

  DatasetReader<StandardEvent> reader = input.newReader();
  DatasetWriter<StandardEvent> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
                @DataOut(name="target_users") View<GenericRecord> output) {

  DatasetReader<GenericRecord> reader = input.newReader();
  DatasetWriter<GenericRecord> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Get the epoch timestamps for every snapshot time of an URL in HBase.
 * 
 * @param url
 *          The URL of the page to get snapshot timestamps for
 * @return The list of timestamps
 */
public List<Long> getSnapshotTimestamps(String url) {
  url = normalizeUrl(url);
  List<Long> snapshotTimestamps = new ArrayList<Long>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    while (reader.hasNext()) {
      snapshotTimestamps.add(reader.next().getFetchedAt());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return snapshotTimestamps;
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Get the most recent WebPageSnapshotModel from HBase
 * 
 * @param url
 *          The URL to get the snapshotted page from HBase
 * @return The WebPageSnapshotModel, or null if there are no fetches for this
 *         URL
 */
private WebPageSnapshotModel getMostRecentWebPageSnapshot(String url) {
  url = normalizeUrl(url);
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    // we don't know the exact timestamp in the key, but we know since keys
    // are in timestamp descending order that the first row for an URL will be
    // the most recent.
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", Long.MAX_VALUE).newReader();
    if (reader.hasNext()) {
      return reader.next();
    } else {
      return null;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Get WebPageSnapshotModels for an URL from HBase since the since param.
 * 
 * @param url
 *          The URL of the page to fetch
 * @param since
 *          The models to fetch since
 * @return The list of models that have been fetched for an URL since the
 *         since param.
 */
private List<WebPageSnapshotModel> getWebPageSnapshotsSince(String url,
    long since) {
  url = normalizeUrl(url);
  List<WebPageSnapshotModel> models = new ArrayList<WebPageSnapshotModel>();
  DatasetReader<WebPageSnapshotModel> reader = null;
  try {
    reader = webPageSnapshotModels.from("url", url)
        .from("fetchedAtRevTs", 0L).to("url", url)
        .to("fetchedAtRevTs", since).newReader();
    while (reader.hasNext()) {
      models.add(reader.next());
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  return models;
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
private void verifyHiveDataset(String tableName, Object[][] valsArray) {
  String datasetUri = String.format("dataset:hive:default/%s",
      tableName.toLowerCase());
  assertTrue(Datasets.exists(datasetUri));
  Dataset dataset = Datasets.load(datasetUri);
  assertFalse(dataset.isEmpty());

  DatasetReader<GenericRecord> reader = dataset.newReader();
  try {
    List<String> expectations = new ArrayList<String>();
    if (valsArray != null) {
      for (Object[] vals : valsArray) {
        expectations.add(Arrays.toString(vals));
      }
    }

    while (reader.hasNext() && expectations.size() > 0) {
      String actual = Arrays.toString(
          convertGenericRecordToArray(reader.next()));
      assertTrue("Expect record: " + actual, expectations.remove(actual));
    }
    assertFalse(reader.hasNext());
    assertEquals(0, expectations.size());
  } finally {
    reader.close();
  }
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Loads the expected number of items when they are available in the dataset.
 * @param expected
 * @return
 */
private <T> List<T> loadWhenAvailable(View<T> view, int expected) {

  for (int attempt = 0; attempt < 20; ++attempt) {

    List<T> items = Lists.newArrayList();

    DatasetReader<T> reader = view.newReader();
    int count = 0;

    try {

      for (; count < expected; ++count) {

        if (!reader.hasNext())
          continue;


        items.add(reader.next());
      }

    } finally {
      reader.close();
    }

    if (count == expected) {
      return items;
    }

    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }

  Assert.fail("Unable to load the expected items.");
  return null;
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
@Override
public void runJob(Map params, JobContext jobContext, Instant nominalTime) {

  View output = (View) params.get("output");

  DatasetWriter<GenericRecord> writer = output.newWriter();

  // Simply write all of the input datasets to the output.
  for (Map.Entry<String,View> param: ((Map<String,View>)  params).entrySet()) {

    if (!param.getKey().equals("output")) {
      DatasetReader<GenericRecord> reader = param.getValue().newReader();

      try {
        while (reader.hasNext()) {

          writer.write(reader.next());
        }
      } finally {

        Closeables.closeQuietly(reader);

      }
    }
  }

  Closeables.closeQuietly(writer);
}
 

import org.kitesdk.data.DatasetReader; //导入方法依赖的package包/类
/**
 * Loads the expected number of items when they are available in the dataset.
 */
private static List<ExampleEvent> loadWhenAvailable(View<ExampleEvent> view,
                                                    int expected,
                                                    int attempts) throws InterruptedException {

  for (int attempt = 0; attempt < attempts; ++attempt) {

    List<ExampleEvent> items = Lists.newArrayList();

    DatasetReader<ExampleEvent> reader = view.newReader();
    int count = 0;

    try {

      while (count < expected && reader.hasNext()) {

        ExampleEvent event = reader.next();

        items.add(event);

        count++;
      }

    } finally {
      reader.close();
    }

    if (count == expected) {
      return items;
    }

    Thread.sleep(1000);

  }

  return null;
}