Java View类代码示例

import org.kitesdk.data.View; //导入依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
                @DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {

  DatasetReader<StandardEvent> reader = input.newReader();
  DatasetWriter<StandardEvent> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
                @DataOut(name="target_users") View<GenericRecord> output) {

  DatasetReader<GenericRecord> reader = input.newReader();
  DatasetWriter<GenericRecord> writer = output.newWriter();

  try {
    while (reader.hasNext()) {

      writer.write(reader.next());
    }
  } finally {

    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(writer);
  }
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
public void run(@DataIn(name="example_events", type=ExampleEvent.class) View<ExampleEvent> input,
                @DataOut(name="odd_users", type=ExampleEvent.class) View<ExampleEvent> output) throws IOException {

  Job job = Job.getInstance(getJobContext().getHadoopConf());
  DatasetKeyInputFormat.configure(job).readFrom(input);
  DatasetKeyOutputFormat.configure(job).writeTo(output);

  JavaPairRDD<ExampleEvent, Void> inputData = getJobContext()
      .getSparkContext()
      .newAPIHadoopRDD(job.getConfiguration(), DatasetKeyInputFormat.class,
          ExampleEvent.class, Void.class);

  JavaPairRDD<ExampleEvent, Void> filteredData = inputData.filter(new KeepOddUsers());

  filteredData.saveAsNewAPIHadoopDataset(job.getConfiguration());
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
@Override
public int run(List<String> args) throws Exception {

  Preconditions.checkState(!Datasets.exists(uri),
      "events dataset already exists");

  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .schema(StandardEvent.class).build();

  View<StandardEvent> events = Datasets.create(uri, descriptor, StandardEvent.class);
  DatasetWriter<StandardEvent> writer = events.newWriter();
  try {
    while (System.currentTimeMillis() - baseTimestamp < 36000) {
      writer.write(generateRandomEvent());
    }
  } finally {
    writer.close();
  }

  System.out.println("Generated " + counter + " events");

  return 0;
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
public static <T> HashSet<T> read(View<T> view) {
  DatasetReader<T> reader = null;
  try {
    reader = view.newReader();
    return Sets.newHashSet(reader.iterator());
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
/**
 * Save all RDDs in the given DStream to the given view.
 * @param dstream
 * @param view
 */
public static <T> void save(JavaDStream<T> dstream, final View<T> view) {

  final String uri = view.getUri().toString();

  dstream.foreachRDD(new Function2<JavaRDD<T>, Time, Void>() {
    @Override
    public Void call(JavaRDD<T> rdd, Time time) throws Exception {

      save(rdd, uri);

      return null;
    }
  });
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
public void run(@DataIn(name="source.users") View<GenericRecord> input,
                @DataOut(name="target.users") View<GenericRecord> output) throws IOException {

  Job job = Job.getInstance();
  DatasetKeyInputFormat.configure(job).readFrom(input);
  DatasetKeyOutputFormat.configure(job).writeTo(output);

  @SuppressWarnings("unchecked")
  JavaPairRDD<GenericData.Record, Void> inputData = getJobContext()
      .getSparkContext()
      .newAPIHadoopRDD(job.getConfiguration(), DatasetKeyInputFormat.class,
          GenericData.Record.class, Void.class);

  inputData.saveAsNewAPIHadoopDataset(job.getConfiguration());
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
/**
 * Loads the expected number of items when they are available in the dataset.
 * @param expected
 * @return
 */
private <T> List<T> loadWhenAvailable(View<T> view, int expected) {

  for (int attempt = 0; attempt < 20; ++attempt) {

    List<T> items = Lists.newArrayList();

    DatasetReader<T> reader = view.newReader();
    int count = 0;

    try {

      for (; count < expected; ++count) {

        if (!reader.hasNext())
          continue;


        items.add(reader.next());
      }

    } finally {
      reader.close();
    }

    if (count == expected) {
      return items;
    }

    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }

  Assert.fail("Unable to load the expected items.");
  return null;
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
@Test
public void testJob() throws IOException {

  List<SmallEvent> events = Lists.newArrayList();

  for (int i = 0; i < 10; ++i) {
    SmallEvent event = SmallEvent.newBuilder()
        .setSessionId("1234")
        .setUserId(i)
        .build();

    events.add(event);
  }

  harness.writeMessages(StreamingSparkApp.TOPIC_NAME, events);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  View<SmallEvent> view = Datasets.load(StreamingSparkApp.EVENTS_DS_URI, SmallEvent.class);

  List<SmallEvent> results = loadWhenAvailable(view, 10);

  Assert.assertEquals(events, results);
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
/**
 * Checks if the given view contains the expected messages within a timeout period.
 */
static boolean checkMessages(View view, List<ExampleEvent> expected, int timeoutSeconds) throws InterruptedException {

  List<ExampleEvent> actual = loadWhenAvailable(view, expected.size(), timeoutSeconds);

  if (actual == null) {
    return false;
  }

  return expected.equals(actual);
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
private DatasetWriter<GenericRecord> newWriter(
    final UserGroupInformation login, final URI uri) {
  View<GenericRecord> view = KerberosUtil.runPrivileged(login,
      new PrivilegedExceptionAction<Dataset<GenericRecord>>() {
        @Override
        public Dataset<GenericRecord> run() {
          return Datasets.load(uri);
        }
      });

  DatasetDescriptor descriptor = view.getDataset().getDescriptor();
  String formatName = descriptor.getFormat().getName();
  Preconditions.checkArgument(allowedFormats().contains(formatName),
      "Unsupported format: " + formatName);

  Schema newSchema = descriptor.getSchema();
  if (targetSchema == null || !newSchema.equals(targetSchema)) {
    this.targetSchema = descriptor.getSchema();
    // target dataset schema has changed, invalidate all readers based on it
    readers.invalidateAll();
  }

  this.reuseDatum = !("parquet".equals(formatName));
  this.datasetName = view.getDataset().getName();

  return view.newWriter();
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  final long startOfToday = startOfDay();

  // the destination dataset
  Dataset<Record> persistent = Datasets.load(
      "dataset:file:/tmp/data/logs", Record.class);

  // the source: anything before today in the staging area
  Dataset<Record> staging = Datasets.load(
      "dataset:file:/tmp/data/logs_staging", Record.class);
  View<Record> ready = staging.toBefore("timestamp", startOfToday);

  ReadableSource<Record> source = CrunchDatasets.asSource(ready);

  PCollection<Record> stagedLogs = read(source);

  getPipeline().write(stagedLogs,
      CrunchDatasets.asTarget(persistent), Target.WriteMode.APPEND);

  PipelineResult result = run();

  if (result.succeeded()) {
    // remove the source data partition from staging
    ready.deleteAll();
    return 0;
  } else {
    return 1;
  }
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
/**
 * Signal the produced views as ready for downstream processing.
 */
protected void signalOutputViews(Map<String,View> views) {

  // If the job specified output parameters,
  // signal them when we complete.
  JobParameters params = getJobParameters();

  if (params != null) {

    Set<String> outputNames = getJobParameters().getOutputNames();

    for (String outputName: outputNames) {

      View view = views.get(outputName);

      if (view instanceof Signalable) {

        ((Signalable) view).signalReady();
      }
    }
  }
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
/**
 * Runs all scheduled jobs in the application using the given
 * nominal time.
 */
public void runScheduledJobs(Instant nominalTime) {

  Map<String,View> views = Maps.newHashMap();

  for (Schedule schedule: app.getSchedules()) {

    for (Schedule.ViewTemplate input: schedule.getViewTemplates().values()) {

      String uri = resolveTemplate(input.getUriTemplate(), nominalTime);

      View view = Datasets.load(uri, input.getInputType());

      views.put(input.getName(), view);
    }
  }

  runScheduledJobs(nominalTime, views);
}
 

import org.kitesdk.data.View; //导入依赖的package包/类
public void run(@DataOut(name="kv-output", type= KeyValues.class) View<KeyValues> output) {


    DatasetWriter<KeyValues> writer = output.newWriter();

    try {

      JobContext context = getJobContext();

      KeyValues kv = KeyValues.newBuilder()
          .setJobsettings(context.getSettings())
          .setOutputsettings(context.getOutputSettings("kv-output"))
          .build();

      writer.write(kv);

    } finally {


      Closeables.closeQuietly(writer);
    }
  }