本文整理汇总了Java中org.kitesdk.data.DatasetWriter.write方法的典型用法代码示例。如果您正苦于以下问题:Java DatasetWriter.write方法的具体用法?Java DatasetWriter.write怎么用?Java DatasetWriter.write使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.kitesdk.data.DatasetWriter
的用法示例。
在下文中一共展示了DatasetWriter.write方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
public void run(@DataIn(name="source.events", type=StandardEvent.class) View<StandardEvent> input,
@DataOut(name="target.events", type=StandardEvent.class) View<StandardEvent> output) {
DatasetReader<StandardEvent> reader = input.newReader();
DatasetWriter<StandardEvent> writer = output.newWriter();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
Closeables.closeQuietly(writer);
}
}
示例2: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
public void run(@DataIn(name="source_users") View<GenericRecord> input,
@DataOut(name="target_users") View<GenericRecord> output) {
DatasetReader<GenericRecord> reader = input.newReader();
DatasetWriter<GenericRecord> writer = output.newWriter();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
Closeables.closeQuietly(writer);
}
}
示例3: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
public void run(@DataOut(name="example_events", type=ExampleEvent.class) Signalable<ExampleEvent> view) {
// Write some test data to the view.
DatasetWriter<ExampleEvent> writer = view.newWriter();
try {
for (int i = 0; i < 10; ++i) {
ExampleEvent event = ExampleEvent.newBuilder()
.setUserId(i)
.setSessionId(Integer.toString(i))
.setTimestamp(getNominalTime().getMillis())
.build();
writer.write(event);
}
}
finally {
writer.close();
}
// Signal that our view is ready.
view.signalReady();
}
示例4: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(List<String> args) throws Exception {
Preconditions.checkState(!Datasets.exists(uri),
"events dataset already exists");
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schema(StandardEvent.class).build();
View<StandardEvent> events = Datasets.create(uri, descriptor, StandardEvent.class);
DatasetWriter<StandardEvent> writer = events.newWriter();
try {
while (System.currentTimeMillis() - baseTimestamp < 36000) {
writer.write(generateRandomEvent());
}
} finally {
writer.close();
}
System.out.println("Generated " + counter + " events");
return 0;
}
示例5: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Create a partition strategy that hash partitions on username with 10 buckets
PartitionStrategy partitionStrategy = new PartitionStrategy.Builder()
.identity("favoriteColor", "favorite_color")
.build();
// Create a dataset of users with the Avro schema
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schemaUri("resource:user.avsc")
.partitionStrategy(partitionStrategy)
.build();
Dataset<Record> users = Datasets.create(
"dataset:hdfs:/tmp/data/users", descriptor, Record.class);
// Get a writer for the dataset and write some users to it
DatasetWriter<Record> writer = null;
try {
writer = users.newWriter();
Random rand = new Random();
GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
for (int i = 0; i < 100; i++) {
Record record = builder.set("username", "user-" + i)
.set("creationDate", System.currentTimeMillis())
.set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
writer.write(record);
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例6: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Create a dataset of users with the Avro schema
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schemaUri("resource:user.avsc")
.build();
Dataset<Record> users = Datasets.create(
"dataset:hdfs:/tmp/data/users", descriptor, Record.class);
// Get a writer for the dataset and write some users to it
DatasetWriter<Record> writer = null;
try {
writer = users.newWriter();
Random rand = new Random();
GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
for (int i = 0; i < 100; i++) {
Record record = builder.set("username", "user-" + i)
.set("creationDate", System.currentTimeMillis())
.set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
writer.write(record);
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例7: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schemaUri("resource:user.avsc")
.format(Formats.PARQUET)
.build();
Dataset<Record> users = Datasets.create(
"dataset:hdfs:/tmp/data/users", descriptor, Record.class);
// Get a writer for the dataset and write some users to it
DatasetWriter<Record> writer = null;
try {
writer = users.newWriter();
Random rand = new Random();
GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
for (int i = 0; i < 100; i++) {
Record record = builder.set("username", "user-" + i)
.set("creationDate", System.currentTimeMillis())
.set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
writer.write(record);
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例8: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Create a dataset of products with the Avro schema
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schema(Product.class)
.build();
Dataset<Product> products = Datasets.create(
"dataset:hdfs:/tmp/data/products", descriptor, Product.class);
// Get a writer for the dataset and write some products to it
DatasetWriter<Product> writer = null;
try {
writer = products.newWriter();
int i = 0;
for (String name : names) {
Product product = new Product();
product.setName(name);
product.setId(i++);
writer.write(product);
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例9: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Create a dataset of users with the Avro schema
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schemaUri("resource:user.avsc")
.build();
Dataset<Record> users = Datasets.create("dataset:hive?dataset=users",
descriptor, Record.class);
// Get a writer for the dataset and write some users to it
DatasetWriter<Record> writer = null;
try {
writer = users.newWriter();
Random rand = new Random();
GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
for (int i = 0; i < 100; i++) {
Record record = builder.set("username", "user-" + i)
.set("creationDate", System.currentTimeMillis())
.set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
writer.write(record);
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例10: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
public void run(@DataOut(name="kv-output", type= KeyValues.class) View<KeyValues> output) {
DatasetWriter<KeyValues> writer = output.newWriter();
try {
JobContext context = getJobContext();
KeyValues kv = KeyValues.newBuilder()
.setJobsettings(context.getSettings())
.setOutputsettings(context.getOutputSettings("kv-output"))
.build();
writer.write(kv);
} finally {
Closeables.closeQuietly(writer);
}
}
示例11: run
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// going to generate a lot of random log messages
final Random rand = new Random();
// data is written to the staging dataset
Dataset<Record> staging = Datasets.load(
"dataset:file:/tmp/data/logs_staging", Record.class);
// this is going to build our simple log records
GenericRecordBuilder builder = new GenericRecordBuilder(
staging.getDescriptor().getSchema());
// generate timestamps 1 second apart starting 1 day ago
final Calendar now = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
final long yesterday = now.getTimeInMillis() - DAY_IN_MILLIS;
DatasetWriter<Record> writer = null;
try {
writer = staging.newWriter();
// generate 15,000 messages, each 5 seconds apart, starting 24 hours ago
// this is a little less than 24 hours worth of messages
for (int second : Ranges.closed(0, 15000).asSet(DiscreteDomains.integers())) {
LOG.info("Generating log message " + second);
builder.set("timestamp", yesterday + second * 5000);
builder.set("component", "GenerateSimpleLogs");
int level = rand.nextInt(LOG_LEVELS.length);
builder.set("level", LOG_LEVELS[level]);
builder.set("message", LOG_MESSAGES[level]);
writer.write(builder.build());
}
if (writer instanceof Flushable) {
((Flushable) writer).flush();
}
} finally {
if (writer != null) {
writer.close();
}
}
return 0;
}
示例12: runJob
import org.kitesdk.data.DatasetWriter; //导入方法依赖的package包/类
@Override
public void runJob(Map params, JobContext jobContext, Instant nominalTime) {
View output = (View) params.get("output");
DatasetWriter<GenericRecord> writer = output.newWriter();
// Simply write all of the input datasets to the output.
for (Map.Entry<String,View> param: ((Map<String,View>) params).entrySet()) {
if (!param.getKey().equals("output")) {
DatasetReader<GenericRecord> reader = param.getValue().newReader();
try {
while (reader.hasNext()) {
writer.write(reader.next());
}
} finally {
Closeables.closeQuietly(reader);
}
}
}
Closeables.closeQuietly(writer);
}