本文整理汇总了Java中org.kitesdk.data.Datasets类的典型用法代码示例。如果您正苦于以下问题:Java Datasets类的具体用法?Java Datasets怎么用?Java Datasets使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Datasets类属于org.kitesdk.data包,在下文中一共展示了Datasets类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: SavePolicy
import org.kitesdk.data.Datasets; //导入依赖的package包/类
private SavePolicy(Context context) {
String uri = context.getString(CONFIG_KITE_ERROR_DATASET_URI);
Preconditions.checkArgument(uri != null, "Must set "
+ CONFIG_KITE_ERROR_DATASET_URI + " when " + CONFIG_FAILURE_POLICY
+ "=save");
if (Datasets.exists(uri)) {
dataset = Datasets.load(uri, AvroFlumeEvent.class);
} else {
DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
.schema(AvroFlumeEvent.class)
.build();
dataset = Datasets.create(uri, descriptor, AvroFlumeEvent.class);
}
nEventsHandled = 0;
}
示例2: testOldConfig
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testOldConfig() throws EventDeliveryException {
config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI, null);
config.put(DatasetSinkConstants.CONFIG_KITE_REPO_URI, FILE_REPO_URI);
config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_NAME, DATASET_NAME);
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
Assert.assertEquals("Should have committed", 0, remaining(in));
}
示例3: testDatasetUriOverridesOldConfig
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testDatasetUriOverridesOldConfig() throws EventDeliveryException {
// CONFIG_KITE_DATASET_URI is still set, otherwise this will cause an error
config.put(DatasetSinkConstants.CONFIG_KITE_REPO_URI, "bad uri");
config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_NAME, "");
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
Assert.assertEquals("Should have committed", 0, remaining(in));
}
示例4: testBatchSize
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testBatchSize() throws EventDeliveryException {
DatasetSink sink = sink(in, config);
// release one record per process call
config.put("kite.batchSize", "2");
Configurables.configure(sink, config);
sink.start();
sink.process(); // process the first and second
sink.roll(); // roll at the next process call
sink.process(); // roll and process the third
Assert.assertEquals(
Sets.newHashSet(expected.subList(0, 2)),
read(Datasets.load(FILE_DATASET_URI)));
Assert.assertEquals("Should have committed", 0, remaining(in));
sink.roll(); // roll at the next process call
sink.process(); // roll, the channel is empty
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
sink.stop();
}
示例5: testTimedFileRolling
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testTimedFileRolling()
throws EventDeliveryException, InterruptedException {
// use a new roll interval
config.put("kite.rollInterval", "1"); // in seconds
DatasetSink sink = sink(in, config);
Dataset<GenericRecord> records = Datasets.load(FILE_DATASET_URI);
// run the sink
sink.start();
sink.process();
Assert.assertEquals("Should have committed", 0, remaining(in));
Thread.sleep(1100); // sleep longer than the roll interval
sink.process(); // rolling happens in the process method
Assert.assertEquals(Sets.newHashSet(expected), read(records));
// wait until the end to stop because it would close the files
sink.stop();
}
示例6: testFileStoreWithSavePolicy
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testFileStoreWithSavePolicy() throws EventDeliveryException {
if (Datasets.exists(ERROR_DATASET_URI)) {
Datasets.delete(ERROR_DATASET_URI);
}
config.put(DatasetSinkConstants.CONFIG_FAILURE_POLICY,
DatasetSinkConstants.SAVE_FAILURE_POLICY);
config.put(DatasetSinkConstants.CONFIG_KITE_ERROR_DATASET_URI,
ERROR_DATASET_URI);
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
Assert.assertEquals("Should have committed", 0, remaining(in));
}
示例7: testCommitOnBatch
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testCommitOnBatch() throws EventDeliveryException {
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
// the transaction should commit during the call to process
Assert.assertEquals("Should have committed", 0, remaining(in));
// but the data won't be visible yet
Assert.assertEquals(0,
read(Datasets.load(FILE_DATASET_URI)).size());
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
}
示例8: testCloseAndCreateWriter
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testCloseAndCreateWriter() throws EventDeliveryException {
config.put(DatasetSinkConstants.CONFIG_FLUSHABLE_COMMIT_ON_BATCH,
Boolean.toString(false));
config.put(DatasetSinkConstants.CONFIG_SYNCABLE_SYNC_ON_BATCH,
Boolean.toString(false));
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.closeWriter();
sink.commitTransaction();
sink.createWriter();
Assert.assertNotNull("Writer should not be null", sink.getWriter());
Assert.assertEquals("Should have committed", 0, remaining(in));
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
}
示例9: testCloseWriter
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testCloseWriter() throws EventDeliveryException {
config.put(DatasetSinkConstants.CONFIG_FLUSHABLE_COMMIT_ON_BATCH,
Boolean.toString(false));
config.put(DatasetSinkConstants.CONFIG_SYNCABLE_SYNC_ON_BATCH,
Boolean.toString(false));
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.closeWriter();
sink.commitTransaction();
Assert.assertNull("Writer should be null", sink.getWriter());
Assert.assertEquals("Should have committed", 0, remaining(in));
sink.stop();
Assert.assertEquals(
Sets.newHashSet(expected),
read(Datasets.load(FILE_DATASET_URI)));
}
示例10: testCreateWriter
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Test
public void testCreateWriter() throws EventDeliveryException {
config.put(DatasetSinkConstants.CONFIG_FLUSHABLE_COMMIT_ON_BATCH,
Boolean.toString(false));
config.put(DatasetSinkConstants.CONFIG_SYNCABLE_SYNC_ON_BATCH,
Boolean.toString(false));
DatasetSink sink = sink(in, config);
// run the sink
sink.start();
sink.process();
sink.commitTransaction();
sink.createWriter();
Assert.assertNotNull("Writer should not be null", sink.getWriter());
Assert.assertEquals("Should have committed", 0, remaining(in));
sink.stop();
Assert.assertEquals(0, read(Datasets.load(FILE_DATASET_URI)).size());
}
示例11: setUp
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@BeforeMethod(alwaysRun = true)
public void setUp() {
initMocks(this);
mockStatic(KiteDatasetExecutor.class);
mockStatic(Datasets.class);
destroyer = new KiteToDestroyer() {
@Override
protected KiteDatasetExecutor getExecutor(String uri, Schema schema,
FileFormat format) {
return executorMock;
}
};
linkConfig = new LinkConfiguration();
toJobConfig = new ToJobConfiguration();
toJobConfig.toJobConfig.uri = "dataset:file:/foo/bar";
toJobConfig.toJobConfig.fileFormat = FileFormat.AVRO;
}
示例12: dataset
import org.kitesdk.data.Datasets; //导入依赖的package包/类
/**
* Ensures the given dataset exists, creating it if it doesn't
* and updating the schema if necessary.
*/
protected void dataset(String uri, DatasetDescriptor descriptor) {
try {
Datasets.create(uri, descriptor);
} catch (DatasetExistsException e) {
Dataset existingDataset = Datasets.load(uri);
DatasetDescriptor updated;
// The given discriptor might not have a location,
// so use the current one.
if (descriptor.getLocation() == null) {
updated = new DatasetDescriptor.Builder(descriptor)
.location(existingDataset.getDescriptor().getLocation())
.build();
} else {
updated = descriptor;
}
Datasets.update(uri, updated);
}
}
示例13: run
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Load the users dataset
Dataset<Record> users = Datasets.load(
"dataset:hdfs:/tmp/data/users", Record.class);
// Get a reader for the dataset and read all the users
DatasetReader<Record> reader = null;
try {
reader = users.newReader();
for (GenericRecord user : reader) {
System.out.println(user);
}
} finally {
if (reader != null) {
reader.close();
}
}
return 0;
}
示例14: run
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Load the users dataset
Dataset<Record> users = Datasets.load(
"dataset:hive?dataset=users", Record.class);
// Get a reader for the dataset and read all the users
DatasetReader<Record> reader = null;
try {
reader = users.newReader();
for (GenericRecord user : users.newReader()) {
System.out.println(user);
}
} finally {
if (reader != null) {
reader.close();
}
}
return 0;
}
示例15: run
import org.kitesdk.data.Datasets; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
// Load the users dataset
Dataset<Record> users = Datasets.load(
"dataset:hdfs:/tmp/data/users", Record.class);
// Get a reader for the dataset and read all the users
DatasetReader<Record> reader = null;
try {
reader = users.with("favoriteColor", "green").newReader();
for (GenericRecord user : reader) {
System.out.println(user);
}
} finally {
if (reader != null) {
reader.close();
}
}
return 0;
}