Java Datasets.delete方法代码示例

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testFileStoreWithSavePolicy() throws EventDeliveryException {
  if (Datasets.exists(ERROR_DATASET_URI)) {
    Datasets.delete(ERROR_DATASET_URI);
  }
  config.put(DatasetSinkConstants.CONFIG_FAILURE_POLICY,
      DatasetSinkConstants.SAVE_FAILURE_POLICY);
  config.put(DatasetSinkConstants.CONFIG_KITE_ERROR_DATASET_URI,
      ERROR_DATASET_URI);
  DatasetSink sink = sink(in, config);

  // run the sink
  sink.start();
  sink.process();
  sink.stop();

  Assert.assertEquals(
      Sets.newHashSet(expected),
      read(Datasets.load(FILE_DATASET_URI)));
  Assert.assertEquals("Should have committed", 0, remaining(in));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Before
public void setup() throws EventDeliveryException {
  Datasets.delete(FILE_DATASET_URI);
  Datasets.create(FILE_DATASET_URI, DESCRIPTOR);

  this.config = new Context();
  config.put("keep-alive", "0");
  this.in = new MemoryChannel();
  Configurables.configure(in, config);

  config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI, FILE_DATASET_URI);

  GenericRecordBuilder builder = new GenericRecordBuilder(RECORD_SCHEMA);
  expected = Lists.<GenericRecord>newArrayList(
      builder.set("id", "1").set("msg", "msg1").build(),
      builder.set("id", "2").set("msg", "msg2").build(),
      builder.set("id", "3").set("msg", "msg3").build());

  putToChannel(in, Iterables.transform(expected,
      new Function<GenericRecord, Event>() {
        private int i = 0;

        @Override
        public Event apply(@Nullable GenericRecord rec) {
          this.i += 1;
          boolean useURI = (i % 2) == 0;
          return event(rec, RECORD_SCHEMA, SCHEMA_FILE, useURI);
        }
      }));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testParquetDataset() throws EventDeliveryException {
  Datasets.delete(FILE_DATASET_URI);
  Dataset<GenericRecord> created = Datasets.create(FILE_DATASET_URI,
      new DatasetDescriptor.Builder(DESCRIPTOR)
          .format("parquet")
          .build());

  DatasetSink sink = sink(in, config);

  // run the sink
  sink.start();
  sink.process();

  // the transaction should not commit during the call to process
  assertThrows("Transaction should still be open", IllegalStateException.class,
      new Callable() {
        @Override
        public Object call() throws EventDeliveryException {
          in.getTransaction().begin();
          return null;
        }
      });
  // The records won't commit until the call to stop()
  Assert.assertEquals("Should not have committed", 0, read(created).size());

  sink.stop();

  Assert.assertEquals(Sets.newHashSet(expected), read(created));
  Assert.assertEquals("Should have committed", 0, remaining(in));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testPartitionedData() throws EventDeliveryException {
  URI partitionedUri = URI.create("dataset:file:target/test_repo/partitioned");
  try {
    Datasets.create(partitionedUri, new DatasetDescriptor.Builder(DESCRIPTOR)
        .partitionStrategy(new PartitionStrategy.Builder()
            .identity("id", 10) // partition by id
            .build())
        .build());

    config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI,
        partitionedUri.toString());
    DatasetSink sink = sink(in, config);

    // run the sink
    sink.start();
    sink.process();
    sink.stop();

    Assert.assertEquals(
        Sets.newHashSet(expected),
        read(Datasets.load(partitionedUri)));
    Assert.assertEquals("Should have committed", 0, remaining(in));
  } finally {
    if (Datasets.exists(partitionedUri)) {
      Datasets.delete(partitionedUri);
    }
  }
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testStartBeforeDatasetCreated() throws EventDeliveryException {
  // delete the dataset created by setup
  Datasets.delete(FILE_DATASET_URI);

  DatasetSink sink = sink(in, config);

  // start the sink
  sink.start();

  // run the sink without a target dataset
  try {
    sink.process();
    Assert.fail("Should have thrown an exception: no such dataset");
  } catch (EventDeliveryException e) {
    // expected
  }

  // create the target dataset
  Datasets.create(FILE_DATASET_URI, DESCRIPTOR);

  // run the sink
  sink.process();
  sink.stop();

  Assert.assertEquals(Sets.newHashSet(expected), read(Datasets.load(FILE_DATASET_URI)));
  Assert.assertEquals("Should have committed", 0, remaining(in));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testMiniClusterStore() throws EventDeliveryException, IOException {
  // setup a minicluster
  MiniDFSCluster cluster = new MiniDFSCluster
      .Builder(new Configuration())
      .build();

  FileSystem dfs = cluster.getFileSystem();
  Configuration conf = dfs.getConf();

  URI hdfsUri = URI.create(
      "dataset:" + conf.get("fs.defaultFS") + "/tmp/repo" + DATASET_NAME);
  try {
    // create a repository and dataset in HDFS
    Datasets.create(hdfsUri, DESCRIPTOR);

    // update the config to use the HDFS repository
    config.put(DatasetSinkConstants.CONFIG_KITE_DATASET_URI, hdfsUri.toString());

    DatasetSink sink = sink(in, config);

    // run the sink
    sink.start();
    sink.process();
    sink.stop();

    Assert.assertEquals(
        Sets.newHashSet(expected),
        read(Datasets.load(hdfsUri)));
    Assert.assertEquals("Should have committed", 0, remaining(in));

  } finally {
    if (Datasets.exists(hdfsUri)) {
      Datasets.delete(hdfsUri);
    }
    cluster.shutdown();
  }
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testMissingSchemaWithSavePolicy() throws EventDeliveryException {
  if (Datasets.exists(ERROR_DATASET_URI)) {
    Datasets.delete(ERROR_DATASET_URI);
  }
  config.put(DatasetSinkConstants.CONFIG_FAILURE_POLICY,
      DatasetSinkConstants.SAVE_FAILURE_POLICY);
  config.put(DatasetSinkConstants.CONFIG_KITE_ERROR_DATASET_URI,
      ERROR_DATASET_URI);
  final DatasetSink sink = sink(in, config);

  Event badEvent = new SimpleEvent();
  badEvent.setHeaders(Maps.<String, String>newHashMap());
  badEvent.setBody(serialize(expected.get(0), RECORD_SCHEMA));
  putToChannel(in, badEvent);

  // run the sink
  sink.start();
  sink.process();
  sink.stop();

  Assert.assertEquals("Good records should have been written",
      Sets.newHashSet(expected),
      read(Datasets.load(FILE_DATASET_URI)));
  Assert.assertEquals("Should not have rolled back", 0, remaining(in));
  Assert.assertEquals("Should have saved the bad event",
      Sets.newHashSet(AvroFlumeEvent.newBuilder()
        .setBody(ByteBuffer.wrap(badEvent.getBody()))
        .setHeaders(toUtf8Map(badEvent.getHeaders()))
        .build()),
      read(Datasets.load(ERROR_DATASET_URI, AvroFlumeEvent.class)));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Test
public void testSerializedWithIncompatibleSchemasWithSavePolicy()
    throws EventDeliveryException {
  if (Datasets.exists(ERROR_DATASET_URI)) {
    Datasets.delete(ERROR_DATASET_URI);
  }
  config.put(DatasetSinkConstants.CONFIG_FAILURE_POLICY,
      DatasetSinkConstants.SAVE_FAILURE_POLICY);
  config.put(DatasetSinkConstants.CONFIG_KITE_ERROR_DATASET_URI,
      ERROR_DATASET_URI);
  final DatasetSink sink = sink(in, config);

  GenericRecordBuilder builder = new GenericRecordBuilder(
      INCOMPATIBLE_SCHEMA);
  GenericData.Record rec = builder.set("username", "koala").build();

  // We pass in a valid schema in the header, but an incompatible schema
  // was used to serialize the record
  Event badEvent = event(rec, INCOMPATIBLE_SCHEMA, SCHEMA_FILE, true);
  putToChannel(in, badEvent);

  // run the sink
  sink.start();
  sink.process();
  sink.stop();

  Assert.assertEquals("Good records should have been written",
      Sets.newHashSet(expected),
      read(Datasets.load(FILE_DATASET_URI)));
  Assert.assertEquals("Should not have rolled back", 0, remaining(in));
  Assert.assertEquals("Should have saved the bad event",
      Sets.newHashSet(AvroFlumeEvent.newBuilder()
        .setBody(ByteBuffer.wrap(badEvent.getBody()))
        .setHeaders(toUtf8Map(badEvent.getHeaders()))
        .build()),
      read(Datasets.load(ERROR_DATASET_URI, AvroFlumeEvent.class)));
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Override
public void tearDown() {
  super.tearDown();
  String uri = "dataset:file:" + getTablePath();
  if (Datasets.exists(uri)) {
    Datasets.delete(uri);
  }
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
/**
 * Merges a dataset into this.
 */
public void mergeDataset(String uri) {
  FileSystemDataset<GenericRecord> update = Datasets.load(uri);
  if (dataset instanceof FileSystemDataset) {
    ((FileSystemDataset<GenericRecord>) dataset).merge(update);
    // And let's completely drop the temporary dataset
    Datasets.delete(uri);
  } else {
    throw new SqoopException(
        KiteConnectorError.GENERIC_KITE_CONNECTOR_0000, uri);
  }
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Delete the users dataset
  boolean success = Datasets.delete("dataset:hdfs:/tmp/data/users");

  return success ? 0 : 1;
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Delete the users dataset
  boolean success = Datasets.delete("dataset:hive?dataset=users");

  return success ? 0 : 1;
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  // Delete the products dataset
  boolean success = Datasets.delete("dataset:hdfs:/tmp/data/products");

  return success ? 0 : 1;
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {

  // Drop the events dataset
  boolean success = Datasets.delete("dataset:hive:/tmp/data/default/events");

  return success ? 0 : 1;
}
 

import org.kitesdk.data.Datasets; //导入方法依赖的package包/类
@After
public void teardown() {
  Datasets.delete(FILE_DATASET_URI);
}