Java DatasetDescriptor.getLocation方法代码示例

import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
/**
 * Creates, if necessary, the given the location for {@code descriptor}.
 *
 * @param conf A Configuration
 * @param descriptor A DatasetDescriptor
 */
static void ensureExists(
    DatasetDescriptor descriptor, Configuration conf) {
  Preconditions.checkArgument(descriptor.getLocation() != null,
      "Cannot get FileSystem for a descriptor with no location");
  final Path dataPath = new Path(descriptor.getLocation());

  final FileSystem fs = fsForPath(dataPath, conf);

  try {
    if (!fs.exists(dataPath)) {
      fs.mkdirs(dataPath);
    }
  } catch (IOException ex) {
    throw new DatasetRepositoryException("Cannot access data location", ex);
  }
}
 

import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
@Test
public void testDeleteRemovesMetadataFiles() throws IOException {
  testCreateMetadataFiles();

  DatasetDescriptor loaded = provider.load(NAME);

  Path namedDirectory = new Path(loaded.getLocation());
  Path metadataDirectory = new Path(namedDirectory, ".metadata");
  Path propertiesFile = new Path(metadataDirectory, "descriptor.properties");
  Path schemaFile = new Path(metadataDirectory, "schema.avsc");

  boolean result = provider.delete(NAME);
  Assert.assertTrue(result);
  Assert.assertFalse("Descriptor properties file should not exist",
      fileSystem.exists(propertiesFile));
  Assert.assertFalse("Descriptor schema file should not exist",
      fileSystem.exists(schemaFile));
  Assert.assertFalse("Metadata directory should not exist",
      fileSystem.exists(metadataDirectory));
  Assert.assertTrue("Named directory should still exist for name:" + NAME,
      fileSystem.exists(namedDirectory));
}
 

import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
@Override
public <E> Dataset<E> create(String name, DatasetDescriptor descriptor) {

  Preconditions.checkArgument(name != null, "Name can not be null");
  Preconditions.checkArgument(descriptor != null,
      "Descriptor can not be null");
  Preconditions.checkArgument(descriptor.getLocation() == null,
      "Descriptor location cannot be set; " +
      "it is assigned by the MetadataProvider");

  final DatasetDescriptor newDescriptor = metadataProvider
      .create(name, descriptor);

  final URI location = newDescriptor.getLocation();
  if (location == null) {
    throw new DatasetRepositoryException(
        "[BUG] MetadataProvider did not assign a location to dataset:" +
        name);
  }

  ensureExists(newDescriptor, conf);

  logger.debug("Created dataset:{} schema:{} datasetPath:{}", new Object[] {
      name, newDescriptor.getSchema(), location.toString() });

  return new FileSystemDataset.Builder()
      .name(name)
      .configuration(conf)
      .descriptor(newDescriptor)
      .partitionKey(newDescriptor.isPartitioned() ?
          com.cloudera.cdk.data.impl.Accessor.getDefault().newPartitionKey() :
          null)
      .build();
}
 

import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
@Override
public DatasetDescriptor create(String name, DatasetDescriptor descriptor) {
  Preconditions.checkArgument(name != null, "Name cannot be null");
  Preconditions.checkArgument(descriptor != null,
      "Descriptor cannot be null");

  logger.debug("Saving dataset metadata name:{} descriptor:{}", name,
      descriptor);

  final Path dataLocation;

  // If the descriptor has a location, use it.
  if (descriptor.getLocation() != null) {
    dataLocation = new Path(descriptor.getLocation());
  } else {
    dataLocation = pathForDataset(name);
  }

  final Path metadataLocation = pathForMetadata(name);

  // get a DatasetDescriptor with the location set
  DatasetDescriptor newDescriptor = new DatasetDescriptor.Builder(descriptor)
      .location(dataLocation)
      .build();

  try {
    if (rootFileSystem.exists(metadataLocation)) {
      throw new DatasetExistsException(
          "Descriptor directory:" + metadataLocation + " already exists");
    }
    // create the directory so that update can do the rest of the work
    rootFileSystem.mkdirs(metadataLocation);
  } catch (IOException e) {
    throw new MetadataProviderException(
        "Unable to create metadata directory:" + metadataLocation +
        " for dataset:" + name, e);
  }

  writeDescriptor(rootFileSystem, metadataLocation, name, newDescriptor);

  return newDescriptor;
}
 

import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
@Override
public <E> Dataset<E> update(String name, DatasetDescriptor descriptor) {
  Preconditions.checkArgument(name != null, "Dataset name cannot be null");
  Preconditions.checkArgument(descriptor != null,
      "DatasetDescriptro cannot be null");

  DatasetDescriptor oldDescriptor = metadataProvider.load(name);

  // oldDescriptor is valid if load didn't throw NoSuchDatasetException

  if (!oldDescriptor.getFormat().equals(descriptor.getFormat())) {
    throw new DatasetRepositoryException("Cannot change dataset format from " +
        oldDescriptor.getFormat() + " to " + descriptor.getFormat());
  }

  final URI oldLocation = oldDescriptor.getLocation();
  if ((oldLocation != null) && !(oldLocation.equals(descriptor.getLocation()))) {
    throw new DatasetRepositoryException(
        "Cannot change the dataset's location");
  }

  if (oldDescriptor.isPartitioned() != descriptor.isPartitioned()) {
    throw new DatasetRepositoryException("Cannot change an unpartitioned dataset to " +
        " partitioned or vice versa.");
  } else if (oldDescriptor.isPartitioned() && descriptor.isPartitioned() &&
      !oldDescriptor.getPartitionStrategy().equals(descriptor.getPartitionStrategy())) {
    throw new DatasetRepositoryException("Cannot change partition strategy from " +
        oldDescriptor.getPartitionStrategy() + " to " + descriptor.getPartitionStrategy());
  }

  // check can read records written with old schema using new schema
  final Schema oldSchema = oldDescriptor.getSchema();
  final Schema newSchema = descriptor.getSchema();
  if (!SchemaValidationUtil.canRead(oldSchema, newSchema)) {
    throw new IncompatibleSchemaException("New schema cannot read data " +
        "written using " +
        "old schema. New schema: " + newSchema.toString(true) + "\nOld schema: " +
        oldSchema.toString(true));
  }

  final DatasetDescriptor updatedDescriptor = metadataProvider
      .update(name, descriptor);

  logger.debug("Updated dataset:{} schema:{} datasetPath:{}", new Object[] {
      name, updatedDescriptor.getSchema(),
      updatedDescriptor.getLocation().toString() });

  return new FileSystemDataset.Builder()
      .name(name)
      .configuration(conf)
      .descriptor(updatedDescriptor)
      .partitionKey(updatedDescriptor.isPartitioned() ?
          com.cloudera.cdk.data.impl.Accessor.getDefault().newPartitionKey() :
          null)
      .build();
}