本文整理汇总了Java中com.cloudera.cdk.data.DatasetDescriptor.getPartitionStrategy方法的典型用法代码示例。如果您正苦于以下问题：Java DatasetDescriptor.getPartitionStrategy方法的具体用法？Java DatasetDescriptor.getPartitionStrategy怎么用？Java DatasetDescriptor.getPartitionStrategy使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类com.cloudera.cdk.data.DatasetDescriptor的用法示例。
在下文中一共展示了DatasetDescriptor.getPartitionStrategy方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: AbstractRangeView
import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
protected AbstractRangeView(Dataset<E> dataset) {
this.dataset = dataset;
final DatasetDescriptor descriptor = dataset.getDescriptor();
if (descriptor.isPartitioned()) {
this.range = new MarkerRange(new MarkerComparator(
descriptor.getPartitionStrategy()));
this.keys = new ThreadLocal<StorageKey>() {
@Override
protected StorageKey initialValue() {
return new StorageKey(descriptor.getPartitionStrategy());
}
};
} else {
// use UNDEFINED, which handles inappropriate calls to range methods
this.range = MarkerRange.UNDEFINED;
this.keys = null; // not used
}
}
示例2: PartitionedDatasetWriter
import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
public PartitionedDatasetWriter(FileSystemView<E> view) {
final DatasetDescriptor descriptor = view.getDataset().getDescriptor();
Preconditions.checkArgument(descriptor.isPartitioned(),
"Dataset " + view.getDataset() + " is not partitioned");
this.view = view;
this.partitionStrategy = descriptor.getPartitionStrategy();
this.maxWriters = Math.min(10, partitionStrategy.getCardinality());
this.state = ReaderWriterState.NEW;
this.reusedKey = new StorageKey(partitionStrategy);
}
示例3: FileSystemDataset
import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
FileSystemDataset(FileSystem fileSystem, Path directory, String name,
DatasetDescriptor descriptor) {
this.fileSystem = fileSystem;
this.directory = directory;
this.name = name;
this.descriptor = descriptor;
this.partitionStrategy =
descriptor.isPartitioned() ? descriptor.getPartitionStrategy() : null;
this.convert = new PathConversion();
this.unbounded = new FileSystemView<E>(this);
// remove this.partitionKey for 0.11.0
this.partitionKey = null;
}
示例4: update
import com.cloudera.cdk.data.DatasetDescriptor; //导入方法依赖的package包/类
@Override
public <E> Dataset<E> update(String name, DatasetDescriptor descriptor) {
Preconditions.checkArgument(name != null, "Dataset name cannot be null");
Preconditions.checkArgument(descriptor != null,
"DatasetDescriptro cannot be null");
DatasetDescriptor oldDescriptor = metadataProvider.load(name);
// oldDescriptor is valid if load didn't throw NoSuchDatasetException
if (!oldDescriptor.getFormat().equals(descriptor.getFormat())) {
throw new DatasetRepositoryException("Cannot change dataset format from " +
oldDescriptor.getFormat() + " to " + descriptor.getFormat());
}
final URI oldLocation = oldDescriptor.getLocation();
if ((oldLocation != null) && !(oldLocation.equals(descriptor.getLocation()))) {
throw new DatasetRepositoryException(
"Cannot change the dataset's location");
}
if (oldDescriptor.isPartitioned() != descriptor.isPartitioned()) {
throw new DatasetRepositoryException("Cannot change an unpartitioned dataset to " +
" partitioned or vice versa.");
} else if (oldDescriptor.isPartitioned() && descriptor.isPartitioned() &&
!oldDescriptor.getPartitionStrategy().equals(descriptor.getPartitionStrategy())) {
throw new DatasetRepositoryException("Cannot change partition strategy from " +
oldDescriptor.getPartitionStrategy() + " to " + descriptor.getPartitionStrategy());
}
// check can read records written with old schema using new schema
final Schema oldSchema = oldDescriptor.getSchema();
final Schema newSchema = descriptor.getSchema();
if (!SchemaValidationUtil.canRead(oldSchema, newSchema)) {
throw new IncompatibleSchemaException("New schema cannot read data " +
"written using " +
"old schema. New schema: " + newSchema.toString(true) + "\nOld schema: " +
oldSchema.toString(true));
}
final DatasetDescriptor updatedDescriptor = metadataProvider
.update(name, descriptor);
logger.debug("Updated dataset:{} schema:{} datasetPath:{}", new Object[] {
name, updatedDescriptor.getSchema(),
updatedDescriptor.getLocation().toString() });
return new FileSystemDataset.Builder()
.name(name)
.configuration(conf)
.descriptor(updatedDescriptor)
.partitionKey(updatedDescriptor.isPartitioned() ?
com.cloudera.cdk.data.impl.Accessor.getDefault().newPartitionKey() :
null)
.build();
}