本文整理汇总了Java中io.confluent.connect.hdfs.partitioner.FieldPartitioner类的典型用法代码示例。如果您正苦于以下问题：Java FieldPartitioner类的具体用法？Java FieldPartitioner怎么用？Java FieldPartitioner使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
FieldPartitioner类属于io.confluent.connect.hdfs.partitioner包，在下文中一共展示了FieldPartitioner类的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: visible
import io.confluent.connect.hdfs.partitioner.FieldPartitioner; //导入依赖的package包/类
@Override
public boolean visible(String name, Map<String, Object> connectorConfigs) {
String partitionerName = (String) connectorConfigs.get(PARTITIONER_CLASS_CONFIG);
try {
@SuppressWarnings("unchecked")
Class<? extends Partitioner> partitioner = (Class<? extends Partitioner>) Class.forName(partitionerName);
if (classNameEquals(partitionerName, DefaultPartitioner.class)) {
return false;
} else if (FieldPartitioner.class.isAssignableFrom(partitioner)) {
// subclass of FieldPartitioner
return name.equals(PARTITION_FIELD_NAME_CONFIG);
} else if (TimeBasedPartitioner.class.isAssignableFrom(partitioner)) {
// subclass of TimeBasedPartitioner
if (classNameEquals(partitionerName, DailyPartitioner.class) || classNameEquals(partitionerName, HourlyPartitioner.class)) {
return name.equals(LOCALE_CONFIG) || name.equals(TIMEZONE_CONFIG);
} else {
return name.equals(PARTITION_DURATION_MS_CONFIG) || name.equals(PATH_FORMAT_CONFIG) || name.equals(LOCALE_CONFIG) || name.equals(TIMEZONE_CONFIG);
}
} else {
throw new ConfigException("Not a valid partitioner class: " + partitionerName);
}
} catch (ClassNotFoundException e) {
throw new ConfigException("Partitioner class not found: " + partitionerName);
}
}
示例2: testWriteRecordFieldPartitioner
import io.confluent.connect.hdfs.partitioner.FieldPartitioner; //导入依赖的package包/类
@Test
public void testWriteRecordFieldPartitioner() throws Exception {
Map<String, Object> config = createConfig();
Partitioner partitioner = new FieldPartitioner();
partitioner.configure(config);
String partitionField = (String) config.get(HdfsSinkConnectorConfig.PARTITION_FIELD_NAME_CONFIG);
TopicPartitionWriter topicPartitionWriter = new TopicPartitionWriter(
TOPIC_PARTITION, storage, writerProvider, partitioner, connectorConfig, context, avroData);
String key = "key";
Schema schema = createSchema();
Struct[] records = createRecords(schema);
Collection<SinkRecord> sinkRecords = createSinkRecords(records, key, schema);
for (SinkRecord record : sinkRecords) {
topicPartitionWriter.buffer(record);
}
topicPartitionWriter.recover();
topicPartitionWriter.write();
topicPartitionWriter.close();
String directory1 = partitioner.generatePartitionedPath(TOPIC, partitionField + "=" + String.valueOf(16));
String directory2 = partitioner.generatePartitionedPath(TOPIC, partitionField + "=" + String.valueOf(17));
String directory3 = partitioner.generatePartitionedPath(TOPIC, partitionField + "=" + String.valueOf(18));
Set<Path> expectedFiles = new HashSet<>();
expectedFiles.add(new Path(FileUtils.committedFileName(url, topicsDir, directory1, TOPIC_PARTITION, 0, 2, extension, ZERO_PAD_FMT)));
expectedFiles.add(new Path(FileUtils.committedFileName(url, topicsDir, directory2, TOPIC_PARTITION, 3, 5, extension, ZERO_PAD_FMT)));
expectedFiles.add(new Path(FileUtils.committedFileName(url, topicsDir, directory3, TOPIC_PARTITION, 6, 8, extension, ZERO_PAD_FMT)));
verify(expectedFiles, records, schema);
}
示例3: testHiveIntegrationFieldPartitionerAvro
import io.confluent.connect.hdfs.partitioner.FieldPartitioner; //导入依赖的package包/类
@Test
public void testHiveIntegrationFieldPartitionerAvro() throws Exception {
Map<String, String> props = createProps();
props.put(HdfsSinkConnectorConfig.HIVE_INTEGRATION_CONFIG, "true");
props.put(HdfsSinkConnectorConfig.PARTITIONER_CLASS_CONFIG, FieldPartitioner.class.getName());
props.put(HdfsSinkConnectorConfig.PARTITION_FIELD_NAME_CONFIG, "int");
HdfsSinkConnectorConfig config = new HdfsSinkConnectorConfig(props);
DataWriter hdfsWriter = new DataWriter(config, context, avroData);
String key = "key";
Schema schema = createSchema();
Struct[] records = createRecords(schema);
ArrayList<SinkRecord> sinkRecords = new ArrayList<>();
long offset = 0;
for (Struct record : records) {
for (long count = 0; count < 3; count++) {
SinkRecord sinkRecord = new SinkRecord(TOPIC, PARTITION, Schema.STRING_SCHEMA, key, schema, record,
offset + count);
sinkRecords.add(sinkRecord);
}
offset = offset + 3;
}
hdfsWriter.write(sinkRecords);
hdfsWriter.close(assignment);
hdfsWriter.stop();
Table table = hiveMetaStore.getTable(hiveDatabase, TOPIC);
List<String> expectedColumnNames = new ArrayList<>();
for (Field field: schema.fields()) {
expectedColumnNames.add(field.name());
}
List<String> actualColumnNames = new ArrayList<>();
for (FieldSchema column: table.getSd().getCols()) {
actualColumnNames.add(column.getName());
}
assertEquals(expectedColumnNames, actualColumnNames);
String partitionFieldName = config.getString(HdfsSinkConnectorConfig.PARTITION_FIELD_NAME_CONFIG);
String directory1 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(16);
String directory2 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(17);
String directory3 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(18);
List<String> expectedPartitions = new ArrayList<>();
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory1));
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory2));
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory3));
List<String> partitions = hiveMetaStore.listPartitions(hiveDatabase, TOPIC, (short)-1);
assertEquals(expectedPartitions, partitions);
ArrayList<String[]> expectedResult = new ArrayList<>();
for (int i = 16; i <= 18; ++i) {
String[] part = {"true", String.valueOf(i), "12", "12.2", "12.2"};
for (int j = 0; j < 3; ++j) {
expectedResult.add(part);
}
}
String result = HiveTestUtils.runHive(hiveExec, "SELECT * FROM " + TOPIC);
String[] rows = result.split("\n");
assertEquals(9, rows.length);
for (int i = 0; i < rows.length; ++i) {
String[] parts = HiveTestUtils.parseOutput(rows[i]);
for (int j = 0; j < expectedResult.get(i).length; ++j) {
assertEquals(expectedResult.get(i)[j], parts[j]);
}
}
}
示例4: testHiveIntegrationFieldPartitionerParquet
import io.confluent.connect.hdfs.partitioner.FieldPartitioner; //导入依赖的package包/类
@Test
public void testHiveIntegrationFieldPartitionerParquet() throws Exception {
Map<String, String> props = createProps();
props.put(HdfsSinkConnectorConfig.HIVE_INTEGRATION_CONFIG, "true");
props.put(HdfsSinkConnectorConfig.PARTITIONER_CLASS_CONFIG, FieldPartitioner.class.getName());
props.put(HdfsSinkConnectorConfig.PARTITION_FIELD_NAME_CONFIG, "int");
HdfsSinkConnectorConfig config = new HdfsSinkConnectorConfig(props);
DataWriter hdfsWriter = new DataWriter(config, context, avroData);
String key = "key";
Schema schema = createSchema();
Struct[] records = createRecords(schema);
ArrayList<SinkRecord> sinkRecords = new ArrayList<>();
long offset = 0;
for (Struct record : records) {
for (long count = 0; count < 3; count++) {
SinkRecord sinkRecord = new SinkRecord(TOPIC, PARTITION, Schema.STRING_SCHEMA, key, schema, record,
offset + count);
sinkRecords.add(sinkRecord);
}
offset = offset + 3;
}
hdfsWriter.write(sinkRecords);
hdfsWriter.close(assignment);
hdfsWriter.stop();
Table table = hiveMetaStore.getTable(hiveDatabase, TOPIC);
List<String> expectedColumnNames = new ArrayList<>();
for (Field field: schema.fields()) {
expectedColumnNames.add(field.name());
}
List<String> actualColumnNames = new ArrayList<>();
for (FieldSchema column: table.getSd().getCols()) {
actualColumnNames.add(column.getName());
}
assertEquals(expectedColumnNames, actualColumnNames);
String partitionFieldName = config.getString(HdfsSinkConnectorConfig.PARTITION_FIELD_NAME_CONFIG);
String directory1 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(16);
String directory2 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(17);
String directory3 = TOPIC + "/" + partitionFieldName + "=" + String.valueOf(18);
List<String> expectedPartitions = new ArrayList<>();
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory1));
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory2));
expectedPartitions.add(FileUtils.directoryName(url, topicsDir, directory3));
List<String> partitions = hiveMetaStore.listPartitions(hiveDatabase, TOPIC, (short)-1);
assertEquals(expectedPartitions, partitions);
ArrayList<String[]> expectedResult = new ArrayList<>();
for (int i = 16; i <= 18; ++i) {
String[] part = {"true", String.valueOf(i), "12", "12.2", "12.2"};
for (int j = 0; j < 3; ++j) {
expectedResult.add(part);
}
}
String result = HiveTestUtils.runHive(hiveExec, "SELECT * FROM " + TOPIC);
String[] rows = result.split("\n");
assertEquals(9, rows.length);
for (int i = 0; i < rows.length; ++i) {
String[] parts = HiveTestUtils.parseOutput(rows[i]);
for (int j = 0; j < expectedResult.get(i).length; ++j) {
assertEquals(expectedResult.get(i)[j], parts[j]);
}
}
}