本文整理汇总了Java中org.apache.hcatalog.data.schema.HCatSchema类的典型用法代码示例。如果您正苦于以下问题:Java HCatSchema类的具体用法?Java HCatSchema怎么用?Java HCatSchema使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
HCatSchema类属于org.apache.hcatalog.data.schema包,在下文中一共展示了HCatSchema类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: generateHCatTableSchema
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private HCatSchema generateHCatTableSchema(ColumnGenerator... extraCols)
throws Exception {
List<HCatFieldSchema> hCatTblCols = new ArrayList<HCatFieldSchema>();
hCatTblCols.clear();
hCatTblCols.add(new HCatFieldSchema("id", HCatFieldSchema.Type.INT, ""));
hCatTblCols
.add(new HCatFieldSchema("msg", HCatFieldSchema.Type.STRING, ""));
for (ColumnGenerator gen : extraCols) {
if (gen.getKeyType() == KeyType.NOT_A_KEY) {
hCatTblCols
.add(new HCatFieldSchema(gen.getName(), gen.getHCatType(), ""));
}
}
HCatSchema hCatTblSchema = new HCatSchema(hCatTblCols);
return hCatTblSchema;
}
示例2: generateHCatDynamicPartitionSchema
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private HCatSchema generateHCatDynamicPartitionSchema(
ColumnGenerator... extraCols) throws Exception {
List<HCatFieldSchema> hCatPartCols = new ArrayList<HCatFieldSchema>();
hCatPartCols.clear();
boolean staticFound = false;
for (ColumnGenerator gen : extraCols) {
if (gen.getKeyType() != KeyType.NOT_A_KEY) {
if (gen.getKeyType() == KeyType.STATIC_KEY && !staticFound) {
staticFound = true;
continue;
}
hCatPartCols
.add(new HCatFieldSchema(gen.getName(), gen.getHCatType(), ""));
}
}
HCatSchema hCatPartSchema = new HCatSchema(hCatPartCols);
return hCatPartSchema;
}
示例3: generateHCatRecords
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private List<HCatRecord> generateHCatRecords(int numRecords,
HCatSchema hCatTblSchema, ColumnGenerator... extraCols) throws Exception {
List<HCatRecord> records = new ArrayList<HCatRecord>();
List<HCatFieldSchema> hCatTblCols = hCatTblSchema.getFields();
int size = hCatTblCols.size();
for (int i = 0; i < numRecords; ++i) {
DefaultHCatRecord record = new DefaultHCatRecord(size);
record.set(hCatTblCols.get(0).getName(), hCatTblSchema, i);
record.set(hCatTblCols.get(1).getName(), hCatTblSchema, "textfield" + i);
boolean staticFound = false;
int idx = 0;
for (int j = 0; j < extraCols.length; ++j) {
if (extraCols[j].getKeyType() == KeyType.STATIC_KEY
&& !staticFound) {
staticFound = true;
continue;
}
record.set(hCatTblCols.get(idx + 2).getName(), hCatTblSchema,
extraCols[j].getHCatValue(i));
++idx;
}
records.add(record);
}
return records;
}
示例4: createRecordReader
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
/**
* Create an {@link org.apache.hcatalog.mapreduce.HCatRecordReader}.
*
* @param split Input split
* @param schema Table schema
* @param taskContext Context
* @return Record reader
* @throws IOException
* @throws InterruptedException
*/
private RecordReader<WritableComparable, HCatRecord>
createRecordReader(InputSplit split,
HCatSchema schema,
TaskAttemptContext taskContext)
throws IOException, InterruptedException {
HCatSplit hcatSplit = HCatUtils.castToHCatSplit(split);
PartInfo partitionInfo = hcatSplit.getPartitionInfo();
JobContext jobContext = taskContext;
Configuration conf = jobContext.getConfiguration();
HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(
conf, partitionInfo);
JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext);
Map<String, String> jobProperties = partitionInfo.getJobProperties();
HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
Map<String, String> valuesNotInDataCols = getColValsNotInDataColumns(
schema, partitionInfo);
return HCatUtils.newHCatReader(storageHandler, valuesNotInDataCols);
}
示例5: getColValsNotInDataColumns
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
/**
* Get values for fields requested by output schema which will not be in the
* data.
*
* @param outputSchema Output schema
* @param partInfo Partition info
* @return Values not in data columns
*/
private static Map<String, String> getColValsNotInDataColumns(
HCatSchema outputSchema,
PartInfo partInfo) {
HCatSchema dataSchema = partInfo.getPartitionSchema();
Map<String, String> vals = new HashMap<String, String>();
for (String fieldName : outputSchema.getFieldNames()) {
if (dataSchema.getPosition(fieldName) == null) {
// this entry of output is not present in the output schema
// so, we first check the table schema to see if it is a part col
if (partInfo.getPartitionValues().containsKey(fieldName)) {
vals.put(fieldName, partInfo.getPartitionValues().get(fieldName));
} else {
vals.put(fieldName, null);
}
}
}
return vals;
}
示例6: extractPartInfo
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
/**
* Extract partition info.
*
* @param schema Table schema
* @param sd Storage descriptor
* @param parameters Parameters
* @param conf Configuration
* @param inputJobInfo Input job info
* @return Partition info
* @throws IOException
*/
private static PartInfo extractPartInfo(
HCatSchema schema, StorageDescriptor sd, Map<String, String> parameters,
Configuration conf, InputJobInfo inputJobInfo) throws IOException {
StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters);
Properties hcatProperties = new Properties();
HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(conf,
storerInfo);
// Copy the properties from storageHandler to jobProperties
Map<String, String> jobProperties =
HCatUtil.getInputJobProperties(storageHandler, inputJobInfo);
for (Map.Entry<String, String> param : parameters.entrySet()) {
hcatProperties.put(param.getKey(), param.getValue());
}
return new PartInfo(schema, storageHandler, sd.getLocation(),
hcatProperties, jobProperties, inputJobInfo.getTableInfo());
}
示例7: testSerializeStruct
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
/**
* Ensure we can do a basic serialization of a STRUCT
*
* @throws Exception
*/
@Test
public void testSerializeStruct() throws Exception {
HCatTable table = mock(HCatTable.class);
HCatSchema structSchema = new HCatSchema(Lists.newArrayList(
getSubFieldSchema("intField", HCatFieldSchema.Type.BIGINT),
getSubFieldSchema("strField", HCatFieldSchema.Type.STRING)));
HCatFieldSchema fieldSchema = new HCatFieldSchema("testField", HCatFieldSchema.Type.STRUCT, structSchema, "");
HiveSerializer serializer = new HiveSerializer(table);
Map<String, Object> testMap = Maps.newHashMap();
testMap.put("intField", 123L);
testMap.put("strField", "This is a string");
byte[] result = serializer.serializeStruct(fieldSchema, testMap, 1);
ByteArrayOutputStream expectedResult = new ByteArrayOutputStream();
expectedResult.write(Bytes.toBytes(123L));
expectedResult.write('\002');
expectedResult.write(Bytes.toBytes("This is a string"));
Assert.assertArrayEquals(expectedResult.toByteArray(), result);
}
示例8: testSerializeStructWithNullField
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
/**
* Ensure we throw an exception when a field within a STRUCT is null, as this is not allowed.
*
* @throws Exception
*/
@Test(expected=IllegalArgumentException.class)
public void testSerializeStructWithNullField() throws Exception {
HCatTable table = mock(HCatTable.class);
HCatSchema structSchema = new HCatSchema(Lists.newArrayList(
getSubFieldSchema("intField", HCatFieldSchema.Type.BIGINT),
getSubFieldSchema("strField", HCatFieldSchema.Type.STRING)));
HCatFieldSchema fieldSchema = new HCatFieldSchema("testField", HCatFieldSchema.Type.STRUCT, structSchema, "");
HiveSerializer serializer = new HiveSerializer(table);
Map<String, Object> testMap = Maps.newHashMap();
serializer.serializeStruct(fieldSchema, testMap, 1);
}
示例9: loadHCatTable
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private void loadHCatTable(HCatSchema hCatSchema, String table,
int count, ColumnGenerator... extraCols)
throws Exception {
Map<String, String> staticKeyMap = new HashMap<String, String>();
for (ColumnGenerator col : extraCols) {
if (col.getKeyType() == KeyType.STATIC_KEY) {
staticKeyMap.put(col.getName(), (String) col.getHCatValue(0));
}
}
loadHCatTable(null, table, staticKeyMap,
hCatSchema, generateHCatRecords(count, hCatSchema, extraCols));
}
示例10: generateHCatPartitionSchema
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private HCatSchema generateHCatPartitionSchema(ColumnGenerator... extraCols)
throws Exception {
List<HCatFieldSchema> hCatPartCols = new ArrayList<HCatFieldSchema>();
for (ColumnGenerator gen : extraCols) {
if (gen.getKeyType() != KeyType.NOT_A_KEY) {
hCatPartCols
.add(new HCatFieldSchema(gen.getName(), gen.getHCatType(), ""));
}
}
HCatSchema hCatPartSchema = new HCatSchema(hCatPartCols);
return hCatPartSchema;
}
示例11: generateHCatStaticPartitionSchema
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
private HCatSchema generateHCatStaticPartitionSchema(
ColumnGenerator... extraCols) throws Exception {
List<HCatFieldSchema> hCatPartCols = new ArrayList<HCatFieldSchema>();
hCatPartCols.clear();
for (ColumnGenerator gen : extraCols) {
if (gen.getKeyType() == KeyType.STATIC_KEY) {
hCatPartCols
.add(new HCatFieldSchema(gen.getName(), gen.getHCatType(), ""));
break;
}
}
HCatSchema hCatPartSchema = new HCatSchema(hCatPartCols);
return hCatPartSchema;
}
示例12: hCatRecordDump
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
public String hCatRecordDump(List<HCatRecord> recs,
HCatSchema schema) throws Exception {
List<String> fields = schema.getFieldNames();
int count = 0;
StringBuilder sb = new StringBuilder(1024);
for (HCatRecord rec : recs) {
sb.append("HCat Record : " + ++count).append('\n');
for (String field : fields) {
sb.append('\t').append(field).append('=');
sb.append(rec.get(field, schema)).append('\n');
sb.append("\n\n");
}
}
return sb.toString();
}
示例13: serializeStruct
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
protected byte[] serializeStruct(HCatFieldSchema field, Map structData, int level) throws IOException {
char separator = (char)separators[level];
ByteArrayOutputStream baos = new ByteArrayOutputStream();
HCatSchema structSchema = field.getStructSubSchema();
boolean first = true;
for(HCatFieldSchema structField: structSchema.getFields()) {
if(!first) {
baos.write(separator);
}
Object obj = structData.get(structField.getName());
if(obj == null) {
throw new IllegalArgumentException(
String.format("STRUCT types cannot have null members - field %s",
structField.getName()));
}
baos.write(serializeHiveType(structField, null, obj, level + 1));
first = false;
}
return baos.toByteArray();
}
示例14: testSerializeMap
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
@Test
public void testSerializeMap() throws Exception {
HCatTable table = mock(HCatTable.class);
HCatSchema mapSchema = new HCatSchema(Lists.newArrayList(
getSubFieldSchema("valueField", HCatFieldSchema.Type.BIGINT)));
HCatFieldSchema fieldSchema = new HCatFieldSchema("testField",
HCatFieldSchema.Type.MAP,
HCatFieldSchema.Type.STRING,
mapSchema, "");
HiveSerializer serializer = new HiveSerializer(table);
Map<String, Long> testMap = Maps.newLinkedHashMap();
testMap.put("field1", 123L);
testMap.put("field2", 456L);
byte[] result = serializer.serializeMap(fieldSchema, testMap, 1);
ByteArrayOutputStream expectedResult = new ByteArrayOutputStream();
expectedResult.write(Bytes.toBytes("field1"));
expectedResult.write('\003');
expectedResult.write(Bytes.toBytes(123L));
expectedResult.write('\002');
expectedResult.write(Bytes.toBytes("field2"));
expectedResult.write('\003');
expectedResult.write(Bytes.toBytes(456L));
byte[] expectedResultBytes = expectedResult.toByteArray();
Assert.assertArrayEquals(expectedResultBytes, result);
}
示例15: validateHCatTableAndTajoSchema
import org.apache.hcatalog.data.schema.HCatSchema; //导入依赖的package包/类
public static void validateHCatTableAndTajoSchema(HCatSchema tblSchema) throws CatalogException {
for (HCatFieldSchema hcatField : tblSchema.getFields()) {
validateHCatFieldAndTajoSchema(hcatField);
}
}