Java LazySimpleSerDe类代码示例

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
public static LazySimpleSerDe getLineSerde(@Nonnull final PrimitiveObjectInspector... OIs)
        throws SerDeException {
    if (OIs.length == 0) {
        throw new IllegalArgumentException("OIs must be specified");
    }
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();

    StringBuilder columnNames = new StringBuilder();
    StringBuilder columnTypes = new StringBuilder();
    for (int i = 0; i < OIs.length; i++) {
        columnNames.append('c').append(i + 1).append(',');
        columnTypes.append(OIs[i].getTypeName()).append(',');
    }
    columnNames.deleteCharAt(columnNames.length() - 1);
    columnTypes.deleteCharAt(columnTypes.length() - 1);

    tbl.setProperty("columns", columnNames.toString());
    tbl.setProperty("columns.types", columnTypes.toString());
    serde.initialize(conf, tbl);
    return serde;
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test text file with zip formatter.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileWithZipFormatter() throws IOException {
  Configuration conf = new Configuration();
  setConf(conf);
  conf.set("test.partfile.dir", partFileTextDir.toString());
  conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
  conf.set(LensConfConstants.QUERY_OUTPUT_HEADER, "");
  conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
  conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
  testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
    getMockedResultSetWithoutComma());
  // validate rows
  List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
  System.out.println("Actual rows:" + actual);
  Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
public static TableDesc getTableDesc(String cols, String colTypes) {
  return (new TableDesc(SequenceFileInputFormat.class,
      HiveSequenceFileOutputFormat.class, Utilities.makeProperties(
      serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode,
      serdeConstants.LIST_COLUMNS, cols,
      serdeConstants.LIST_COLUMN_TYPES, colTypes,
      serdeConstants.SERIALIZATION_LIB,LazySimpleSerDe.class.getName())));
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
public static LazySimpleSerDe getKeyValueLineSerde(
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI) throws SerDeException {
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty("columns", "key,value");
    tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
    serde.initialize(conf, tbl);
    return serde;
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
private static void loadValues(Object2ObjectMap<Object, Object> map, File file,
        PrimitiveObjectInspector keyOI, PrimitiveObjectInspector valueOI) throws IOException,
        SerDeException {
    if (!file.exists()) {
        return;
    }
    if (!file.getName().endsWith(".crc")) {
        if (file.isDirectory()) {
            for (File f : file.listFiles()) {
                loadValues(map, f, keyOI, valueOI);
            }
        } else {
            LazySimpleSerDe serde = HiveUtils.getKeyValueLineSerde(keyOI, valueOI);
            StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
            StructField keyRef = lineOI.getStructFieldRef("key");
            StructField valueRef = lineOI.getStructFieldRef("value");
            PrimitiveObjectInspector keyRefOI = (PrimitiveObjectInspector) keyRef.getFieldObjectInspector();
            PrimitiveObjectInspector valueRefOI = (PrimitiveObjectInspector) valueRef.getFieldObjectInspector();

            BufferedReader reader = null;
            try {
                reader = HadoopUtils.getBufferedReader(file);
                String line;
                while ((line = reader.readLine()) != null) {
                    Text lineText = new Text(line);
                    Object lineObj = serde.deserialize(lineText);
                    List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
                    Object f0 = fields.get(0);
                    Object f1 = fields.get(1);
                    Object k = keyRefOI.getPrimitiveJavaObject(f0);
                    Object v = valueRefOI.getPrimitiveWritableObject(valueRefOI.copyObject(f1));
                    map.put(k, v);
                }
            } finally {
                IOUtils.closeQuietly(reader);
            }
        }
    }
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
public static String getDataFormat(StorageDescriptor descriptor) {
  Preconditions.checkNotNull(descriptor);

  String serde = descriptor.getSerdeInfo().getSerializationLib();
  String inputFormat = descriptor.getInputFormat();

  if (LazySimpleSerDe.class.getName().equals(serde)) {
    if (TextInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.TEXT;
    } else if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinarySerDe.class.getName().equals(serde)) {
    if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinaryColumnarSerDe.class.getName().equals(serde) || ColumnarSerDe.class.getName().equals(serde)) {
    if (RCFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.RCFILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (ParquetHiveSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.PARQUET;
  } else if (AvroSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.AVRO;
  } else if (OrcSerde.class.getName().equals(serde)) {
    return BuiltinStorages.ORC;
  } else if (RegexSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.REGEX;
  } else {
    throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
  }
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test result formatter in memory result.
 *
 * @throws InterruptedException the interrupted exception
 * @throws IOException          Signals that an I/O exception has occurred.
 */
@Test(dataProvider = "mediaTypeData")
public void testResultFormatterInMemoryResult(MediaType mt) throws InterruptedException, IOException {
  LensConf conf = new LensConf();
  conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false");
  conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, null, mt);

  queryService.conf.set(LensConfConstants.RESULT_FS_READ_URL, "filereadurl://");
  testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, "filereadurl://", mt);
  queryService.conf.unset(LensConfConstants.RESULT_FS_READ_URL);
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test serde.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testSerde() throws IOException {
  Configuration conf = new Configuration();
  conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
  conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
    getMockedResultSetWithoutComma());
  validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());

  // validate rows
  Assert.assertEquals(readFinalOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
    getExpectedTextRowsWithoutComma());
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test compression with custom serde.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testCompressionWithCustomSerde() throws IOException {
  Configuration conf = new Configuration();
  conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
  conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  conf.setBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION, true);
  testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt.gz",
    getMockedResultSetWithoutComma());
  validateSerde(LazySimpleSerDe.class.getCanonicalName(), Text.class.getCanonicalName());
  // validate rows
  Assert.assertEquals(readCompressedFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
    getExpectedTextRowsWithoutComma());
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test text file with zip formatter.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileWithZipFormatter() throws IOException {
  Configuration conf = new Configuration();
  setConf(conf);
  conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
  conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
  conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
  testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
    getMockedResultSetWithoutComma());
  // validate rows
  List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
  Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
/**
 * Test text file with serde header.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileWithSerdeHeader() throws IOException {
  Configuration conf = new Configuration();
  setConf(conf);
  conf.set("test.partfile.dir", partFileTextDir.toString());
  conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
  conf.set(LensConfConstants.QUERY_OUTPUT_HEADER, "");
  conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
  testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".txt",
    getMockedResultSetWithoutComma());
  // validate rows
  Assert.assertEquals(readFinalOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8"),
    getExpectedTextRowsWithoutComma());
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
private void initAccumuloSerdeParameters(Configuration conf, Properties properties)
        throws SerDeException{
    String colMapping = properties.getProperty(COLUMN_MAPPINGS);
    String colTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    String name = getClass().getName();
    fetchCols = AccumuloHiveUtils.parseColumnMapping(colMapping);
    if (colTypeProperty == null) {
        StringBuilder builder = new StringBuilder();
        for (String fetchCol : fetchCols) { //default to all string if no column type property.
            builder.append(serdeConstants.STRING_TYPE_NAME + ":");
        }
        int indexOfLastColon = builder.lastIndexOf(":");
        builder.replace(indexOfLastColon, indexOfLastColon+1, "");
        properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, builder.toString());
    }

    serDeParameters = LazySimpleSerDe.initSerdeParams(conf, properties, name);
    if (fetchCols.size() != serDeParameters.getColumnNames().size()) {
        throw new SerDeException(name + ": Hive table definition has "
                + serDeParameters.getColumnNames().size() +
                " elements while " + COLUMN_MAPPINGS + " has " +
                fetchCols.size() + " elements. " + printColumnMismatchTip(fetchCols.size(),
                serDeParameters.getColumnNames().size()));
    }

    if(log.isInfoEnabled())
        log.info("Serde initialized successfully for column mapping: " + colMapping);
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
public static Table createColumnsetSchema(String name, List<String> columns,
    List<String> partCols, Configuration conf) throws MetaException {

  if (columns == null) {
    throw new MetaException("columns not specified for table " + name);
  }

  Table tTable = new Table();
  tTable.setTableName(name);
  tTable.setSd(new StorageDescriptor());
  StorageDescriptor sd = tTable.getSd();
  sd.setSerdeInfo(new SerDeInfo());
  SerDeInfo serdeInfo = sd.getSerdeInfo();
  serdeInfo.setSerializationLib(LazySimpleSerDe.class.getName());
  serdeInfo.setParameters(new HashMap<String, String>());
  serdeInfo.getParameters().put(
      org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");

  List<FieldSchema> fields = new ArrayList<FieldSchema>();
  sd.setCols(fields);
  for (String col : columns) {
    FieldSchema field = new FieldSchema(col,
        org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME, "'default'");
    fields.add(field);
  }

  tTable.setPartitionKeys(new ArrayList<FieldSchema>());
  for (String partCol : partCols) {
    FieldSchema part = new FieldSchema();
    part.setName(partCol);
    part.setType(org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME); // default
                                                                           // partition
                                                                           // key
    tTable.getPartitionKeys().add(part);
  }
  sd.setNumBuckets(-1);
  return tTable;
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
Hive1SerDeParametersShim(Configuration configuration, Properties properties, String serDeName)
    throws SerDeException {
  this.realSerDeParameters = LazySimpleSerDe.initSerdeParams(configuration, properties,
      serDeName);
}
 

import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; //导入依赖的package包/类
private long loadPredictionModel(Map<Object, PredictionModel> label2model, File file,
        PrimitiveObjectInspector labelOI, PrimitiveObjectInspector featureOI,
        WritableFloatObjectInspector weightOI) throws IOException, SerDeException {
    long count = 0L;
    if (!file.exists()) {
        return count;
    }
    if (!file.getName().endsWith(".crc")) {
        if (file.isDirectory()) {
            for (File f : file.listFiles()) {
                count += loadPredictionModel(label2model, f, labelOI, featureOI, weightOI);
            }
        } else {
            LazySimpleSerDe serde = HiveUtils.getLineSerde(labelOI, featureOI, weightOI);
            StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
            StructField c1ref = lineOI.getStructFieldRef("c1");
            StructField c2ref = lineOI.getStructFieldRef("c2");
            StructField c3ref = lineOI.getStructFieldRef("c3");
            PrimitiveObjectInspector c1refOI = (PrimitiveObjectInspector) c1ref.getFieldObjectInspector();
            PrimitiveObjectInspector c2refOI = (PrimitiveObjectInspector) c2ref.getFieldObjectInspector();
            FloatObjectInspector c3refOI = (FloatObjectInspector) c3ref.getFieldObjectInspector();

            BufferedReader reader = null;
            try {
                reader = HadoopUtils.getBufferedReader(file);
                String line;
                while ((line = reader.readLine()) != null) {
                    count++;
                    Text lineText = new Text(line);
                    Object lineObj = serde.deserialize(lineText);
                    List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
                    Object f0 = fields.get(0);
                    Object f1 = fields.get(1);
                    Object f2 = fields.get(2);
                    if (f0 == null || f1 == null || f2 == null) {
                        continue; // avoid the case that key or value is null
                    }
                    Object label = c1refOI.getPrimitiveWritableObject(c1refOI.copyObject(f0));
                    PredictionModel model = label2model.get(label);
                    if (model == null) {
                        model = createModel();
                        label2model.put(label, model);
                    }
                    Object k = c2refOI.getPrimitiveWritableObject(c2refOI.copyObject(f1));
                    float v = c3refOI.get(f2);
                    model.set(k, new WeightValue(v, false));
                }
            } finally {
                IOUtils.closeQuietly(reader);
            }
        }
    }
    return count;
}