Java SerDeUtils类代码示例

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
/**
 * Serialize a object into bytes.
 *
 * @param foi object inspector
 * @param obj object to be serialized
 * @param useJsonSerialize true to use json serialization
 * @return object in serialized bytes
 * @throws IOException when error happens
 */
protected byte[] serializeToBytes(ObjectInspector foi, ObjectInspector doi, Object obj, boolean useJsonSerialize) throws IOException {
    serializeStream.reset();
    boolean isNotNull;
    if (!foi.getCategory().equals(Category.PRIMITIVE)
            && useJsonSerialize) {
        isNotNull = serialize(SerDeUtils.getJSONString(obj, foi),
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, doi, 1);
    } else {
        isNotNull = serialize(obj, foi, doi, 1);
    }
    if (!isNotNull) {
        return null;
    }
    byte[] key = new byte[serializeStream.getCount()];
    System.arraycopy(serializeStream.getData(), 0, key, 0, serializeStream.getCount());

    return key;
}
 

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
/**
 * Serialize a object into bytes.
 * @param foi object inspector
 * @param decalred output object inspector
 * @param obj object to be serialized
 * @param useJsonSerialize true to use json serialization
 * @return object in serialized bytes
 * @throws IOException when error happens
 */
protected byte[] serializeToBytes(ObjectInspector foi, ObjectInspector doi, Object obj, boolean useJsonSerialize) throws IOException {
  serializeStream.reset();
  boolean isNotNull;
  if (!foi.getCategory().equals(Category.PRIMITIVE)
              && useJsonSerialize) {
    isNotNull = serialize(SerDeUtils.getJSONString(obj, foi),
                PrimitiveObjectInspectorFactory.javaStringObjectInspector, doi, 1);
  } else {
    isNotNull = serialize(obj, foi, doi, 1);
  }
  if (!isNotNull) {
    return null;
  }
  byte[] key = new byte[serializeStream.getCount()];
  System.arraycopy(serializeStream.getData(), 0, key, 0, serializeStream.getCount());

  return key;
}
 

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
/**
 * Given an object and object inspector pair, traverse the object
 * and generate a Text representation of the object.
 */
@Override
public Writable serialize(Object obj, ObjectInspector objInspector)
  throws SerDeException {
  StringBuilder sb = new StringBuilder();
  try {

    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> structFields = soi.getAllStructFieldRefs();
    assert (columnNames.size() == structFields.size());
    if (obj == null) {
      sb.append("null");
    } else {
      sb.append(SerDeUtils.LBRACE);
      for (int i = 0; i < structFields.size(); i++) {
        if (i > 0) {
          sb.append(SerDeUtils.COMMA);
        }
        appendWithQuotes(sb, columnNames.get(i));
        sb.append(SerDeUtils.COLON);
        buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
          structFields.get(i).getFieldObjectInspector());
      }
      sb.append(SerDeUtils.RBRACE);
    }

  } catch (IOException e) {
    LOG.warn("Error generating json text from object.", e);
    throw new SerDeException(e);
  }
  return new Text(sb.toString());
}
 

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
private static Path createDummyFileForEmptyPartition(Path path, JobConf job, MapWork work,
    Path hiveScratchDir, String alias, int sequenceNumber)
        throws Exception {

  String strPath = path.toString();

  // The input file does not exist, replace it by a empty file
  PartitionDesc partDesc = work.getPathToPartitionInfo().get(strPath);
  if (partDesc.getTableDesc().isNonNative()) {
    // if this isn't a hive table we can't create an empty file for it.
    return path;
  }

  Properties props = SerDeUtils.createOverlayedProperties(
      partDesc.getTableDesc().getProperties(), partDesc.getProperties());
  HiveOutputFormat outFileFormat = HiveFileFormatUtils.getHiveOutputFormat(job, partDesc);

  boolean oneRow = partDesc.getInputFileFormatClass() == OneNullRowInputFormat.class;

  Path newPath = createEmptyFile(hiveScratchDir, outFileFormat, job,
      sequenceNumber, props, oneRow);

  if (LOG.isInfoEnabled()) {
    LOG.info("Changed input file " + strPath + " to empty file " + newPath);
  }

  // update the work
  String strNewPath = newPath.toString();

  LinkedHashMap<String, ArrayList<String>> pathToAliases = work.getPathToAliases();
  pathToAliases.put(strNewPath, pathToAliases.get(strPath));
  pathToAliases.remove(strPath);

  work.setPathToAliases(pathToAliases);

  LinkedHashMap<String, PartitionDesc> pathToPartitionInfo = work.getPathToPartitionInfo();
  pathToPartitionInfo.put(strNewPath, pathToPartitionInfo.get(strPath));
  pathToPartitionInfo.remove(strPath);
  work.setPathToPartitionInfo(pathToPartitionInfo);

  return newPath;
}
 

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
private static StringBuilder appendWithQuotes(StringBuilder sb, String value) {
  return sb == null ? null : sb.append(SerDeUtils.QUOTE).append(value).append(SerDeUtils.QUOTE);
}
 

import org.apache.hadoop.hive.serde2.SerDeUtils; //导入依赖的package包/类
/**
 * getDeserializer
 *
 * Get the Deserializer for a table given its name and properties.
 *
 * @param conf
 *          hadoop config
 * @param schema
 *          the properties to use to instantiate the deserializer
 * @return
 *   Returns instantiated deserializer by looking up class name of deserializer stored in passed
 *   in properties. Also, initializes the deserializer with schema stored in passed in properties.
 * @exception MetaException
 *              if any problems instantiating the Deserializer
 *
 *              todo - this should move somewhere into serde.jar
 *
 */
static public Deserializer getDeserializer(Configuration conf,
    Properties schema) throws MetaException {
  String lib = schema
      .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
  try {
    Deserializer deserializer = SerDeUtils.lookupDeserializer(lib);
    (deserializer).initialize(conf, schema);
    return deserializer;
  } catch (Exception e) {
    log.error(e, "error in initSerDe: %s %s", e.getClass().getName(), e.getMessage());
    MetaStoreUtils.printStackTrace(e);
    throw new MetaException(e.getClass().getName() + " " + e.getMessage());
  }
}