Java HiveStorageHandler类代码示例

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; //导入依赖的package包/类
/**
 * Utility method which gets table or partition {@link InputFormat} class. First it
 * tries to get the class name from given StorageDescriptor object. If it doesn't contain it tries to get it from
 * StorageHandler class set in table properties. If not found throws an exception.
 * @param job {@link JobConf} instance needed incase the table is StorageHandler based table.
 * @param sd {@link StorageDescriptor} instance of currently reading partition or table (for non-partitioned tables).
 * @param table Table object
 * @throws Exception
 */
public static Class<? extends InputFormat<?, ?>> getInputFormatClass(final JobConf job, final StorageDescriptor sd,
    final Table table) throws Exception {
  final String inputFormatName = sd.getInputFormat();
  if (Strings.isNullOrEmpty(inputFormatName)) {
    final String storageHandlerClass = table.getParameters().get(META_TABLE_STORAGE);
    if (Strings.isNullOrEmpty(storageHandlerClass)) {
      throw new ExecutionSetupException("Unable to get Hive table InputFormat class. There is neither " +
          "InputFormat class explicitly specified nor StorageHandler class");
    }
    final HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(job, storageHandlerClass);
    return (Class<? extends InputFormat<?, ?>>) storageHandler.getInputFormatClass();
  } else {
    return (Class<? extends InputFormat<?, ?>>) Class.forName(inputFormatName) ;
  }
}
 

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; //导入依赖的package包/类
private static PartInfo extractPartInfo(HCatSchema schema, StorageDescriptor sd,
                    Map<String, String> parameters, Configuration conf,
                    InputJobInfo inputJobInfo) throws IOException {

  StorerInfo storerInfo = InternalUtil.extractStorerInfo(sd, parameters);

  Properties hcatProperties = new Properties();
  HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(conf, storerInfo);

  // copy the properties from storageHandler to jobProperties
  Map<String, String> jobProperties =
      HCatRSUtil.getInputJobProperties(storageHandler, inputJobInfo);

  for (String key : parameters.keySet()) {
    hcatProperties.put(key, parameters.get(key));
  }
  // FIXME
  // Bloating partinfo with inputJobInfo is not good
  return new PartInfo(schema, storageHandler, sd.getLocation(),
    hcatProperties, jobProperties, inputJobInfo.getTableInfo());
}
 

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; //导入依赖的package包/类
private IMetaStoreClient createMetaStoreClient() throws MetaException {
  HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
    @Override
    public HiveMetaHook getHook(Table tbl) throws MetaException {
      if (tbl == null) {
        return null;
      }

      try {
        HiveStorageHandler storageHandler =
            HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE));
        return storageHandler == null ? null : storageHandler.getMetaHook();
      } catch (HiveException e) {
        LOG.error(e.toString());
        throw new MetaException("Failed to get storage handler: " + e);
      }
    }
  };

  return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
}
 

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; //导入依赖的package包/类
/**
 * Instantiates a new hcat partition info.
 * @param partitionSchema the partition schema
 * @param storageHandler the storage handler
 * @param location the location
 * @param hcatProperties hcat-specific properties at the partition
 * @param jobProperties the job properties
 * @param tableInfo the table information
 */
public PartInfo(HCatSchema partitionSchema, HiveStorageHandler storageHandler,
        String location, Properties hcatProperties,
        Map<String, String> jobProperties, HCatTableInfo tableInfo) {
  this.partitionSchema = partitionSchema;
  this.location = location;
  this.hcatProperties = hcatProperties;
  this.jobProperties = jobProperties;
  this.tableInfo = tableInfo;

  this.storageHandlerClassName = storageHandler.getClass().getName();
  this.inputFormatClassName = storageHandler.getInputFormatClass().getName();
  this.serdeClassName = storageHandler.getSerDeClass().getName();
  this.outputFormatClassName = storageHandler.getOutputFormatClass().getName();
}
 

import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; //导入依赖的package包/类
public static Map<String, String> getInputJobProperties(
    HiveStorageHandler storageHandler, InputJobInfo inputJobInfo) {
  Properties props = inputJobInfo.getTableInfo().getStorerInfo().getProperties();
  props.put(serdeConstants.SERIALIZATION_LIB,storageHandler.getSerDeClass().getName());
  TableDesc tableDesc = new TableDesc(storageHandler.getInputFormatClass(),
          storageHandler.getOutputFormatClass(), props);
  if (tableDesc.getJobProperties() == null) {
    tableDesc.setJobProperties(new HashMap<String, String>());
  }

  Properties mytableProperties = tableDesc.getProperties();
  mytableProperties.setProperty(
      org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,
      inputJobInfo.getDatabaseName() + "." + inputJobInfo.getTableName());

  Map<String, String> jobProperties = new HashMap<String, String>();
  try {
    tableDesc.getJobProperties().put(
            HCatConstants.HCAT_KEY_JOB_INFO,
            HCatRSUtil.serialize(inputJobInfo));

    storageHandler.configureInputJobProperties(tableDesc, jobProperties);
  } catch (IOException e) {
    throw new IllegalStateException(
            "Failed to configure StorageHandler", e);
  }

  return jobProperties;
}