Java Table.setTableType方法代码示例

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table constructAvroTable(String database, String tableName, Schema schema, Partitioner partitioner)
    throws HiveMetaStoreException {
  Table table = newTable(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(avroSerde);
  try {
    table.setInputFormatClass(avroInputFormat);
    table.setOutputFormatClass(avroOutputFormat);
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString());
  return table;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table constructParquetTable(String database, String tableName, Schema schema, Partitioner partitioner) throws HiveMetaStoreException {
  Table table = newTable(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(getHiveParquetSerde());
  try {
    table.setInputFormatClass(getHiveParquetInputFormat());
    table.setOutputFormatClass(getHiveParquetOutputFormat());
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  // convert copycat schema schema to Hive columns
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  return table;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table constructAvroTable(String database, String tableName, Schema schema, Partitioner partitioner)
    throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(avroSerde);
  try {
    table.setInputFormatClass(avroInputFormat);
    table.setOutputFormatClass(avroOutputFormat);
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString());
  return table;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table constructParquetTable(String database, String tableName, Schema schema, Partitioner partitioner) throws HiveMetaStoreException {
  Table table = new Table(database, tableName);
  table.setTableType(TableType.EXTERNAL_TABLE);
  table.getParameters().put("EXTERNAL", "TRUE");
  String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName);
  table.setDataLocation(new Path(tablePath));
  table.setSerializationLib(getHiveParquetSerde());
  try {
    table.setInputFormatClass(getHiveParquetInputFormat());
    table.setOutputFormatClass(getHiveParquetOutputFormat());
  } catch (HiveException e) {
    throw new HiveMetaStoreException("Cannot find input/output format:", e);
  }
  // convert copycat schema schema to Hive columns
  List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema);
  table.setFields(columns);
  table.setPartCols(partitioner.partitionFields());
  return table;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
/**
 * Creates the hive table.
 *
 * @param tableName the table name
 * @throws HiveException the hive exception
 */
public static void createHiveTable(String tableName, Map<String, String> parameters) throws HiveException {
  List<FieldSchema> columns = new ArrayList<FieldSchema>();
  columns.add(new FieldSchema("col1", "string", ""));
  List<FieldSchema> partCols = new ArrayList<FieldSchema>();
  partCols.add(new FieldSchema("pcol1", "string", ""));
  Map<String, String> params = new HashMap<String, String>();
  params.put("test.hive.table.prop", "tvalue");
  if (null != parameters && !parameters.isEmpty()) {
    params.putAll(parameters);
  }
  Table tbl = Hive.get().newTable(tableName);
  tbl.setTableType(TableType.MANAGED_TABLE);
  tbl.getTTable().getSd().setCols(columns);
  tbl.setPartCols(partCols);
  tbl.getTTable().getParameters().putAll(params);
  Hive.get().createTable(tbl);
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table createTestTable(String databaseName, String tableName) throws HiveException {
    Table table = new Table(databaseName, tableName);
    table.setInputFormatClass(TextInputFormat.class);
    table.setFields(new ArrayList<FieldSchema>() {{
        add(new FieldSchema("col1", "string", "comment1"));
    }
    });
    table.setTableType(TableType.EXTERNAL_TABLE);
    table.setDataLocation(new Path("somehdfspath"));
    return table;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
private Table createCubeHiveTable(AbstractCubeTable table) throws LensException {
  try {
    Table tbl = getClient().newTable(table.getName().toLowerCase());
    tbl.setTableType(TableType.MANAGED_TABLE);
    tbl.getTTable().getSd().setCols(table.getColumns());
    tbl.getTTable().getParameters().putAll(table.getProperties());
    getClient().createTable(tbl);
    // do get to update cache
    getTable(tbl.getTableName());
    return tbl;
  } catch (Exception e) {
    throw new LensException("Exception creating table", e);
  }
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
/**
 * Creates the temp metastore table.
 *
 * @param dataLocation the data location
 * @param metadata     the metadata
 * @return the string
 * @throws HiveException the hive exception
 */
protected String createTempMetastoreTable(String dataLocation, QueryResultSetMetadata metadata) throws HiveException {
  String tableName = "lens_rdd_" + UUID.randomUUID().toString().replace("-", "_");

  Hive hiveClient = Hive.get(HIVE_CONF);
  Table tbl = hiveClient.newTable("default." + tableName);
  tbl.setTableType(TableType.MANAGED_TABLE);
  tbl.setInputFormatClass(INPUT_FORMAT);
  // String outputFormat = null;
  // tbl.setOutputFormatClass(outputFormat);

  // Add columns
  for (ResultColumn rc : metadata.getColumns()) {
    tbl.getCols().add(new FieldSchema(rc.getName(), toHiveType(rc.getType()), "default"));
    System.out.println("@@@@ COL " + rc.getName() + " TYPE " + toHiveType(rc.getType()));
  }

  tbl.getPartCols().add(new FieldSchema(TEMP_TABLE_PART_COL, "string", "default"));
  hiveClient.createTable(tbl);

  log.info("Table {} created", tableName);

  // Add partition to the table
  AddPartitionDesc partitionDesc = new AddPartitionDesc("default", tableName, false);
  Map<String, String> partSpec = new HashMap<String, String>();
  partSpec.put(TEMP_TABLE_PART_COL, TEMP_TABLE_PART_VAL);
  partitionDesc.addPartition(partSpec, dataLocation);
  hiveClient.createPartitions(partitionDesc);
  log.info("Created partition in {} for data in {}", tableName, dataLocation);

  return tableName;
}
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
public void createTable(String tableName, String dataFile) throws HiveException {

    File filedataFile = new File(dataFile);
    Path dataFilePath = new Path(filedataFile.toURI());
    Path partDir = dataFilePath.getParent();

    // Create table
    List<FieldSchema> columns = new ArrayList<FieldSchema>();

    // Label is optional. Not used for unsupervised models.
    // If present, label will be the first column, followed by features
    if (labelColumn != null) {
      columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
    }

    for (String feature : features) {
      columns.add(new FieldSchema(feature, "double", "Feature " + feature));
    }

    Table tbl = Hive.get(conf).newTable(database + "." + tableName);
    tbl.setTableType(TableType.MANAGED_TABLE);
    tbl.getTTable().getSd().setCols(columns);
    // tbl.getTTable().getParameters().putAll(new HashMap<String, String>());
    tbl.setInputFormatClass(TextInputFormat.class);
    tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
    tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");

    List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
    partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
    tbl.setPartCols(partCols);

    Hive.get(conf).dropTable(database, tableName, false, true);
    Hive.get(conf).createTable(tbl, true);
    log.info("Created table {}", tableName);

    // Add partition for the data file
    AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName,
        false);
    Map<String, String> partSpec = new HashMap<String, String>();
    partSpec.put("dummy_partition_col", "dummy_val");
    partitionDesc.addPartition(partSpec, partDir.toUri().toString());
    Hive.get(conf).createPartitions(partitionDesc);
    log.info("{}: Added partition {}", tableName, partDir.toUri().toString());
  }
 

import org.apache.hadoop.hive.ql.metadata.Table; //导入方法依赖的package包/类
/**
 * Creates the example table.
 *
 * @param conf           the conf
 * @param database       the database
 * @param tableName      the table name
 * @param sampleDataFile the sample data file
 * @param labelColumn    the label column
 * @param features       the features
 * @throws HiveException the hive exception
 */
public static void createTable(HiveConf conf, String database, String tableName, String sampleDataFile,
  String labelColumn, Map<String, String> tableParams, String... features) throws HiveException {

  Path dataFilePath = new Path(sampleDataFile);
  Path partDir = dataFilePath.getParent();

  // Create table
  List<FieldSchema> columns = new ArrayList<FieldSchema>();

  // Label is optional. Not used for unsupervised models.
  // If present, label will be the first column, followed by features
  if (labelColumn != null) {
    columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
  }

  for (String feature : features) {
    columns.add(new FieldSchema(feature, "double", "Feature " + feature));
  }

  Table tbl = Hive.get(conf).newTable(database + "." + tableName);
  tbl.setTableType(TableType.MANAGED_TABLE);
  tbl.getTTable().getSd().setCols(columns);
  tbl.getTTable().getParameters().putAll(tableParams);
  tbl.setInputFormatClass(TextInputFormat.class);
  tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
  tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");

  List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
  partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
  tbl.setPartCols(partCols);

  Hive.get(conf).createTable(tbl, false);
  log.info("Created table {}", tableName);

  // Add partition for the data file
  AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName, false);
  Map<String, String> partSpec = new HashMap<String, String>();
  partSpec.put("dummy_partition_col", "dummy_val");
  partitionDesc.addPartition(partSpec, partDir.toUri().toString());
  Hive.get(conf).createPartitions(partitionDesc);
  log.info("{}: Added partition {}", tableName, partDir.toUri().toString());
}