Java ColumnStatisticsDesc类代码示例

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void typical() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"), newFieldSchema("c"));
  Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1"));
  List<Partition> partitions = Lists.newArrayList(newPartition(table, "b", "d"));
  statisticsPerPartitionName.put("a=b/c=d", columnStats);

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);
  List<String> expectedName = Lists.newArrayList("a=b/c=d");

  assertThat(partitionsAndStatistics.getPartitionNames(), is(expectedName));
  assertThat(partitionsAndStatistics.getPartitions(), is(partitions));
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db1", "t1");
  statsDesc.setPartName("a=b/c=d");
  ColumnStatistics expectedStats = new ColumnStatistics(statsDesc, columnStats);
  assertThat(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0)), is(expectedStats));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void transformInboundSetPartitionStatsRequest() throws Exception {
  SetPartitionsStatsRequest setPartitionsStatsRequest = new SetPartitionsStatsRequest();
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  setPartitionsStatsRequest.setColStats(Lists.newArrayList(columnStatistics));
  SetPartitionsStatsRequest result = databaseMapping
      .transformInboundSetPartitionStatsRequest(setPartitionsStatsRequest);
  assertThat(result, is(sameInstance(setPartitionsStatsRequest)));
  ColumnStatistics resultColStats = result.getColStats().get(0);
  assertThat(resultColStats, is(sameInstance(columnStatistics)));
  assertThat(resultColStats.getStatsDesc(), is(sameInstance(statsDesc)));
  assertThat(resultColStats.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
private void setColumnStatistics(TableName tableName, TableStatistics tableStatistics, Table table, Predicate<FieldSchema> fieldsPredicate)
        throws TException
{
    ColumnStatisticsDesc hiveColumnStatisticsDesc = new ColumnStatisticsDesc();
    hiveColumnStatisticsDesc.setIsTblLevel(false);
    hiveColumnStatisticsDesc.setDbName(getSchema(tableName));
    hiveColumnStatisticsDesc.setTableName(tableName.getSchemalessNameInDatabase());

    List<ColumnStatisticsObj> hiveColumnStatisticsObjs = table.getSd().getCols().stream()
            .filter(fieldsPredicate)
            .map(fieldSchema -> toHiveColumnStatistics(fieldSchema, tableStatistics.getColumns().get(fieldSchema.getName())))
            .collect(toList());

    org.apache.hadoop.hive.metastore.api.ColumnStatistics hiveColumnStatistics = new org.apache.hadoop.hive.metastore.api.ColumnStatistics();
    hiveColumnStatistics.setStatsDesc(hiveColumnStatisticsDesc);
    hiveColumnStatistics.setStatsObj(hiveColumnStatisticsObjs);
    client.update_table_column_statistics(hiveColumnStatistics);
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Before
public void prepare() throws Exception {
  when(metaStoreClientSupplier.get()).thenReturn(mockMetaStoreClient);
  when(replicaCatalog.getName()).thenReturn(NAME);

  hiveConf = new HiveConf();
  hiveConf.setVar(ConfVars.METASTOREURIS, REPLICA_META_STORE_URIS);
  replica = newReplica(ReplicationMode.FULL);
  tableLocation = temporaryFolder.newFolder("table_location").toURI().toString();

  sourceTable = newTable();
  existingPartition = newPartition("one", "two");

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(COLUMN_A, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(0, 1)));
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(COLUMN_B, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1, 2)));
  columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, DB_NAME, TABLE_NAME);
  columnStatistics = new ColumnStatistics(statsDesc, columnStatisticsObjs);

  tableAndStatistics = new TableAndStatistics(sourceTable, columnStatistics);

  existingReplicaTable = new Table(sourceTable);

  when(mockReplicaLocationManager.getTableLocation()).thenReturn(new Path(tableLocation));
  when(mockReplicaLocationManager.getPartitionBaseLocation()).thenReturn(new Path(tableLocation));

  when(mockMetaStoreClient.getTable(DB_NAME, TABLE_NAME)).thenReturn(existingReplicaTable);
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
private ColumnStatistics newPartitionStatistics(String... values) {
  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(COLUMN_A, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(0, 1)));
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(COLUMN_B, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1, 2)));
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  statsDesc.setPartName(partitionName(values));
  return new ColumnStatistics(statsDesc, columnStatisticsObjs);
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Override
public ColumnStatistics transform(ColumnStatistics columnStatistics) {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "new_db", "new_table");
  statsDesc.setPartName("part=newPart");
  columnStatistics.setStatsDesc(statsDesc);
  return columnStatistics;
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void newTableWithNameMappingsAndStats() {
  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  TableAndStatistics source = new TableAndStatistics(sourceTable,
      new ColumnStatistics(new ColumnStatisticsDesc(true, DB_NAME, TABLE_NAME), columnStatisticsObjs));

  TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, source, MAPPED_DB_NAME, MAPPED_TABLE_NAME,
      REPLICA_DATA_DESTINATION, FULL);
  Table replica = replicaAndStats.getTable();
  ColumnStatistics replicaStatistics = replicaAndStats.getStatistics();

  assertThat(replica.getDbName(), is(MAPPED_DB_NAME));
  assertThat(replica.getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
  assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
  assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
      is(SOURCE_META_STORE_URIS));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
  assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
  assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
  assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));

  assertThat(replicaStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaStatistics.getStatsObj().size(), is(2));
  assertThat(replicaStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void newReplicaPartitionStatisticsWithTransformation() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY,
      PartitionTransformation.IDENTITY, COLUMN_STATISTICS_TRANSFORMATION);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is("new_db"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is("new_table"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("part=newPart"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
public static Table createUnpartitionedTable(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String table,
    URI location)
  throws TException {
  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toString());
  sd.setParameters(new HashMap<String, String>());
  sd.setInputFormat(TextInputFormat.class.getName());
  sd.setOutputFormat(TextOutputFormat.class.getName());
  sd.setSerdeInfo(new SerDeInfo());
  sd.getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.OpenCSVSerde");

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, database, table);
  ColumnStatisticsData statsData = new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1L, 2L));
  ColumnStatisticsObj cso1 = new ColumnStatisticsObj("id", "bigint", statsData);
  List<ColumnStatisticsObj> statsObj = Collections.singletonList(cso1);
  metaStoreClient.updateTableColumnStatistics(new ColumnStatistics(statsDesc, statsObj));

  return hiveTable;
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
public static Table createPartitionedTable(
    HiveMetaStoreClient metaStoreClient,
    String database,
    String table,
    URI location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toString());
  sd.setParameters(new HashMap<String, String>());
  sd.setInputFormat(TextInputFormat.class.getName());
  sd.setOutputFormat(TextOutputFormat.class.getName());
  sd.setSerdeInfo(new SerDeInfo());
  sd.getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.OpenCSVSerde");

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, database, table);
  ColumnStatisticsData statsData = new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1L, 2L));
  ColumnStatisticsObj cso1 = new ColumnStatisticsObj("id", "bigint", statsData);
  List<ColumnStatisticsObj> statsObj = Collections.singletonList(cso1);
  metaStoreClient.updateTableColumnStatistics(new ColumnStatistics(statsDesc, statsObj));

  return hiveTable;
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Before
public void init() {
  stats = new ColumnStatistics(new ColumnStatisticsDesc(true, "database", "table"),
      ImmutableList.of(
          new ColumnStatisticsObj("a", "int",
              new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1L, 2L))),
          new ColumnStatisticsObj("b", "string",
              new ColumnStatisticsData(_Fields.STRING_STATS, new StringColumnStatsData(10L, 3L, 0L, 1L)))));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void transformInboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformInboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void transformOutboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformOutboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(OUT_DB_NAME));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
ColumnStatistics newReplicaPartitionStatistics(
    Table replicaTable,
    Partition replicaPartition,
    ColumnStatistics sourcePartitionStatistics) {
  ColumnStatisticsDesc statisticsDesc = new ColumnStatisticsDesc(false, replicaPartition.getDbName(),
      replicaPartition.getTableName());
  try {
    statisticsDesc.setPartName(Warehouse.makePartName(replicaTable.getPartitionKeys(), replicaPartition.getValues()));
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }

  return columnStatisticsTransformation
      .transform(new ColumnStatistics(statisticsDesc, sourcePartitionStatistics.getStatsObj()));
}
 

import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; //导入依赖的package包/类
@Test
public void newReplicaPartitionStatistics() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}