Java Group类代码示例

import parquet.example.data.Group; //导入依赖的package包/类
public PFileWriter build(){
    try {
        this.parquetWriter = new ParquetWriter<Group>(
                file,
                gws,
                CompressionCodecName.SNAPPY,
                1024,
                1024,
                512,
                true,
                false,
                ParquetProperties.WriterVersion.PARQUET_1_0,
                conf);
    }catch (IOException ioe){
        LOG.error(ioe.toString());
    }
    return this;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
public static ParquetWriter<Group> initWriter(String fileName, Map<String, String> metas)
        throws IOException{


    GroupWriteSupport.setSchema(schema, conf);


    ParquetWriter<Group> writer = new ParquetWriter<Group>(
            initFile(fileName),
            new GroupWriteSupport(metas),
            CompressionCodecName.SNAPPY,
            1024,
            1024,
            512,
            true,
            false,
            ParquetProperties.WriterVersion.PARQUET_1_0,
            conf);

    return writer;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * transform data in group into cells(List<cell> - > {@link org.apache.hadoop.hbase.client.Result}</>)
 * @param group
 * @return
 */
public static List<Cell> groupToCells(Group group){

    List<Cell> cells = new LinkedList<>();
    if(group != null){
        cells = new LinkedList<>();
        GroupType groupType = group.getType();
        List<Type> types = groupType.getFields();
        byte [] rowKey = group.getBinary(HConstants.ROW_KEY, 0).getBytes();

        long timestamp = group.getLong(HConstants.TIME_STAMP, 0);

        for(Type t : types){
            if(! t.getName().equals(HConstants.ROW_KEY) && ! t.getName().equals(HConstants.TIME_STAMP)){
                String name = t.getName();
                String [] names = name.split(":");
                if(names.length == 2) {
                    byte[] value = group.getBinary(name, 0).getBytes();
                    Cell cell = new KeyValue(rowKey, names[0].getBytes(), names[1].getBytes(), timestamp, value);
                    cells.add(cell);
                }
            }
        }
    }
    return cells;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
public static List<Group> readFile(File f, Filter filter) throws IOException {
    Configuration conf = new Configuration();
    GroupWriteSupport.setSchema(schema, conf);

    ParquetReader<Group> reader =
            ParquetReader.builder(new GroupReadSupport(), new Path(f.getAbsolutePath()))
                    .withConf(conf)
                    .withFilter(filter)
                    .build();

    Group current;
    List<Group> users = new ArrayList<Group>();

    current = reader.read();
    while (current != null) {
        users.add(current);
        current = reader.read();
    }

    return users;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
@Test
public void testUserDefinedByInstance() throws Exception {
    LongColumn name = longColumn("id");

    final HashSet<Long> h = new HashSet<Long>();
    h.add(20L);
    h.add(27L);
    h.add(28L);

    FilterPredicate pred = userDefined(name, new SetInFilter(h));

    List<Group> found = PhoneBookWriter.readFile(phonebookFile, FilterCompat.get(pred));

    assertFilter(found, new UserFilter() {

        public boolean keep(User u) {
            return u != null && h.contains(u.getId());
        }
    });
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * transform mutation to group
 * assume that sfg is not null
 * @param gf
 * @return
 */

public Group asGroup(GroupFactory gf){

    Group group = gf.newGroup().append(ROW_KEY, Bytes.toString(row));
    CellScanner cellScanner = cellScanner();
    try {
        while (cellScanner.advance()){
            Cell cell = cellScanner.current();
            group.append(
                    Bytes.toString(cell.getFamily()) +":"+ Bytes.toString(cell.getQualifier()),
                    Bytes.toString(cell.getValue())
            );
        }
        group.append("timestamp", System.currentTimeMillis());
    }catch (IOException io){
        LOG.error(io);
    }
    return group;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
public String toString(String indent) {
  StringBuilder result = new StringBuilder();
  int i = 0;
  for (Type field : this.schema.getFields()) {
    String name = field.getName();
    List<Object> values = this.data[i];
    for (Object value : values) {
      result.append(indent).append(name);
      if (value == null) {
        result.append(": NULL\n");
      } else if (value instanceof Group) {
        result.append("\n").append(((ParquetGroup) value).toString(indent + "  "));
      } else {
        result.append(": ").append(value.toString()).append("\n");
      }
    }
    i++;
  }
  return result.toString();
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * Build a {@link ParquetWriter<Group>} for given file path with a block size.
 * @param blockSize
 * @param stagingFile
 * @return
 * @throws IOException
 */
public ParquetWriter<Group> getWriter(int blockSize, Path stagingFile)
    throws IOException {
  State state = this.destination.getProperties();
  int pageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_PAGE_SIZE), DEFAULT_PAGE_SIZE);
  int dictPageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_DICTIONARY_PAGE_SIZE), DEFAULT_BLOCK_SIZE);
  boolean enableDictionary =
      state.getPropAsBoolean(getProperty(WRITER_PARQUET_DICTIONARY), DEFAULT_IS_DICTIONARY_ENABLED);
  boolean validate = state.getPropAsBoolean(getProperty(WRITER_PARQUET_VALIDATE), DEFAULT_IS_VALIDATING_ENABLED);
  String rootURI = state.getProp(WRITER_FILE_SYSTEM_URI, LOCAL_FS_URI);
  Path absoluteStagingFile = new Path(rootURI, stagingFile);
  CompressionCodecName codec = getCodecFromConfig();
  GroupWriteSupport support = new GroupWriteSupport();
  Configuration conf = new Configuration();
  GroupWriteSupport.setSchema(this.schema, conf);
  ParquetProperties.WriterVersion writerVersion = getWriterVersion();
  return new ParquetWriter<>(absoluteStagingFile, support, codec, blockSize, pageSize, dictPageSize, enableDictionary,
      validate, writerVersion, conf);
}
 

import parquet.example.data.Group; //导入依赖的package包/类
private List<Group> readParquetFiles(File outputFile)
    throws IOException {
  ParquetReader<Group> reader = null;
  List<Group> records = new ArrayList<>();
  try {
    reader = new ParquetReader<>(new Path(outputFile.toString()), new SimpleReadSupport());
    for (Group value = reader.read(); value != null; value = reader.read()) {
      records.add(value);
    }
  } finally {
    if (reader != null) {
      try {
        reader.close();
      } catch (Exception ex) {
        System.out.println(ex.getMessage());
      }
    }
  }
  return records;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
@Override
protected void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException {
  Mean mean = new Mean();
  for (DoubleWritable val : values) {
    mean.increment(val.get());
  }
  Group group = factory.newGroup()
      .append("symbol", key.toString())
      .append("avg", mean.getResult());
  context.write(null, group);
}
 

import parquet.example.data.Group; //导入依赖的package包/类
@Override
public void map(Void key,
                Group value,
                Context context) throws IOException, InterruptedException {
  context.write(new Text(value.getString("symbol", 0)),
      new DoubleWritable(Double.valueOf(value.getValueToString(2, 0))));
}
 

import parquet.example.data.Group; //导入依赖的package包/类
public void append(Group group){
    try {
        parquetWriter.write(group);
    }catch (IOException ioe){
        LOG.error(ioe.toString());
    }
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * read a row
 */
@Override
public Group readGroup() {
    Group group = null;
    try {
        group = reader.read();
    }catch (IOException ioe){
     LOG.error(ioe);
    }
    return group;
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * read value from parquet as cell
 *
 * @return
 */
@Override
public List<Cell> readCells() {
    List<Cell> cells = new LinkedList<>();
    Group group = readGroup();
    if(group == null)
        return null;
    else {
        List<ColumnDescriptor> columns = schema.getColumns();
        for (ColumnDescriptor column : columns){

        }
        return cells;
    }
}
 

import parquet.example.data.Group; //导入依赖的package包/类
/**
 * @return the next {@link Group}
 */
public Group nextRow() {
    Group rs = curr;
    curr = next;
    next = reader.readGroup();
    return rs;
}