本文整理汇总了Java中parquet.example.data.Group类的典型用法代码示例。如果您正苦于以下问题:Java Group类的具体用法?Java Group怎么用?Java Group使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Group类属于parquet.example.data包,在下文中一共展示了Group类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: build
import parquet.example.data.Group; //导入依赖的package包/类
public PFileWriter build(){
try {
this.parquetWriter = new ParquetWriter<Group>(
file,
gws,
CompressionCodecName.SNAPPY,
1024,
1024,
512,
true,
false,
ParquetProperties.WriterVersion.PARQUET_1_0,
conf);
}catch (IOException ioe){
LOG.error(ioe.toString());
}
return this;
}
示例2: initWriter
import parquet.example.data.Group; //导入依赖的package包/类
public static ParquetWriter<Group> initWriter(String fileName, Map<String, String> metas)
throws IOException{
GroupWriteSupport.setSchema(schema, conf);
ParquetWriter<Group> writer = new ParquetWriter<Group>(
initFile(fileName),
new GroupWriteSupport(metas),
CompressionCodecName.SNAPPY,
1024,
1024,
512,
true,
false,
ParquetProperties.WriterVersion.PARQUET_1_0,
conf);
return writer;
}
示例3: groupToCells
import parquet.example.data.Group; //导入依赖的package包/类
/**
* transform data in group into cells(List<cell> - > {@link org.apache.hadoop.hbase.client.Result}</>)
* @param group
* @return
*/
public static List<Cell> groupToCells(Group group){
List<Cell> cells = new LinkedList<>();
if(group != null){
cells = new LinkedList<>();
GroupType groupType = group.getType();
List<Type> types = groupType.getFields();
byte [] rowKey = group.getBinary(HConstants.ROW_KEY, 0).getBytes();
long timestamp = group.getLong(HConstants.TIME_STAMP, 0);
for(Type t : types){
if(! t.getName().equals(HConstants.ROW_KEY) && ! t.getName().equals(HConstants.TIME_STAMP)){
String name = t.getName();
String [] names = name.split(":");
if(names.length == 2) {
byte[] value = group.getBinary(name, 0).getBytes();
Cell cell = new KeyValue(rowKey, names[0].getBytes(), names[1].getBytes(), timestamp, value);
cells.add(cell);
}
}
}
}
return cells;
}
示例4: readFile
import parquet.example.data.Group; //导入依赖的package包/类
public static List<Group> readFile(File f, Filter filter) throws IOException {
Configuration conf = new Configuration();
GroupWriteSupport.setSchema(schema, conf);
ParquetReader<Group> reader =
ParquetReader.builder(new GroupReadSupport(), new Path(f.getAbsolutePath()))
.withConf(conf)
.withFilter(filter)
.build();
Group current;
List<Group> users = new ArrayList<Group>();
current = reader.read();
while (current != null) {
users.add(current);
current = reader.read();
}
return users;
}
示例5: testUserDefinedByInstance
import parquet.example.data.Group; //导入依赖的package包/类
@Test
public void testUserDefinedByInstance() throws Exception {
LongColumn name = longColumn("id");
final HashSet<Long> h = new HashSet<Long>();
h.add(20L);
h.add(27L);
h.add(28L);
FilterPredicate pred = userDefined(name, new SetInFilter(h));
List<Group> found = PhoneBookWriter.readFile(phonebookFile, FilterCompat.get(pred));
assertFilter(found, new UserFilter() {
public boolean keep(User u) {
return u != null && h.contains(u.getId());
}
});
}
示例6: asGroup
import parquet.example.data.Group; //导入依赖的package包/类
/**
* transform mutation to group
* assume that sfg is not null
* @param gf
* @return
*/
public Group asGroup(GroupFactory gf){
Group group = gf.newGroup().append(ROW_KEY, Bytes.toString(row));
CellScanner cellScanner = cellScanner();
try {
while (cellScanner.advance()){
Cell cell = cellScanner.current();
group.append(
Bytes.toString(cell.getFamily()) +":"+ Bytes.toString(cell.getQualifier()),
Bytes.toString(cell.getValue())
);
}
group.append("timestamp", System.currentTimeMillis());
}catch (IOException io){
LOG.error(io);
}
return group;
}
示例7: toString
import parquet.example.data.Group; //导入依赖的package包/类
public String toString(String indent) {
StringBuilder result = new StringBuilder();
int i = 0;
for (Type field : this.schema.getFields()) {
String name = field.getName();
List<Object> values = this.data[i];
for (Object value : values) {
result.append(indent).append(name);
if (value == null) {
result.append(": NULL\n");
} else if (value instanceof Group) {
result.append("\n").append(((ParquetGroup) value).toString(indent + " "));
} else {
result.append(": ").append(value.toString()).append("\n");
}
}
i++;
}
return result.toString();
}
示例8: getWriter
import parquet.example.data.Group; //导入依赖的package包/类
/**
* Build a {@link ParquetWriter<Group>} for given file path with a block size.
* @param blockSize
* @param stagingFile
* @return
* @throws IOException
*/
public ParquetWriter<Group> getWriter(int blockSize, Path stagingFile)
throws IOException {
State state = this.destination.getProperties();
int pageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_PAGE_SIZE), DEFAULT_PAGE_SIZE);
int dictPageSize = state.getPropAsInt(getProperty(WRITER_PARQUET_DICTIONARY_PAGE_SIZE), DEFAULT_BLOCK_SIZE);
boolean enableDictionary =
state.getPropAsBoolean(getProperty(WRITER_PARQUET_DICTIONARY), DEFAULT_IS_DICTIONARY_ENABLED);
boolean validate = state.getPropAsBoolean(getProperty(WRITER_PARQUET_VALIDATE), DEFAULT_IS_VALIDATING_ENABLED);
String rootURI = state.getProp(WRITER_FILE_SYSTEM_URI, LOCAL_FS_URI);
Path absoluteStagingFile = new Path(rootURI, stagingFile);
CompressionCodecName codec = getCodecFromConfig();
GroupWriteSupport support = new GroupWriteSupport();
Configuration conf = new Configuration();
GroupWriteSupport.setSchema(this.schema, conf);
ParquetProperties.WriterVersion writerVersion = getWriterVersion();
return new ParquetWriter<>(absoluteStagingFile, support, codec, blockSize, pageSize, dictPageSize, enableDictionary,
validate, writerVersion, conf);
}
示例9: readParquetFiles
import parquet.example.data.Group; //导入依赖的package包/类
private List<Group> readParquetFiles(File outputFile)
throws IOException {
ParquetReader<Group> reader = null;
List<Group> records = new ArrayList<>();
try {
reader = new ParquetReader<>(new Path(outputFile.toString()), new SimpleReadSupport());
for (Group value = reader.read(); value != null; value = reader.read()) {
records.add(value);
}
} finally {
if (reader != null) {
try {
reader.close();
} catch (Exception ex) {
System.out.println(ex.getMessage());
}
}
}
return records;
}
示例10: reduce
import parquet.example.data.Group; //导入依赖的package包/类
@Override
protected void reduce(Text key, Iterable<DoubleWritable> values, Context context) throws IOException, InterruptedException {
Mean mean = new Mean();
for (DoubleWritable val : values) {
mean.increment(val.get());
}
Group group = factory.newGroup()
.append("symbol", key.toString())
.append("avg", mean.getResult());
context.write(null, group);
}
示例11: map
import parquet.example.data.Group; //导入依赖的package包/类
@Override
public void map(Void key,
Group value,
Context context) throws IOException, InterruptedException {
context.write(new Text(value.getString("symbol", 0)),
new DoubleWritable(Double.valueOf(value.getValueToString(2, 0))));
}
示例12: append
import parquet.example.data.Group; //导入依赖的package包/类
public void append(Group group){
try {
parquetWriter.write(group);
}catch (IOException ioe){
LOG.error(ioe.toString());
}
}
示例13: readGroup
import parquet.example.data.Group; //导入依赖的package包/类
/**
* read a row
*/
@Override
public Group readGroup() {
Group group = null;
try {
group = reader.read();
}catch (IOException ioe){
LOG.error(ioe);
}
return group;
}
示例14: readCells
import parquet.example.data.Group; //导入依赖的package包/类
/**
* read value from parquet as cell
*
* @return
*/
@Override
public List<Cell> readCells() {
List<Cell> cells = new LinkedList<>();
Group group = readGroup();
if(group == null)
return null;
else {
List<ColumnDescriptor> columns = schema.getColumns();
for (ColumnDescriptor column : columns){
}
return cells;
}
}
示例15: nextRow
import parquet.example.data.Group; //导入依赖的package包/类
/**
* @return the next {@link Group}
*/
public Group nextRow() {
Group rs = curr;
curr = next;
next = reader.readGroup();
return rs;
}