Java RowConstants类代码示例

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
    final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);

    LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
    byte[] start = encoder.createBuf();
    byte[] end = encoder.createBuf();

    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
    encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start);

    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
    encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end);
    byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning
    System.arraycopy(end, 0, temp, 0, end.length);
    end = temp;

    List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys);

    KylinConfig config = cubeSeg.getCubeDesc().getConfig();
    int hbaseCaching = config.getHBaseScanCacheRows();
    int hbaseMaxResultSize = config.getHBaseScanMaxResultSize();
    //        if (isMemoryHungry(selectedColBlocks))
    //            hbaseCaching /= 10;

    return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize);
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private List<byte[]> getRowKeysDifferentShards(byte[] halfCookedKey) {
    final short cuboidShardNum = cubeSeg.getCuboidShardNum(cuboid.getId());

    if (!cubeSeg.isEnableSharding()) {
        return Lists.newArrayList(halfCookedKey);//not shard to append at head, so it is already well cooked
    } else {
        List<byte[]> ret = Lists.newArrayList();
        for (short i = 0; i < cuboidShardNum; ++i) {
            short shard = ShardingHash.normalize(cubeSeg.getCuboidBaseShard(cuboid.getId()), i, cubeSeg.getTotalShards(cuboid.getId()));
            byte[] cookedKey = Arrays.copyOf(halfCookedKey, halfCookedKey.length);
            BytesUtil.writeShort(shard, cookedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
            ret.add(cookedKey);
        }
        return ret;
    }
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
    List<Integer> keys = new ArrayList<Integer>();
    Iterator<Integer> it = hllcMap.keySet().iterator();
    while (it.hasNext()) {
        keys.add(it.next());
    }
    Collections.sort(keys);
    it = keys.iterator();
    while (it.hasNext()) {
        int key = it.next();
        HyperLogLogPlusCounter hllc = hllcMap.get(key);
        ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
        buf.clear();
        hllc.writeRegisters(buf);
        buf.flip();
        context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
        // context.write(new Text("ErrorRate_" + key), new
        // LongWritable((long)hllc.getErrorRate()));
    }

}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private void init() {
    int[] offsets = new int[columns.length];
    int o = RowConstants.ROWKEY_CUBOIDID_LEN;
    for (int i = 0; i < columns.length; i++) {
        offsets[i] = o;
        o += columnSizes[i];
    }
    this.columnOffsets = offsets;

    this.columnsAsList = Arrays.asList(columns);

    HashMap<TblColRef, Integer> map = Maps.newHashMap();
    for (int i = 0; i < columns.length; i++) {
        map.put(columns[i], i);
    }
    this.columnIdxMap = map;
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private void init(Collection<ColumnValueRange> andDimensionRanges) {
    int size = andDimensionRanges.size();
    Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size);
    for (ColumnValueRange dimRange : andDimensionRanges) {
        TblColRef column = dimRange.getColumn();
        startValues.put(column, dimRange.getBeginValue());
        stopValues.put(column, dimRange.getEndValue());
        fuzzyValues.put(column, dimRange.getEqualValues());

        TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
        if (column.equals(partitionDateColumnRef)) {
            initPartitionRange(dimRange);
        }
    }

    AbstractRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
    this.startKey = encoder.encode(startValues);
    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
    // In order to make stopRow inclusive add a trailing 0 byte. #See Scan.setStopRow(byte [] stopRow)
    this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES);

    // always fuzzy match cuboid ID to lock on the selected cuboid
    this.fuzzyKeys = buildFuzzyKeys(fuzzyValues);
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private static byte[][] getSplitsByRegionCount(int regionCount) {
    byte[][] result = new byte[regionCount - 1][];
    for (int i = 1; i < regionCount; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
    Iterator<Integer> it = hllcMap.keySet().iterator();
    while (it.hasNext()) {
        int key = it.next();
        HyperLogLogPlusCounter hllc = hllcMap.get(key);
        ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
        buf.clear();
        hllc.writeRegisters(buf);
        buf.flip();
        context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit()));
    }
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private byte[] getBytes(String str) throws IOException {
    HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
    StringTokenizer tokenizer = new StringTokenizer(str, ColumnCardinalityMapper.DEFAULT_DELIM);
    int i = 0;
    while (tokenizer.hasMoreTokens()) {
        String temp = i + "_" + tokenizer.nextToken();
        i++;
        hllc.add(Bytes.toBytes(temp));
    }
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    buf.clear();
    hllc.writeRegisters(buf);
    buf.flip();
    return buf.array();
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
@Test
public void basicTest() {
    MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("HLLC16") };
    MeasureCodec codec = new MeasureCodec(descs);

    DoubleWritable d = new DoubleWritable(1.0);
    LongWritable l = new LongWritable(2);
    BigDecimal b = new BigDecimal("333.1234567");
    HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16);
    hllc.add("1234567");
    hllc.add("abcdefg");
    HyperLogLogPlusCounter hllc2 = new HyperLogLogPlusCounter(16);
    hllc.add("1234567");
    hllc.add("abcdefg");
    Object values[] = new Object[] { d, l, b, hllc, hllc2 };

    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);

    codec.encode(values, buf);
    buf.flip();
    System.out.println("size: " + buf.limit());

    Object copy[] = new Object[values.length];
    codec.decode(buf, copy);

    assertTrue(Arrays.equals(values, copy));
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
private void init(Collection<ColumnValueRange> andDimensionRanges) {
    int size = andDimensionRanges.size();
    Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size);
    for (ColumnValueRange dimRange : andDimensionRanges) {
        TblColRef column = dimRange.getColumn();
        startValues.put(column, dimRange.getBeginValue());
        stopValues.put(column, dimRange.getEndValue());
        fuzzyValues.put(column, dimRange.getEqualValues());

        TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
        if (column.equals(partitionDateColumnRef)) {
            initPartitionRange(dimRange);
        }
    }

    AbstractRowKeyEncoder encoder = AbstractRowKeyEncoder.createInstance(cubeSeg, cuboid);

    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);

    this.startKey = encoder.encode(startValues);

    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);

    // In order to make stopRow inclusive add a trailing 0 byte. #See
    // Scan.setStopRow(byte [] stopRow)
    this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES);

    // restore encoder defaults for later reuse (note
    // AbstractRowKeyEncoder.createInstance() caches instances)
    encoder.setBlankByte(AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE);

    // always fuzzy match cuboid ID to lock on the selected cuboid
    this.fuzzyKeys = buildFuzzyKeys(fuzzyValues);
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
@Override
protected void doSetup(Context context) throws IOException, InterruptedException {
    super.bindCurrentConfiguration(context.getConfiguration());

    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
    segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);

    config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeManager = CubeManager.getInstance(config);
    cube = cubeManager.getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    mergedCubeSegment = cube.getSegmentById(segmentID);

    // int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
    newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];// size will auto-grow
    newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);

    // decide which source segment
    FileSplit fileSplit = (FileSplit) context.getInputSplit();
    IMROutput2.IMRMergeOutputFormat outputFormat = MRUtil.getBatchMergeOutputSide2(mergedCubeSegment).getOuputFormat();
    sourceCubeSegment = outputFormat.findSourceSegment(fileSplit, cube);

    rowKeySplitter = new RowKeySplitter(sourceCubeSegment, 65, 255);
    rowKeyEncoderProvider = new RowKeyEncoderProvider(mergedCubeSegment);

    measureDescs = cubeDesc.getMeasures();
    codec = new BufferedMeasureCodec(measureDescs);
    measureObjs = new Object[measureDescs.size()];
    outputValue = new Text();

    dictMeasures = Lists.newArrayList();
    oldDicts = Maps.newHashMap();
    newDicts = Maps.newHashMap();
    for (int i = 0; i < measureDescs.size(); i++) {
        MeasureDesc measureDesc = measureDescs.get(i);
        MeasureType measureType = measureDesc.getFunction().getMeasureType();
        List<TblColRef> columns = measureType.getColumnsNeedDictionary(measureDesc.getFunction());
        boolean needReEncode = false;
        for (TblColRef col : columns) {
            //handle the column that all records is null
            if (sourceCubeSegment.getDictionary(col) == null) {
                continue;
            }

            oldDicts.put(col, sourceCubeSegment.getDictionary(col));
            newDicts.put(col, mergedCubeSegment.getDictionary(col));
            if (!sourceCubeSegment.getDictionary(col).equals(mergedCubeSegment.getDictionary(col))) {
                needReEncode = true;
            }
        }
        if (needReEncode) {
            dictMeasures.add(Pair.newPair(i, measureType.newIngester()));
        }
    }
}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
@Ignore
@Test
public void testMapReduceWithSlr() throws IOException {

    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentID = "198va32a-a33e-4b69-83dd-0bb8b1f8c53b";
    mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_ID, segmentID);

    byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
    byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
    Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));

    mapReduceDriver.addInput(input1);

    List<Pair<Text, Text>> result = mapReduceDriver.run();

    assertEquals(4, result.size());

    byte[] resultKey = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 0, -104, -106, -128, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
    byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 };
    Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));

    //As we will truncate decimal(KYLIN-766), value will no longer equals to resultValue
    Collection<Text> keys = Collections2.transform(result, new Function<Pair<Text, Text>, Text>() {
        @Nullable
        @Override
        public Text apply(Pair<Text, Text> input) {
            return input.getFirst();
        }
    });
    assertTrue(keys.contains(output1.getFirst()));
    assertTrue(!result.contains(output1));

    long[] keySet = new long[result.size()];

    System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 }));
    for (int i = 0; i < result.size(); i++) {
        byte[] bytes = new byte[result.get(i).getFirst().getLength()];
        System.arraycopy(result.get(i).getFirst().getBytes(), RowConstants.ROWKEY_SHARDID_LEN, bytes, 0, result.get(i).getFirst().getLength() - RowConstants.ROWKEY_SHARDID_LEN);
        System.out.println(Bytes.toLong(bytes));
        keySet[i] = Bytes.toLong(bytes);
    }

    // refer to CuboidSchedulerTest.testGetSpanningCuboid()
    assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet);

}
 

import org.apache.kylin.cube.kv.RowConstants; //导入依赖的package包/类
public int getRowKeyPreambleSize() {
    return isEnableSharding() ? RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN : RowConstants.ROWKEY_CUBOIDID_LEN;
}