Java Cuboid类代码示例

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public SegmentCubeTupleIterator(CubeSegmentScanner scanner, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
    this.scanner = scanner;
    this.cuboid = cuboid;
    this.selectedDimensions = selectedDimensions;
    this.selectedMetrics = selectedMetrics;
    this.tupleInfo = returnTupleInfo;
    this.tuple = new Tuple(returnTupleInfo);
    this.context = context;

    CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
    int[] gtDimsIdx = mapping.getDimIndexes(selectedDimensions);
    int[] gtMetricsIdx = mapping.getMetricsIndexes(selectedMetrics);
    // gtColIdx = gtDimsIdx + gtMetricsIdx
    int[] gtColIdx = new int[gtDimsIdx.length + gtMetricsIdx.length];
    System.arraycopy(gtDimsIdx, 0, gtColIdx, 0, gtDimsIdx.length);
    System.arraycopy(gtMetricsIdx, 0, gtColIdx, gtDimsIdx.length, gtMetricsIdx.length);

    this.gtValues = getGTValuesIterator(scanner.iterator(), scanner.getScanRequest(), gtDimsIdx, gtMetricsIdx);
    this.cubeTupleConverter = ((GTCubeStorageQueryBase) context.getStorageQuery()).newCubeTupleConverter(
            scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo);
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
        Set<TblColRef> groups, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
private void enableStreamAggregateIfBeneficial(Cuboid cuboid, Set<TblColRef> groupsD, StorageContext context) {
    CubeDesc cubeDesc = cuboid.getCubeDesc();
    boolean enabled = cubeDesc.getConfig().isStreamAggregateEnabled();

    Set<TblColRef> shardByInGroups = Sets.newHashSet();
    for (TblColRef col : cubeDesc.getShardByColumns()) {
        if (groupsD.contains(col)) {
            shardByInGroups.add(col);
        }
    }
    if (!shardByInGroups.isEmpty()) {
        enabled = false;
        logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: "
                + shardByInGroups);
    }

    if (!context.isNeedStorageAggregation()) {
        enabled = false;
        logger.debug("Aggregate partition results is not beneficial because no storage aggregation");
    }

    if (enabled) {
        context.enableStreamAggregate();
    }
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
@Override
protected void doSetup(Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    bindCurrentConfiguration(conf);
    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeSeg = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID));
    cubeDesc = cube.getDescriptor();
    baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    dictCols = Lists.newArrayList(cubeDesc.getAllColumnsNeedDictionaryBuilt());

    flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat();

    intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc);
    dictionaryColumnIndex = new int[dictCols.size()];
    for (int i = 0; i < dictCols.size(); i++) {
        TblColRef colRef = dictCols.get(i);
        int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef);
        dictionaryColumnIndex[i] = columnIndexOnFlatTbl;
    }

    reducerMapping = new FactDistinctColumnsReducerMapping(cube,
            conf.getInt(BatchConstants.CFG_HLL_REDUCER_NUM, 1));
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap) {
    final CubeDesc cubeDesc = cubeSegment.getCubeDesc();
    final List<Integer> rowkeyColumnSize = Lists.newArrayList();
    final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    final List<TblColRef> columnList = baseCuboid.getColumns();
    final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap();
    final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId());

    for (int i = 0; i < columnList.size(); i++) {
        rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding());
    }

    Map<Long, Double> sizeMap = Maps.newHashMap();
    for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) {
        sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(),
                baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize));
    }
    return sizeMap;
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
private static void printOneCuboidInfo(long parent, long cuboidID, Map<Long, Long> cuboidRows,
        Map<Long, Double> cuboidSizes, int dimensionCount, int depth, PrintWriter out) {
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < depth; i++) {
        sb.append("    ");
    }
    String cuboidName = Cuboid.getDisplayName(cuboidID, dimensionCount);
    sb.append("|---- Cuboid ").append(cuboidName);

    long rowCount = cuboidRows.get(cuboidID);
    double size = cuboidSizes.get(cuboidID);
    sb.append(", est row: ").append(rowCount).append(", est MB: ").append(formatDouble(size));

    if (parent != -1) {
        sb.append(", shrink: ").append(formatDouble(100.0 * cuboidRows.get(cuboidID) / cuboidRows.get(parent)))
                .append("%");
    }

    out.println(sb.toString());
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
@Override
public void write(long cuboidId, GTRecord record) throws IOException {
    byte[] key = createKey(cuboidId, record);
    final Cuboid cuboid = Cuboid.findForMandatory(cubeDesc, cuboidId);
    final int nDims = cuboid.getColumns().size();
    final ImmutableBitSet bitSet = new ImmutableBitSet(nDims, nDims + cubeDesc.getMeasures().size());

    for (int i = 0; i < nColumns; i++) {
        final Object[] values = record.getValues(bitSet, measureValues);
        final KeyValue keyValue = keyValueCreators.get(i).create(key, 0, key.length, values);
        final Put put = new Put(copy(key, 0, key.length));
        byte[] family = copy(keyValue.getFamilyArray(), keyValue.getFamilyOffset(), keyValue.getFamilyLength());
        byte[] qualifier = copy(keyValue.getQualifierArray(), keyValue.getQualifierOffset(), keyValue.getQualifierLength());
        byte[] value = copy(keyValue.getValueArray(), keyValue.getValueOffset(), keyValue.getValueLength());
        put.add(family, qualifier, value);
        puts.add(put);
    }
    if (puts.size() >= BATCH_PUT_THRESHOLD) {
        flush();
    }
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public static CoprocessorProjector makeForObserver(final CubeSegment cubeSegment, final Cuboid cuboid, final Collection<TblColRef> dimensionColumns) {

        RowKeyEncoder rowKeyMaskEncoder = new RowKeyEncoder(cubeSegment, cuboid) {
            @Override
            protected void fillHeader(byte[] bytes) {
                Arrays.fill(bytes, 0, this.getHeaderLength(), (byte) 0xff);
            }

            @Override
            protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) {
                byte bits = dimensionColumns.contains(column) ? (byte) 0xff : 0x00;
                Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, bits);
            }
        };

        byte[] mask = rowKeyMaskEncoder.encode(new String[cuboid.getColumns().size()]);
        return new CoprocessorProjector(mask, dimensionColumns.size() != 0);
    }
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public int getBuildLevel() {
    int ret = 1;//base cuboid => partial cube root
    if (this.getPartialCubeFullMask() == Cuboid.getBaseCuboidId(cubeDesc)) {
        ret -= 1;//if partial cube's root is base cuboid, then one round less agg
    }

    ret += getNormalDims().size();
    for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
        ret += hierarchyMask.allMasks.length;
    }
    for (Long joint : joints) {
        if ((joint & this.getHierarchyDimsMask()) == 0) {
            ret += 1;
        }
    }

    return ret;
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
    super(cubeSeg, cuboid);
    enableSharding = cubeSeg.isEnableSharding();
    headerLength = cubeSeg.getRowKeyPreambleSize();
    Set<TblColRef> shardByColumns = cubeSeg.getCubeDesc().getShardByColumns();
    if (shardByColumns.size() > 1) {
        throw new IllegalStateException("Does not support multiple UHC now");
    }
    colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap());
    for (TblColRef column : cuboid.getColumns()) {
        if (shardByColumns.contains(column)) {
            uhcOffset = bodyLength;
            uhcLength = colIO.getColumnLength(column);
        }
        bodyLength += colIO.getColumnLength(column);
    }
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
        Map<TblColRef, Dictionary<String>> dictionaryMap) {
    super(cuboidScheduler, flatDesc, dictionaryMap);
    this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    this.totalCuboidCount = cuboidScheduler.getCuboidCount();

    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);

    List<String> metricsAggrFuncsList = Lists.newArrayList();

    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Dropping cube '" + cubeName + "'");
        // load projects before remove cube from project

        // delete cube instance and cube desc
        CubeInstance cube = getCube(cubeName);

        // remove cube and update cache
        crud.delete(cube);
        Cuboid.clearCache(cube);

        if (deleteDesc && cube.getDescriptor() != null) {
            CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
        }

        // delete cube from project
        ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);

        return cube;
    }
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
@Override
public Tuple2<ByteArray, Object[]> call(Row row) throws Exception {
    if (initialized == false) {
        synchronized (SparkCubingByLayer.class) {
            if (initialized == false) {
                KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
                CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
                CubeDesc cubeDesc = cubeInstance.getDescriptor();
                CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
                CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                        EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
                long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
                Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
                baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                        AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                        MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
                initialized = true;
            }
        }
    }
    String[] rowArray = rowToArray(row);
    baseCuboidBuilder.resetAggrs();
    byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
    Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
    return new Tuple2<>(new ByteArray(rowKey), result);
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount,
        Map<Long, Long> rowCountMap, Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap,
        Set<Long> currentCuboidSet) {
    Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L
            : hitFrequencyMap.get(cuboidId);
    float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount;
    long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
            : queryMatchMap.get(cuboidId);
    boolean ifExist = currentCuboidSet.contains(cuboidId);
    long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId);

    NodeInfo node = new NodeInfo();
    node.setId(cuboidId);
    node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount));
    node.setQueryCount(queryCount);
    node.setQueryRate(queryRate);
    node.setExactlyMatchCount(queryExactlyMatchCount);
    node.setExisted(ifExist);
    node.setRowCount(rowCount);
    return node;
}
 

import org.apache.kylin.cube.cuboid.Cuboid; //导入依赖的package包/类
public static long estimatedCubeSize(String cubeName, long[] cardinality) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cubeInstance.getDescriptor();

    CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);

    long totalSpace = 0;

    while (!cuboidQueue.isEmpty()) {
        long cuboidID = cuboidQueue.pop();
        Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
        for (Long sc : spanningCuboid) {
            cuboidQueue.push(sc);
        }

        totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
    }
    return totalSpace;
}