Java DeriveInfo类代码示例

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
protected Set<TblColRef> expandDerived(Collection<TblColRef> cols, Set<TblColRef> derivedPostAggregation) {
    Set<TblColRef> expanded = Sets.newHashSet();
    for (TblColRef col : cols) {
        if (cubeDesc.hasHostColumn(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            for (TblColRef hostCol : hostInfo.columns) {
                expanded.add(hostCol);
                if (hostInfo.isOneToOne == false)
                    derivedPostAggregation.add(hostCol);
            }
        } else {
            expanded.add(col);
        }
    }
    return expanded;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
@SuppressWarnings("unchecked")
protected Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Set<CompareTupleFilter> compareTupleFilterSet = findSingleValuesCompFilters(filter);

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (CompareTupleFilter compFilter : compareTupleFilterSet) {
        TblColRef tblColRef = compFilter.getColumn();
        if (cubeDesc.isExtendedColumn(tblColRef)) {
            throw new CubeDesc.CannotFilterExtendedColumnException(tblColRef);
        }
        if (cubeDesc.isDerived(compFilter.getColumn())) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(tblColRef);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(compFilter.getColumn());
        }
    }
    return resultD;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null)
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isExtendedColumn(derived)) {
        throw new CubeDesc.CannotFilterExtendedColumnException(derived);
    }
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    LookupStringTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
            : getLookupStringTableForDerived(derived, hostInfo);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(translatedFilter, collector);
    }
    return translatedFilter;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
public static IDerivedColumnFiller newDerivedColumnFiller(List<TblColRef> rowColumns, TblColRef[] hostCols, DeriveInfo deriveInfo, TupleInfo tupleInfo, CubeManager cubeMgr, CubeSegment cubeSegment) {

        int[] hostIndex = new int[hostCols.length];
        for (int i = 0; i < hostCols.length; i++) {
            hostIndex[i] = rowColumns.indexOf(hostCols[i]);
        }
        String[] derivedFieldNames = new String[deriveInfo.columns.length];
        for (int i = 0; i < deriveInfo.columns.length; i++) {
            derivedFieldNames[i] = tupleInfo.getFieldName(deriveInfo.columns[i]);
        }

        switch (deriveInfo.type) {
        case LOOKUP:
            LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSegment, deriveInfo.dimension);
            return new LookupFiller(hostIndex, lookupTable, deriveInfo, derivedFieldNames);
        case PK_FK:
            // composite key are split, see CubeDesc.initDimensionColumns()
            return new PKFKFiller(hostIndex[0], derivedFieldNames[0]);
        default:
            throw new IllegalArgumentException();
        }
    }
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private Set<TblColRef> expandDerived(Collection<TblColRef> cols, Set<TblColRef> derivedPostAggregation) {
    Set<TblColRef> expanded = Sets.newHashSet();
    for (TblColRef col : cols) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            for (TblColRef hostCol : hostInfo.columns) {
                expanded.add(hostCol);
                if (hostInfo.isOneToOne == false)
                    derivedPostAggregation.add(hostCol);
            }
        } else {
            expanded.add(col);
        }
    }
    return expanded;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null || compf.getValues().isEmpty())
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(compf, collector);
    }
    return translatedFilter;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo) {
    int pkfkCount = 0;
    int lookupCount = 0;
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        for (DeriveInfo deriveInfo : entry.getValue()) {
            if (deriveInfo.type == DeriveType.PK_FK)
                pkfkCount++;
            if (deriveInfo.type == DeriveType.LOOKUP)
                lookupCount++;
        }
    }
    return Pair.newPair(pkfkCount, lookupCount);
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
public LookupFiller(int[] hostIndex, LookupStringTable lookupTable, DeriveInfo deriveInfo, String[] derivedFieldNames) {
    this.hostIndex = hostIndex;
    this.hostLen = hostIndex.length;
    this.lookupKey = new Array<String>(new String[hostLen]);
    this.lookupTable = lookupTable;
    this.derivedIndex = new int[deriveInfo.columns.length];
    this.derivedLen = derivedIndex.length;
    this.derivedFieldNames = derivedFieldNames;

    for (int i = 0; i < derivedLen; i++) {
        derivedIndex[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex();
    }
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private TupleInfo buildTupleInfo(Cuboid cuboid) {
    TupleInfo info = new TupleInfo();
    int index = 0;
    rowKeyDecoder.setCuboid(cuboid);
    List<TblColRef> rowColumns = rowKeyDecoder.getColumns();
    List<String> colNames = rowKeyDecoder.getNames(context.getAliasMap());
    for (int i = 0; i < rowColumns.size(); i++) {
        TblColRef column = rowColumns.get(i);
        if (!dimensions.contains(column)) {
            continue;
        }
        // add normal column
        info.setField(colNames.get(i), rowColumns.get(i), rowColumns.get(i).getType().getName(), index++);
    }

    // derived columns and filler
    Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cubeSeg.getCubeDesc().getHostToDerivedInfo(rowColumns, null);
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        TblColRef[] hostCols = entry.getKey().data;
        for (DeriveInfo deriveInfo : entry.getValue()) {
            // mark name for each derived field
            for (TblColRef derivedCol : deriveInfo.columns) {
                String derivedField = getFieldName(derivedCol, context.getAliasMap());
                info.setField(derivedField, derivedCol, derivedCol.getType().getName(), index++);
            }
            // add filler
            info.addDerivedColumnFiller(Tuple.newDerivedColumnFiller(rowColumns, hostCols, deriveInfo, info, CubeManager.getInstance(this.cube.getConfig()), cubeSeg));
        }
    }

    for (RowValueDecoder rowValueDecoder : this.rowValueDecoders) {
        List<String> names = rowValueDecoder.getNames();
        MeasureDesc[] measures = rowValueDecoder.getMeasures();
        for (int i = 0; i < measures.length; i++) {
            String dataType = measures[i].getFunction().getSQLType();
            info.setField(names.get(i), null, dataType, index++);
        }
    }
    return info;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return (Set<TblColRef>) Collections.EMPTY_SET;
    }

    Set<TblColRef> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
                result.add(compFilter.getColumn());
            }
        }
    }

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (TblColRef col : result) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(col);
        }
    }
    return resultD;
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
private void collectColumns(TblColRef col, Set<TblColRef> collector) {
    if (cubeDesc.isDerived(col)) {
        DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
        for (TblColRef h : hostInfo.columns)
            collector.add(h);
    } else {
        collector.add(col);
    }
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {

        TblColRef derivedCol = compf.getColumn();
        TblColRef[] hostCols = hostInfo.columns;
        TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns();

        if (hostInfo.type == DeriveType.PK_FK) {
            assert hostCols.length == 1;
            CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
            newComp.addChild(new ColumnTupleFilter(hostCols[0]));
            newComp.addChild(new ConstantTupleFilter(compf.getValues()));
            return new Pair<TupleFilter, Boolean>(newComp, false);
        }

        assert hostInfo.type == DeriveType.LOOKUP;
        assert hostCols.length == pkCols.length;

        int di = derivedCol.getColumnDesc().getZeroBasedIndex();
        int[] pi = new int[pkCols.length];
        int hn = hostCols.length;
        for (int i = 0; i < hn; i++) {
            pi[i] = pkCols[i].getColumnDesc().getZeroBasedIndex();
        }

        Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
        SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
        for (String[] row : lookup.getAllRows()) {
            tuple.value = row[di];
            if (compf.evaluate(tuple, FilterCodeSystemFactory.getFilterCodeSystem(derivedCol.getColumnDesc().getType()))) {
                collect(row, pi, satisfyingHostRecords);
            }
        }

        TupleFilter translated;
        boolean loosened;
        if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
            logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}", //
                    satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
            );
            translated = buildRangeFilter(hostCols, satisfyingHostRecords);
            loosened = true;
        } else {
            translated = buildInFilter(hostCols, satisfyingHostRecords);
            loosened = false;
        }

        return new Pair<TupleFilter, Boolean>(translated, loosened);
    }
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
        Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo returnTupleInfo) {
    this.cubeSeg = cubeSeg;
    this.cuboid = cuboid;
    this.gtColIdx = gtColIdx;
    this.tupleInfo = returnTupleInfo;
    this.derivedColFillers = Lists.newArrayList();

    nSelectedDims = selectedDimensions.size();
    tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];

    // measure types don't have this many, but aligned length make programming easier
    measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];

    advMeasureFillers = Lists.newArrayListWithCapacity(1);
    advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);

    ////////////

    int i = 0;

    // pre-calculate dimension index mapping to tuple
    for (TblColRef dim : selectedDimensions) {
        tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
        i++;
    }

    for (FunctionDesc metric : selectedMetrics) {
        if (metric.needRewrite()) {
            String rewriteFieldName = metric.getRewriteFieldName();
            tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
        } else {
            // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
            TblColRef col = metric.getParameter().getColRefs().get(0);
            tupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
        }

        MeasureType<?> measureType = metric.getMeasureType();
        if (measureType.needAdvancedTupleFilling()) {
            Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
            advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
            advMeasureIndexInGTValues.add(i);
        } else {
            measureTypes[i] = measureType;
        }

        i++;
    }

    // prepare derived columns and filler
    Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboid.getColumns(), null);
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        TblColRef[] hostCols = entry.getKey().data;
        for (DeriveInfo deriveInfo : entry.getValue()) {
            IDerivedColumnFiller filler = newDerivedColumnFiller(hostCols, deriveInfo);
            if (filler != null) {
                derivedColFillers.add(filler);
            }
        }
    }
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
@SuppressWarnings("unchecked")
protected LookupStringTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    return cubeMgr.getLookupTable(seg, hostInfo.join);
}
 

import org.apache.kylin.cube.model.CubeDesc.DeriveInfo; //导入依赖的package包/类
public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {

        TblColRef derivedCol = compf.getColumn();
        TblColRef[] hostCols = hostInfo.columns;
        TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns();

        if (hostInfo.type == DeriveType.PK_FK) {
            assert hostCols.length == 1;
            CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
            newComp.addChild(new ColumnTupleFilter(hostCols[0]));
            newComp.addChild(new ConstantTupleFilter(compf.getValues()));
            return new Pair<TupleFilter, Boolean>(newComp, false);
        }

        assert hostInfo.type == DeriveType.LOOKUP;
        assert hostCols.length == pkCols.length;

        int di = derivedCol.getColumn().getZeroBasedIndex();
        int[] pi = new int[pkCols.length];
        int hn = hostCols.length;
        for (int i = 0; i < hn; i++) {
            pi[i] = pkCols[i].getColumn().getZeroBasedIndex();
        }

        Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
        SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
        for (String[] row : lookup.getAllRows()) {
            tuple.value = row[di];
            if (compf.evaluate(tuple)) {
                collect(row, pi, satisfyingHostRecords);
            }
        }

        TupleFilter translated;
        boolean loosened;
        if (satisfyingHostRecords.size() > IN_THRESHOLD) {
            translated = buildRangeFilter(hostCols, satisfyingHostRecords);
            loosened = true;
        } else {
            translated = buildInFilter(hostCols, satisfyingHostRecords);
            loosened = false;
        }

        return new Pair<TupleFilter, Boolean>(translated, loosened);
    }