Java LookupStringTable类代码示例

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
    long ts = System.currentTimeMillis();

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(cubeSeg.getCubeInstance().getConfig());
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(cubeSeg.getCubeInstance().getConfig());

    String tableName = join.getPKSide().getTableIdentity();
    String[] pkCols = join.getPrimaryKey();
    String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
    if (snapshotResPath == null)
        throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

    try {
        SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotResPath);
        TableDesc tableDesc = metaMgr.getTableDesc(tableName, cubeSegment.getProject());
        EnhancedStringLookupTable enhancedStringLookupTable = new EnhancedStringLookupTable(tableDesc, pkCols, snapshot);
        logger.info("Time to get lookup up table for {} is {} ", join.getPKSide().getTableName(), (System.currentTimeMillis() - ts));
        return enhancedStringLookupTable;
    } catch (IOException e) {
        throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
    }
}
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null)
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isExtendedColumn(derived)) {
        throw new CubeDesc.CannotFilterExtendedColumnException(derived);
    }
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    LookupStringTable lookup = cubeDesc.getHostInfo(derived).type == CubeDesc.DeriveType.PK_FK ? null
            : getLookupStringTableForDerived(derived, hostInfo);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(translatedFilter, collector);
    }
    return translatedFilter;
}
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {

            String tableName = join.getPKSide().getTableIdentity();
            String[] pkCols = join.getPrimaryKey();
            String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
            if (snapshotResPath == null)
                throw new IllegalStateException("No snapshot for table '" + tableName + "' found on cube segment"
                        + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

            try {
                SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
                TableDesc tableDesc = getTableManager().getTableDesc(tableName, cubeSegment.getProject());
                return new LookupStringTable(tableDesc, pkCols, snapshot);
            } catch (IOException e) {
                throw new IllegalStateException(
                        "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
            }
        }
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {

        String tableName = dim.getTable();
        String[] pkCols = dim.getJoin().getPrimaryKey();
        String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
        if (snapshotResPath == null)
            throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

        try {
            SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
            TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
            return new LookupStringTable(tableDesc, pkCols, snapshot);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
        }
    }
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public static IDerivedColumnFiller newDerivedColumnFiller(List<TblColRef> rowColumns, TblColRef[] hostCols, DeriveInfo deriveInfo, TupleInfo tupleInfo, CubeManager cubeMgr, CubeSegment cubeSegment) {

        int[] hostIndex = new int[hostCols.length];
        for (int i = 0; i < hostCols.length; i++) {
            hostIndex[i] = rowColumns.indexOf(hostCols[i]);
        }
        String[] derivedFieldNames = new String[deriveInfo.columns.length];
        for (int i = 0; i < deriveInfo.columns.length; i++) {
            derivedFieldNames[i] = tupleInfo.getFieldName(deriveInfo.columns[i]);
        }

        switch (deriveInfo.type) {
        case LOOKUP:
            LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSegment, deriveInfo.dimension);
            return new LookupFiller(hostIndex, lookupTable, deriveInfo, derivedFieldNames);
        case PK_FK:
            // composite key are split, see CubeDesc.initDimensionColumns()
            return new PKFKFiller(hostIndex[0], derivedFieldNames[0]);
        default:
            throw new IllegalArgumentException();
        }
    }
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null || compf.getValues().isEmpty())
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(compf, collector);
    }
    return translatedFilter;
}
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public LookupFiller(int[] hostIndex, LookupStringTable lookupTable, DeriveInfo deriveInfo, String[] derivedFieldNames) {
    this.hostIndex = hostIndex;
    this.hostLen = hostIndex.length;
    this.lookupKey = new Array<String>(new String[hostLen]);
    this.lookupTable = lookupTable;
    this.derivedIndex = new int[deriveInfo.columns.length];
    this.derivedLen = derivedIndex.length;
    this.derivedFieldNames = derivedFieldNames;

    for (int i = 0; i < derivedLen; i++) {
        derivedIndex[i] = deriveInfo.columns[i].getColumn().getZeroBasedIndex();
    }
}
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {

        TblColRef derivedCol = compf.getColumn();
        TblColRef[] hostCols = hostInfo.columns;
        TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns();

        if (hostInfo.type == DeriveType.PK_FK) {
            assert hostCols.length == 1;
            CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
            newComp.addChild(new ColumnTupleFilter(hostCols[0]));
            newComp.addChild(new ConstantTupleFilter(compf.getValues()));
            return new Pair<TupleFilter, Boolean>(newComp, false);
        }

        assert hostInfo.type == DeriveType.LOOKUP;
        assert hostCols.length == pkCols.length;

        int di = derivedCol.getColumnDesc().getZeroBasedIndex();
        int[] pi = new int[pkCols.length];
        int hn = hostCols.length;
        for (int i = 0; i < hn; i++) {
            pi[i] = pkCols[i].getColumnDesc().getZeroBasedIndex();
        }

        Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
        SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
        for (String[] row : lookup.getAllRows()) {
            tuple.value = row[di];
            if (compf.evaluate(tuple, FilterCodeSystemFactory.getFilterCodeSystem(derivedCol.getColumnDesc().getType()))) {
                collect(row, pi, satisfyingHostRecords);
            }
        }

        TupleFilter translated;
        boolean loosened;
        if (satisfyingHostRecords.size() > KylinConfig.getInstanceFromEnv().getDerivedInThreshold()) {
            logger.info("Deciding to loosen filter on derived filter as host candidates number {} exceeds threshold {}", //
                    satisfyingHostRecords.size(), KylinConfig.getInstanceFromEnv().getDerivedInThreshold()
            );
            translated = buildRangeFilter(hostCols, satisfyingHostRecords);
            loosened = true;
        } else {
            translated = buildInFilter(hostCols, satisfyingHostRecords);
            loosened = false;
        }

        return new Pair<TupleFilter, Boolean>(translated, loosened);
    }
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
@SuppressWarnings("unchecked")
protected LookupStringTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    return cubeMgr.getLookupTable(seg, hostInfo.join);
}
 

import org.apache.kylin.dict.lookup.LookupStringTable; //导入依赖的package包/类
public static Pair<TupleFilter, Boolean> translate(LookupStringTable lookup, DeriveInfo hostInfo, CompareTupleFilter compf) {

        TblColRef derivedCol = compf.getColumn();
        TblColRef[] hostCols = hostInfo.columns;
        TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns();

        if (hostInfo.type == DeriveType.PK_FK) {
            assert hostCols.length == 1;
            CompareTupleFilter newComp = new CompareTupleFilter(compf.getOperator());
            newComp.addChild(new ColumnTupleFilter(hostCols[0]));
            newComp.addChild(new ConstantTupleFilter(compf.getValues()));
            return new Pair<TupleFilter, Boolean>(newComp, false);
        }

        assert hostInfo.type == DeriveType.LOOKUP;
        assert hostCols.length == pkCols.length;

        int di = derivedCol.getColumn().getZeroBasedIndex();
        int[] pi = new int[pkCols.length];
        int hn = hostCols.length;
        for (int i = 0; i < hn; i++) {
            pi[i] = pkCols[i].getColumn().getZeroBasedIndex();
        }

        Set<Array<String>> satisfyingHostRecords = Sets.newHashSet();
        SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
        for (String[] row : lookup.getAllRows()) {
            tuple.value = row[di];
            if (compf.evaluate(tuple)) {
                collect(row, pi, satisfyingHostRecords);
            }
        }

        TupleFilter translated;
        boolean loosened;
        if (satisfyingHostRecords.size() > IN_THRESHOLD) {
            translated = buildRangeFilter(hostCols, satisfyingHostRecords);
            loosened = true;
        } else {
            translated = buildInFilter(hostCols, satisfyingHostRecords);
            loosened = false;
        }

        return new Pair<TupleFilter, Boolean>(translated, loosened);
    }