Java RowKeyDesc类代码示例

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.setSignature(desc.calculateSignature());
    desc.updateRandomUuid();
    return desc;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
    long groupMask = aggrGroupMask.groupMask;
    int n = mathCalcCuboidCount_combination(rowkey, groupMask);
    n -= 2; // exclude group all 1 and all 0

    long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
    if (nonUniqueMask > 0) {
        // exclude duplicates caused by non-unique columns
        // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
        n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
    }

    if (hasTail) {
        n *= 2; // tail being 1 and 0
        n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
    }

    return n;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
    if (colMask == 0) // no column selected
        return 0;

    int count = 1;

    for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
        long hierBits = colMask & hierMask.fullMask;
        if (hierBits != 0) {
            count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
            colMask &= ~hierBits;
        }
    }

    count *= Math.pow(2, Long.bitCount(colMask));

    return count;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private void generateZeroTailBase(long cuboid, Collection<Long> result) {
    RowKeyDesc rowkey = cubeDef.getRowkey();

    long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();

    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
            long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
            if (zeroTail > 0 && zeroTail != cuboid) {
                result.add(zeroTail);
            }
        }
        if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
            break;
    }
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
public Collection<Long> findSmallerSibling(long cuboid) {
    if (!Cuboid.isValid(cubeDef, cuboid)) {
        return Collections.emptyList();
    }

    RowKeyDesc rowkey = cubeDef.getRowkey();

    // do combination in all related groups
    long groupAllBitMask = 0;
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((mask.groupMask & cuboid) > 0) {
            groupAllBitMask |= mask.groupMask;
        }
    }

    long groupBitValue = cuboid & groupAllBitMask;
    long leftBitValue = cuboid & ~groupAllBitMask;
    long[] groupOneBits = bits(groupAllBitMask);

    Collection<Long> siblings = new HashSet<Long>();
    combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
    return siblings;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
public static boolean isValid(CubeDesc cube, long cuboidID) {
    RowKeyDesc rowkey = cube.getRowkey();

    if (cuboidID < 0) {
        throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
    }

    if (checkBaseCuboid(rowkey, cuboidID)) {
        return true;
    }

    if (checkMandatoryColumns(rowkey, cuboidID) == false) {
        return false;
    }

    if (checkAggregationGroup(rowkey, cuboidID) == false) {
        return false;
    }

    if (checkHierarchy(rowkey, cuboidID) == false) {
        return false;
    }

    return true;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
    List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
    // if no hierarchy defined in metadata
    if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
        return true;
    }

    hier: for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
        long result = cuboidID & hierarchyMasks.fullMask;
        if (result > 0) {
            // if match one of the hierarchy constrains, return true;
            for (long mask : hierarchyMasks.allMasks) {
                if (result == mask) {
                    continue hier;
                }
            }
            return false;
        }
    }
    return true;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    List<TblColRef> columns = baseCuboid.getColumns();

    ArrayList<Integer> factDictCols = new ArrayList<Integer>();
    RowKeyDesc rowkey = cubeDesc.getRowkey();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    for (int i = 0; i < columns.size(); i++) {
        TblColRef col = columns.get(i);
        if (rowkey.isUseDictionary(col) == false)
            continue;

        String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
        if (cubeDesc.getModel().isFactTable(scanTable)) {
            factDictCols.add(i);
        }
    }
    this.factDictCols = new int[factDictCols.size()];
    for (int i = 0; i < factDictCols.size(); i++)
        this.factDictCols[i] = factDictCols.get(i);

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }
    if(rcd.length == 0){
        context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
        return;       	
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
        }
        if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
        }
    }

}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
public static int mathCalcCuboidCount(CubeDesc cube) {
    int result = 1; // 1 for base cuboid

    RowKeyDesc rowkey = cube.getRowkey();
    AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
    for (int i = 0; i < aggrGroupMasks.length; i++) {
        boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
        result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
    }

    return result;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private Collection<Long> generateChildren(long cuboid) {
    Collection<Long> result = new HashSet<Long>();

    // generate zero tail cuboid -- the one with all 1 in the first
    // aggregation group and all 0 for the rest bits
    generateZeroTailBase(cuboid, result);

    RowKeyDesc rowkey = cubeDef.getRowkey();
    long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if (belongTo(cuboidWithoutMandatory, mask) == false)
            continue;

        long[] groupOneBitMasks = mask.groupOneBitMasks;
        for (int i = 0; i < groupOneBitMasks.length; i++) {
            long oneBit = groupOneBitMasks[i];
            if ((cuboid & oneBit) == 0)
                continue;

            long child = cuboid ^ oneBit;
            if (Cuboid.isValid(cubeDef, child)) {
                result.add(child);
            }
        }

        if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
            break;
    }

    return result;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
    long baseCuboidId = rowkey.getFullMask();
    if (cuboidID > baseCuboidId) {
        throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
    }
    return baseCuboidId == cuboidID;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
    long mandatoryColumnMask = rowkey.getMandatoryColumnMask();

    // note the all-zero cuboid (except for mandatory) is not valid
    if (cuboidID <= mandatoryColumnMask)
        return false;

    return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
}
 

import org.apache.kylin.cube.model.RowKeyDesc; //导入依赖的package包/类
private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
    long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
    long leftover;
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
            leftover = cuboidWithoutMandatory & ~mask.groupMask;
            return leftover == 0 || leftover == mask.leftoverMask;
        }
    }

    leftover = cuboidWithoutMandatory & rowkey.getTailMask();
    return leftover == 0 || leftover == rowkey.getTailMask();
}