本文整理匯總了Java中org.apache.kylin.cube.cuboid.Cuboid.getBaseCuboidId方法的典型用法代碼示例。如果您正苦於以下問題:Java Cuboid.getBaseCuboidId方法的具體用法?Java Cuboid.getBaseCuboidId怎麽用?Java Cuboid.getBaseCuboidId使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.kylin.cube.cuboid.Cuboid
的用法示例。
在下文中一共展示了Cuboid.getBaseCuboidId方法的15個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Java代碼示例。
示例1: doSetup
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
protected void doSetup(Context context) throws IOException {
Configuration conf = context.getConfiguration();
bindCurrentConfiguration(conf);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
cube = CubeManager.getInstance(config).getCube(cubeName);
cubeSeg = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID));
cubeDesc = cube.getDescriptor();
baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
dictCols = Lists.newArrayList(cubeDesc.getAllColumnsNeedDictionaryBuilt());
flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat();
intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc);
dictionaryColumnIndex = new int[dictCols.size()];
for (int i = 0; i < dictCols.size(); i++) {
TblColRef colRef = dictCols.get(i);
int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef);
dictionaryColumnIndex[i] = columnIndexOnFlatTbl;
}
reducerMapping = new FactDistinctColumnsReducerMapping(cube,
conf.getInt(BatchConstants.CFG_HLL_REDUCER_NUM, 1));
}
示例2: getBuildLevel
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
public int getBuildLevel() {
int ret = 1;//base cuboid => partial cube root
if (this.getPartialCubeFullMask() == Cuboid.getBaseCuboidId(cubeDesc)) {
ret -= 1;//if partial cube's root is base cuboid, then one round less agg
}
ret += getNormalDims().size();
for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
ret += hierarchyMask.allMasks.length;
}
for (Long joint : joints) {
if ((joint & this.getHierarchyDimsMask()) == 0) {
ret += 1;
}
}
return ret;
}
示例3: InMemCubeBuilder
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
Map<TblColRef, Dictionary<String>> dictionaryMap) {
super(cuboidScheduler, flatDesc, dictionaryMap);
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
List<String> metricsAggrFuncsList = Lists.newArrayList();
for (int i = 0; i < measureCount; i++) {
MeasureDesc measureDesc = measureDescs[i];
metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
}
this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}
示例4: call
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
public Tuple2<ByteArray, Object[]> call(Row row) throws Exception {
if (initialized == false) {
synchronized (SparkCubingByLayer.class) {
if (initialized == false) {
KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
initialized = true;
}
}
}
String[] rowArray = rowToArray(row);
baseCuboidBuilder.resetAggrs();
byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
return new Tuple2<>(new ByteArray(rowKey), result);
}
示例5: estimatedCubeSize
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
public static long estimatedCubeSize(String cubeName, long[] cardinality) {
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeManager cubeManager = CubeManager.getInstance(config);
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
LinkedList<Long> cuboidQueue = new LinkedList<Long>();
cuboidQueue.push(baseCuboid);
long totalSpace = 0;
while (!cuboidQueue.isEmpty()) {
long cuboidID = cuboidQueue.pop();
Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
for (Long sc : spanningCuboid) {
cuboidQueue.push(sc);
}
totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
}
return totalSpace;
}
示例6: doSetup
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
protected void doSetup(Mapper.Context context) throws IOException {
super.doSetup(context);
long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
GTInfo gtInfo = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, baseCuboid),
new CubeDimEncMap(cubeDesc, dictionaryMap));
keyValueBuffer = ByteBuffer.allocate(gtInfo.getMaxRecordLength());
keyOffset = cubeSegment.getRowKeyPreambleSize();
}
示例7: printCuboidInfoTreeEntry
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
if (cuboidScheduler == null) {
throw new UnsupportedOperationException("cuboid scheduler is null");
}
long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
int dimensionCount = Long.bitCount(baseCuboid);
printCuboidInfoTree(-1L, baseCuboid, cuboidScheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
}
示例8: checkMandatoryColumns
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
private boolean checkMandatoryColumns(long cuboidID) {
if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
return false;
} else {
//base cuboid is always valid
if (cuboidID == Cuboid.getBaseCuboidId(cubeDesc)) {
return true;
}
//cuboid with only mandatory columns maybe valid
return isMandatoryOnlyValid || (cuboidID & ~mandatoryColumnMask) != 0;
}
}
示例9: testEncodeAndDecodeWithUtf8
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
CubeDesc cubeDesc = cube.getDescriptor();
String[] data = new String[8];
data[0] = "2012-12-15";
data[1] = "11848";
data[2] = "Health & Beauty";
data[3] = "Fragrances";
data[4] = "Women";
data[5] = "刊登格式測試";// UTF-8
data[6] = "0";
data[7] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
rowKeyDecoder.decode(encodedKey);
List<String> values = rowKeyDecoder.getValues();
assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}
示例10: testEncodeWithoutSlr
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Test
public void testEncodeWithoutSlr() throws Exception {
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
// CubeSegment seg = cube.getTheOnlySegment();
CubeDesc cubeDesc = cube.getDescriptor();
// String data =
// "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
String[] data = new String[8];
data[0] = "2012-12-15";
data[1] = "11848";
data[2] = "Health & Beauty";
data[3] = "Fragrances";
data[4] = "Women";
data[5] = "FP-GTC";
data[6] = "0";
data[7] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length);
assertEquals(255, Bytes.toLong(cuboidId));
assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
示例11: testEncodeWithSlr
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Ignore
@Test
public void testEncodeWithSlr() throws Exception {
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
// CubeSegment seg = cube.getTheOnlySegment();
CubeDesc cubeDesc = cube.getDescriptor();
// String data =
// "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
String[] data = new String[9];
data[0] = "123456789";
data[1] = "2012-12-15";
data[2] = "11848";
data[3] = "Health & Beauty";
data[4] = "Fragrances";
data[5] = "Women";
data[6] = "FP-GTC";
data[7] = "0";
data[8] = "15";
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
@SuppressWarnings("unused")
byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength());
byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
assertEquals(0, Bytes.toShort(shard));
// assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
assertEquals(511, Bytes.toLong(cuboidId));
assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
示例12: testEncodeWithSlr2
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Ignore
@Test
public void testEncodeWithSlr2() throws Exception {
CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
// CubeSegment seg = cube.getTheOnlySegment();
CubeDesc cubeDesc = cube.getDescriptor();
// String data =
// "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
String[] data = new String[9];
data[0] = "123456789";
data[1] = null;
data[2] = null;
data[3] = null;
data[4] = null;
data[5] = null;
data[6] = null;
data[7] = null;
data[8] = null;
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);
byte[] encodedKey = rowKeyEncoder.encode(data);
assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
@SuppressWarnings("unused")
byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 18 + rowKeyEncoder.getHeaderLength());
byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
assertEquals(0, Bytes.toShort(shard));
//assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
assertEquals(511, Bytes.toLong(cuboidId));
assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
}
示例13: setup
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
protected void setup(Context context) throws IOException {
super.publishConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
metadataManager = MetadataManager.getInstance(config);
cube = CubeManager.getInstance(config).getCube(cubeName);
cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
cubeDesc = cube.getDescriptor();
factTableDesc = metadataManager.getTableDesc(cubeDesc.getFactTable());
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
// intermediateTableDesc = new
// JoinedFlatTableDesc(cube.getDescriptor());
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
measureCodec = new MeasureCodec(cubeDesc.getMeasures());
measures = new Object[cubeDesc.getMeasures().size()];
int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
keyBytesBuf = new byte[colCount][];
bytesSplitter = new BytesSplitter(factTableDesc.getColumns().length, 4096);
nullValue = new byte[] { (byte) '\\', (byte) 'N' };// As in Hive, null
// value is
// represented by \N
prepareJoins();
prepareMetrics();
}
示例14: setup
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
protected void setup(Context context) throws IOException {
super.publishConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
}
byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());
cube = CubeManager.getInstance(config).getCube(cubeName);
cubeDesc = cube.getDescriptor();
cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);
bytesSplitter = new BytesSplitter(200, 4096);
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
measureCodec = new MeasureCodec(cubeDesc.getMeasures());
measures = new Object[cubeDesc.getMeasures().size()];
int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
keyBytesBuf = new byte[colCount][];
initNullBytes();
}
示例15: setup
import org.apache.kylin.cube.cuboid.Cuboid; //導入方法依賴的package包/類
@Override
protected void setup(Context context) throws IOException {
super.publishConfiguration(context.getConfiguration());
Configuration conf = context.getConfiguration();
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
cube = CubeManager.getInstance(config).getCube(cubeName);
cubeDesc = cube.getDescriptor();
intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
List<TblColRef> columns = baseCuboid.getColumns();
ArrayList<Integer> factDictCols = new ArrayList<Integer>();
RowKeyDesc rowkey = cubeDesc.getRowkey();
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
for (int i = 0; i < columns.size(); i++) {
TblColRef col = columns.get(i);
if (rowkey.isUseDictionary(col) == false)
continue;
String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
if (cubeDesc.getModel().isFactTable(scanTable)) {
factDictCols.add(i);
}
}
this.factDictCols = new int[factDictCols.size()];
for (int i = 0; i < factDictCols.size(); i++)
this.factDictCols[i] = factDictCols.get(i);
schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}