Java BytesUtil.readUnsigned方法代码示例

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
public byte[] getFirstValue() {
    int nodeOffset = headSize;
    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
    while (true) {
        int valueLen = BytesUtil.readUnsigned(trieBytes, nodeOffset + firstByteOffset - 1, 1);
        bytes.write(trieBytes, nodeOffset + firstByteOffset, valueLen);
        if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
            break;
        }
        nodeOffset = headSize + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
        if (nodeOffset == headSize) {
            break;
        }
    }
    return bytes.toByteArray();
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    final TblColRef literalCol = getRawColumn(function);
    final Dictionary<String> rawColDict = dictionaryMap.get(literalCol);
    final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol) : -1;

    return new IAdvMeasureFiller() {
        private List<ByteArray> rawList;
        private Iterator<ByteArray> rawIterator;
        private int expectRow = 0;

        @SuppressWarnings("unchecked")
        @Override
        public void reload(Object measureValue) {
            this.rawList = (List<ByteArray>) measureValue;
            this.rawIterator = rawList.iterator();
            this.expectRow = 0;
        }

        @Override
        public int getNumOfRows() {
            return rawList.size();
        }

        @Override
        public void fillTuple(Tuple tuple, int row) {
            if (expectRow++ != row)
                throw new IllegalStateException();

            ByteArray raw = rawIterator.next();
            int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length());
            String colValue = rawColDict.getValueFromId(key);
            tuple.setDimensionValue(literalTupleIdx, colValue);
        }
    };
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public String decode(byte[] bytes, int offset, int len) {
    int id = BytesUtil.readUnsigned(bytes, offset, len);
    try {
        String value = dict.getValueFromId(id);
        return value;
    } catch (IllegalArgumentException e) {
        logger.error("Can't get dictionary value from " + dict + " (id = " + id + ")");
        return "";
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
/**
 * returns a code point from [0, nValues), preserving order of value
 *
 * @param n            -- the offset of current node
 * @param inp          -- input value bytes to lookup
 * @param o            -- offset in the input value bytes matched so far
 * @param inpEnd       -- end of input
 * @param roundingFlag -- =0: return -1 if not found
 *                     -- <0: return closest smaller if not found, return -1
 *                     -- >0: return closest bigger if not found, return nValues
 */
private int lookupSeqNoFromValue(int n, byte[] inp, int o, int inpEnd, int roundingFlag) {
    while (true) {
        // match the current node
        int p = n + firstByteOffset; // start of node's value
        int end = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1); // end of node's value
        for (; p < end && o < inpEnd; p++, o++) { // note matching start from [0]
            if (trieBytes[p] != inp[o]) {
                return -1; // mismatch
            }
        }

        // node completely matched, is input all consumed?
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
        if (o == inpEnd) {
            return p == end && isEndOfValue ? BytesUtil.readUnsigned(trieBytes, end, sizeOfId) : -1;
        }

        // find a child to continue
        int c = headSize + (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        if (c == headSize) // has no children
            return -1;
        byte inpByte = inp[o];
        int comp;
        while (true) {
            p = c + firstByteOffset;
            comp = BytesUtil.compareByteUnsigned(trieBytes[p], inpByte);
            if (comp == 0) { // continue in the matching child, reset n and loop again
                n = c;
                break;
            } else if (comp < 0) { // try next child
                if (checkFlag(c, BIT_IS_LAST_CHILD))
                    return -1;
                c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1) + (checkFlag(c, BIT_IS_END_OF_VALUE) ? sizeOfId : 0);
            } else { // children are ordered by their first value byte
                return -1;
            }
        }
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private AppendDictNode rebuildTrieTreeR(int n, AppendDictNode parent) {
    AppendDictNode root = null;
    while (true) {
        int p = n + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);

        byte[] value = new byte[parLen];
        System.arraycopy(trieBytes, p, value, 0, parLen);

        AppendDictNode node = new AppendDictNode(value, isEndOfValue);
        if (isEndOfValue) {
            int id = BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
            node.id = id;
        }

        if (parent == null) {
            root = node;
        } else {
            parent.addChild(node);
        }

        if (childOffset != 0) {
            rebuildTrieTreeR(childOffset + headSize, node);
        }

        if (checkFlag(n, BIT_IS_LAST_CHILD)) {
            break;
        } else {
            n += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
        }
    }
    return root;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private void init(byte[] trieBytes) {
    this.trieBytes = trieBytes;
    if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
        throw new IllegalArgumentException("Wrong file type (magic does not match)");

    try {
        DataInputStream headIn = new DataInputStream(//
                new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
        this.headSize = headIn.readShort();
        this.bodyLen = headIn.readInt();
        this.sizeChildOffset = headIn.read();
        this.sizeNoValuesBeneath = headIn.read();
        this.baseId = headIn.readShort();
        this.maxValueLength = headIn.readShort();
        if (maxValueLength < 0) {
            throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
                    + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
        }

        String converterName = headIn.readUTF();
        if (converterName.isEmpty() == false)
            setConverterByName(converterName);

        this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
        this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
        this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
        this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
        enableCache();
    } catch (Exception e) {
        if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
/**
 * returns a code point from [0, nValues), preserving order of value, or -1
 * if not found
 *
 * @param n           -- the offset of current node
 * @param seq         -- the code point under track
 * @param returnValue -- where return value is written to
 */
private int lookupValueFromSeqNo(int n, int seq, byte[] returnValue, int offset) {
    int o = offset;
    while (true) {
        // write current node value
        int p = n + firstByteOffset;
        int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        System.arraycopy(trieBytes, p, returnValue, o, len);
        o += len;

        // if the value is ended
        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
        if (isEndOfValue) {
            seq--;
            if (seq < 0)
                return o - offset;
        }

        // find a child to continue
        int c = getChildOffset(n);
        if (c == headSize) // has no children
            return -1; // no child? corrupted dictionary!
        int nValuesBeneath;
        while (true) {
            nValuesBeneath = BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
            if (seq - nValuesBeneath < 0) { // value is under this child, reset n and loop again
                n = c;
                break;
            } else { // go to next child
                seq -= nValuesBeneath;
                if (checkFlag(c, BIT_IS_LAST_CHILD))
                    return -1; // no more child? corrupted dictionary!
                p = c + firstByteOffset;
                c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
            }
        }
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public int getPartition(SelfDefineSortableKey skey, Text value, int numReduceTasks) {
    Text key = skey.getText();
    if (key.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER) {
        Long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
        return reducerMapping.getReducerIdForCuboidRowCount(cuboidId);
    } else if (key.getBytes()[0] == FactDistinctColumnsReducerMapping.MARK_FOR_PARTITION_COL) {
        return reducerMapping.getReducerIdForDatePartitionColumn();
    } else {
        return BytesUtil.readUnsigned(key.getBytes(), 0, 1);
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public Integer deserialize(ByteBuffer in) {
    return BytesUtil.readUnsigned(in, 4);
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public Object deserialize(ByteBuffer in) {
    int id = BytesUtil.readUnsigned(in, dict.getSizeOfId());
    return dict.getValueFromId(id);
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
public boolean doCheck() {
    int offset = headSize;
    HashSet<Integer> parentSet = new HashSet<>();
    boolean lastChild = false;

    while (offset < trieBytes.length) {
        if (lastChild) {
            boolean contained = parentSet.remove(offset - headSize);
            // Can't find parent, the data is corrupted
            if (!contained) {
                return false;
            }
            lastChild = false;
        }
        int p = offset + firstByteOffset;
        int childOffset = (int) (BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
        int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
        boolean isEndOfValue = checkFlag(offset, BIT_IS_END_OF_VALUE);

        // Copy value overflow, the data is corrupted
        if (trieBytes.length < p + parLen) {
            return false;
        }

        // Check id is fine
        if (isEndOfValue) {
            BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
        }

        // Record it if has children
        if (childOffset != 0) {
            parentSet.add(childOffset);
        }

        // brothers done, move to next parent
        if (checkFlag(offset, BIT_IS_LAST_CHILD)) {
            lastChild = true;
        }

        // move to next node
        offset += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
    }

    // ParentMap is empty, meaning all nodes has parent, the data is correct
    return parentSet.isEmpty();
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public int getPartition(SelfDefineSortableKey skey, NullWritable value, int numReduceTasks) {
    return BytesUtil.readUnsigned(skey.getText().getBytes(), 0, 1);
}