Java BytesUtil.writeUnsigned方法代码示例

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public void encode(String valueStr, byte[] output, int outputOffset) {
    try {
        int id = dict.getIdFromValue(valueStr, roundingFlag);
        BytesUtil.writeUnsigned(id, output, outputOffset, fixedLen);
    } catch (IllegalArgumentException ex) {
        for (int i = outputOffset; i < outputOffset + fixedLen; i++) {
            output[i] = defaultByte;
        }
        logger.error("Can't translate value " + valueStr + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", defaultByte));
    }
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Test
public void testNormal() {
    int size = 100;
    List<ByteArray> valueList = new ArrayList<ByteArray>(size);
    for (Integer i = 0; i < size; i++) {
        ByteArray key = new ByteArray(1);
        BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
        valueList.add(key);
    }
    agg.aggregate(valueList);
    agg.aggregate(valueList);
    assertEquals(valueList.size() * 2, agg.getState().size());
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private List<ByteArray> getValueList(int size) {
    if (size == -1) {
        return null;
    }
    List<ByteArray> valueList = new ArrayList<ByteArray>(size);
    for (Integer i = 0; i < size; i++) {
        ByteArray key = new ByteArray(1);
        BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
        valueList.add(key);
    }
    return valueList;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private int build_writeNode(AppendDictNode n, int offset, boolean isLastChild, int sizeChildOffset, int sizeId, byte[] trieBytes) {
    int o = offset;

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException("Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    if (n.isEndOfValue) {
        checkValidId(n.id);
        BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
        o += sizeId;
    }

    return o;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
    int o = offset;
    if (o > _2GB)
        throw new IllegalStateException();

    // childOffset
    if (isLastChild)
        trieBytes[o] |= TrieDictionary.BIT_IS_LAST_CHILD;
    if (n.isEndOfValue)
        trieBytes[o] |= TrieDictionary.BIT_IS_END_OF_VALUE;
    o += sizeChildOffset;

    // nValuesBeneath
    BytesUtil.writeUnsigned(n.nValuesBeneath, trieBytes, o, sizeNoValuesBeneath);
    o += sizeNoValuesBeneath;

    // nValueBytes
    if (n.part.length > 255)
        throw new RuntimeException();
    BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
    o++;

    // valueBytes
    System.arraycopy(n.part, 0, trieBytes, o, n.part.length);
    o += n.part.length;

    return o;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private static byte[][] getSplitsByRegionCount(int regionCount) {
    byte[][] result = new byte[regionCount - 1][];
    for (int i = 1; i < regionCount; ++i) {
        byte[] split = new byte[RowConstants.ROWKEY_SHARDID_LEN];
        BytesUtil.writeUnsigned(i, split, 0, RowConstants.ROWKEY_SHARDID_LEN);
        result[i - 1] = split;
    }
    return result;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public void serialize(Integer value, ByteBuffer out) {
    BytesUtil.writeUnsigned(value, 4, out);
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
@Override
public void serialize(Object value, ByteBuffer buf) {
    int id = dict.getIdFromValue(value == null ? null : value.toString(), roundingFlag);
    BytesUtil.writeUnsigned(id, dict.getSizeOfId(), buf);
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
public byte[] buildTrieBytes() {
    Stats stats = Stats.stats(this);
    int sizeChildOffset = stats.mbpn_sizeChildOffset;
    int sizeId = stats.mbpn_sizeId;

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(AppendTrieDictionary.HEAD_MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt(stats.mbpn_footprint); // body size
        headOut.writeInt(stats.nValues);
        headOut.write(sizeChildOffset);
        headOut.write(sizeId);
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, AppendTrieDictionary.HEAD_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are
    }

    byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<AppendDictNode> open = new LinkedList<AppendDictNode>();
    IdentityHashMap<AppendDictNode, Integer> offsetMap = new IdentityHashMap<AppendDictNode, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(this, o);
    o = build_writeNode(this, o, true, sizeChildOffset, sizeId, trieBytes);
    if (this.children.isEmpty() == false)
        open.addLast(this);

    while (open.isEmpty() == false) {
        AppendDictNode parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            AppendDictNode c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeChildOffset, sizeId, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
    int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
    BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
    trieBytes[parentOffset] |= flags;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
protected byte[] buildTrieBytes(int baseId) {
    checkOverflowParts(this.root);

    Stats stats = stats();
    int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
    int sizeChildOffset = stats.mbpn_sizeChildOffset;

    if (stats.mbpn_footprint <= 0) // must never happen, but let us be cautious
        throw new IllegalStateException("Too big dictionary, dictionary cannot be bigger than 2GB");
    if (stats.mbpn_footprint > _2GB)
        throw new RuntimeException("Too big dictionary, dictionary cannot be bigger than 2GB");

    // write head
    byte[] head;
    try {
        ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
        DataOutputStream headOut = new DataOutputStream(byteBuf);
        headOut.write(TrieDictionary.MAGIC);
        headOut.writeShort(0); // head size, will back fill
        headOut.writeInt((int) stats.mbpn_footprint); // body size
        headOut.write(sizeChildOffset);
        headOut.write(sizeNoValuesBeneath);
        positiveShortPreCheck(baseId, "baseId");
        headOut.writeShort(baseId);
        positiveShortPreCheck(stats.maxValueLength, "stats.maxValueLength");
        headOut.writeShort(stats.maxValueLength);
        headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
        headOut.close();
        head = byteBuf.toByteArray();
        BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2);
    } catch (IOException e) {
        throw new RuntimeException(e); // shall not happen, as we are writing in memory
    }

    byte[] trieBytes = new byte[(int) stats.mbpn_footprint + head.length];
    System.arraycopy(head, 0, trieBytes, 0, head.length);

    LinkedList<Node> open = new LinkedList<Node>();
    IdentityHashMap<Node, Integer> offsetMap = new IdentityHashMap<Node, Integer>();

    // write body
    int o = head.length;
    offsetMap.put(root, o);
    o = build_writeNode(root, o, true, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
    if (root.children.isEmpty() == false)
        open.addLast(root);

    while (open.isEmpty() == false) {
        Node parent = open.removeFirst();
        build_overwriteChildOffset(offsetMap.get(parent), o - head.length, sizeChildOffset, trieBytes);
        for (int i = 0; i < parent.children.size(); i++) {
            Node c = parent.children.get(i);
            boolean isLastChild = (i == parent.children.size() - 1);
            offsetMap.put(c, o);
            o = build_writeNode(c, o, isLastChild, sizeNoValuesBeneath, sizeChildOffset, trieBytes);
            if (c.children.isEmpty() == false)
                open.addLast(c);
        }
    }

    if (o != trieBytes.length)
        throw new RuntimeException();
    return trieBytes;
}
 

import org.apache.kylin.common.util.BytesUtil; //导入方法依赖的package包/类
private byte[] getByteArrayForShort(short v) {
    byte[] split = new byte[Bytes.SIZEOF_SHORT];
    BytesUtil.writeUnsigned(v, split, 0, Bytes.SIZEOF_SHORT);
    return split;
}