Java CanalEntry.Entry方法代码示例

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
@Override
public boolean filter(CanalEntry.Entry entry) {

    //过滤掉事物头尾等 非 row data 的 entry
    boolean rowData = entry.getEntryType() == CanalEntry.EntryType.ROWDATA;

    if (!rowData) {
        return false;
    }

    //只保存 insert update delete 类型的 时间
    boolean eventType = filterEventType(entry.getHeader().getEventType());

    if (!eventType) {
        return false;
    }

    return true;
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (CanalEntry.Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition});
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
/**
 * 消费当前消息
 */
private void consumerMessage(Message message) {
    log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
    try {
        for (CanalEntry.Entry e : message.getEntries()) {
            if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
            CanalEntry.Header header = e.getHeader();
            if (header.getExecuteTime() < startRtTime
                    || header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
                    || !handle.startHandle(header)) continue;
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
                if (rowChange.getIsDdl()) continue;
                handle.rowChangeHandle(rowChange);
            } catch (InvalidProtocolBufferException e1) {
                log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
            }
        }
        handle.ack(message.getId());
    } finally {
        handle.finishMessageHandle();
    }
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
@Override
public void put(List<Event> list) throws InterruptedException, CanalStoreException {
    if (CollectionUtils.isNotEmpty(list)) {
        for (Event event : list) {
            CanalEntry.Entry entry = event.getEntry();
            try {
                CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                CanalEntry.EventType eventType = rowChange.getEventType();

                if (eventType == CanalEntry.EventType.INSERT || eventType == CanalEntry.EventType.UPDATE || eventType == CanalEntry.EventType.DELETE) {
                    List<CanalEntry.RowData> rowDatas = rowChange.getRowDatasList();
                    for (CanalEntry.RowData rowData : rowDatas) {
                        sendKafkaMsg(rowData, entry, eventType);
                    }
                }
            } catch (InvalidProtocolBufferException e) {
                LOGGER.error(e.getMessage(), e);
            }
        }
    }
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
@Override
public List<IGenericMessage> unwrapMessages(byte[] data) throws IOException {
    List<IGenericMessage> list = new ArrayList<>();

    CanalPacket.Messages cMessage = CanalPacket.Messages.parseFrom(data);
    List<ByteString> strings = cMessage.getMessagesList();
    for (ByteString str : strings) {
        CanalEntry.Entry ent = CanalEntry.Entry.parseFrom(str);
        MysqlGenericMessage message = new MysqlGenericMessage(ent);

        list.add(message);
    }
    return list;
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
@Override
public byte[] wrapMessages(List<IGenericMessage> list) throws IOException {
    CanalPacket.Messages.Builder builder = CanalPacket.Messages.newBuilder();
    builder.setBatchId(0);

    for (IGenericMessage obj : list) {
        MysqlGenericMessage message = (MysqlGenericMessage) obj;
        CanalEntry.Entry ent = message.getEntry();

        builder.addMessages(ent.toByteString());
    }
    CanalPacket.Messages cMessage = builder.build();
    return cMessage.toByteArray();
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
@Override
public ElasticsearchMetadata trans(Message message) {
    List<CanalEntry.Entry> entries = message.getEntries();
    ElasticsearchMetadata elasticsearchMetadata = TotoroObjectPool.esMetadata();
    elasticsearchMetadata.setBatchId(message.getId());
    elasticsearchMetadata.setTransForm(this);
    if (entries != null && entries.size() > 0) {
        EsEntryArrayList esEntryList = TotoroObjectPool.esEntryArrayList();

        entries.forEach(entry -> {
            if (messageFilterChain.filter(entry)) {
                try {
                    ElasticsearchMetadata.EsEntry esEntry = getElasticsearchMetadata(entry);
                    esEntryList.add(esEntry);
                } catch (InvalidProtocolBufferException e) {
                    e.printStackTrace();
                }
            }
        });
        elasticsearchMetadata.setEsEntries(esEntryList);
    }

    logger.info("Trans form complete message id =====> {}", message.getId());

    if (logger.isDebugEnabled()) {
        logger.debug("Trans form complete elasticsearch metadata  =====> {}", elasticsearchMetadata.toString());
    }
    return elasticsearchMetadata;
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
private ElasticsearchMetadata.EsEntry getElasticsearchMetadata(CanalEntry.Entry entry) throws InvalidProtocolBufferException {

        final String database = entry.getHeader().getSchemaName(); // => index
        final String table = entry.getHeader().getTableName();// => type
        final CanalEntry.RowChange change = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
        final List<CanalEntry.RowData> rowDataList = change.getRowDatasList();

        CanalEntry.EventType eventType = entry.getHeader().getEventType();
        final int esEventType = esAdapter.getEsEventType(eventType);


        EsRowDataArrayList esRowDataList = TotoroObjectPool.esRowDataArrayList();

        for (CanalEntry.RowData rowData : rowDataList) {
            List<CanalEntry.Column> columnList = esAdapter.getColumnList(esEventType, rowData);
            ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData();
            EsColumnHashMap columnMap = TotoroObjectPool.esColumnHashMap();
            columnList.forEach(column -> columnMap.put(column.getName(), column.getValue()));
            esRowData.setRowData(columnMap);
            esRowData.setIdColumn(esAdapter.getEsIdColumn(database, table));//获取es对应的id Column
            esRowDataList.add(esRowData);
        }

        ElasticsearchMetadata.EsEntry esEntry = TotoroObjectPool.esEntry();

        esEntry.setIndex(database)
                .setType(table)
                .setEsRowDatas(esRowDataList)
                .setEventType(esEventType);

        return esEntry;
    }
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
                                                                                                       throws CanalSinkException,
                                                                                                       InterruptedException {
    List rowDatas = entrys;
    if (filterTransactionEntry) {
        rowDatas = new ArrayList<CanalEntry.Entry>();
        for (CanalEntry.Entry entry : entrys) {
            if (entry.getEntryType() == EntryType.ROWDATA) {
                rowDatas.add(entry);
            }
        }
    }

    return sinkData(rowDatas, remoteAddress);
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
private boolean sinkData(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress)
                                                                                        throws InterruptedException {
    boolean hasRowData = false;
    boolean hasHeartBeat = false;
    List<Event> events = new ArrayList<Event>();
    for (CanalEntry.Entry entry : entrys) {
        Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
        if (!doFilter(event)) {
            continue;
        }

        events.add(event);
        hasRowData |= (entry.getEntryType() == EntryType.ROWDATA);
        hasHeartBeat |= (entry.getEntryType() == EntryType.HEARTBEAT);
    }

    if (hasRowData) {
        // 存在row记录
        return doSink(events);
    } else if (hasHeartBeat) {
        // 存在heartbeat记录，直接跳给后续处理
        return doSink(events);
    } else {
        // 需要过滤的数据
        if (filterEmtryTransactionEntry && !CollectionUtils.isEmpty(events)) {
            long currentTimestamp = events.get(0).getEntry().getHeader().getExecuteTime();
            // 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
            if (Math.abs(currentTimestamp - lastEmptyTransactionTimestamp) > emptyTransactionInterval
                || lastEmptyTransactionCount.incrementAndGet() > emptyTransctionThresold) {
                lastEmptyTransactionCount.set(0L);
                lastEmptyTransactionTimestamp = currentTimestamp;
                return doSink(events);
            }
        }

        // 直接返回true，忽略空的事务头和尾
        return true;
    }
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
public void start() throws CanalStoreException {
    super.start();
    if (Integer.bitCount(bufferSize) != 1) {
        throw new IllegalArgumentException("bufferSize must be a power of 2");
    }

    Assert.notNull(flushCallback, "flush callback is null!");
    indexMask = bufferSize - 1;
    entries = new CanalEntry.Entry[bufferSize];
}
 

import com.alibaba.otter.canal.protocol.CanalEntry; //导入方法依赖的package包/类
private void put(CanalEntry.Entry data) throws InterruptedException {
    // 首先检查是否有空位
    if (checkFreeSlotAt(putSequence.get() + 1)) {
        long current = putSequence.get();
        long next = current + 1;

        // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
        entries[getIndex(next)] = data;
        putSequence.set(next);
    } else {
        flush();// buffer区满了，刷新一下
        put(data);// 继续加一下新数据
    }
}