C++ MemoryBuffer::reserveTruncate方法代码示例

 void testSet(bool initial)
 {
     unsigned now = msTick();
     bool setValue = !initial;
     bool clearValue = initial;
     const unsigned numBits = 400;
     const unsigned passes = 10000;
     for (unsigned pass=0; pass < passes; pass++)
     {
         Owned<IBitSet> bs = createThreadSafeBitSet();
         testSet1(initial, bs, 0, numBits, setValue, clearValue);
     }
     unsigned elapsed = msTick()-now;
     fprintf(stdout, "Bit test (%u) time taken = %dms\n", initial, elapsed);
     now = msTick();
     for (unsigned pass=0; pass < passes; pass++)
     {
         Owned<IBitSet> bs = createBitSet();
         testSet1(initial, bs, 0, numBits, setValue, clearValue);
     }
     elapsed = msTick()-now;
     fprintf(stdout, "Bit test [thread-unsafe version] (%u) time taken = %dms\n", initial, elapsed);
     now = msTick();
     size32_t bitSetMemSz = getBitSetMemoryRequirement(numBits+5);
     MemoryBuffer mb;
     void *mem = mb.reserveTruncate(bitSetMemSz);
     for (unsigned pass=0; pass < passes; pass++)
     {
         Owned<IBitSet> bs = createBitSet(bitSetMemSz, mem);
         testSet1(initial, bs, 0, numBits, setValue, clearValue);
     }
     elapsed = msTick()-now;
     fprintf(stdout, "Bit test [thread-unsafe version, fixed memory] (%u) time taken = %dms\n", initial, elapsed);
 }

void CLogSerializer::loadSendLogs(GuidSet& ackSet, GuidMap& missedLogs, unsigned long& total_missed)//
{
    try
    {
        Close(); //release old file io, if any
        m_file = createIFile(m_FilePath.str());
        m_fileio = m_file->open(IFOread);
        if (m_fileio == 0)
            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());

        offset_t finger = 0;
        total_missed = 0;
        while(true)
        {
            char dataSize[9];
            memset(dataSize, 0, 9);
            size32_t bytesRead = m_fileio->read(finger,8,dataSize);
            if(bytesRead==0)
                break;

            MemoryBuffer data;
            int dataLen = atoi(dataSize);
            finger+=9;
            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
            if(bytesRead==0)
                break;

            StringBuffer GUID,lostlogStr;
            splitLogRecord(data,GUID,lostlogStr);

            if (ackSet.find(GUID.str())==ackSet.end() && missedLogs.find(GUID.str()) == missedLogs.end())
            {
                if(total_missed % TRACE_INTERVAL == 0)
                    DBGLOG("Miss #%lu GUID: <%s>", total_missed, GUID.str());
                missedLogs[GUID.str()] = lostlogStr.str();
                total_missed++;
            }
            finger+=dataLen;
        }
    }
    catch(IException* ex)
    {
        StringBuffer errorStr;
        ex->errorMessage(errorStr);
        ERRLOG("Exception caught within CSendLogSerializer::LoadDataMap: %s",errorStr.str());
        ex->Release();
    }
    catch(...)
    {
        DBGLOG("Unknown Exception thrown in CSendLogSerializer::LoadDataMap");
    }
    Close();
}

void CLogSerializer::loadAckedLogs(GuidSet& ackedLogs)//
{
    try
    {
        Close(); //release old file io, if any
        m_file = createIFile(m_FilePath.str());
        m_fileio = m_file->open(IFOread);
        if (m_fileio == 0)
            throw MakeStringException(-1, "Unable to open logging file %s",m_FilePath.str());

        offset_t finger = 0;
        m_ItemCount = 0;
        while(true)
        {
            char dataSize[9];
            memset(dataSize, 0, 9);
            size32_t bytesRead = m_fileio->read(finger,8,dataSize);
            if(bytesRead==0)
                break;

            MemoryBuffer data;
            int dataLen = atoi(dataSize);
            finger+=9;
            bytesRead = m_fileio->read(finger,dataLen,data.reserveTruncate(dataLen));
            if(bytesRead==0)
                break;

            StringBuffer GUID, line;
            splitLogRecord(data, GUID, line);
            ackedLogs.insert(GUID.str());
            m_ItemCount++;

            finger+=dataLen;
        }
        fileSize = finger;
        DBGLOG("Total acks loaded %lu", m_ItemCount);
    }
    catch(IException* ex)
    {
        StringBuffer errorStr;
        ex->errorMessage(errorStr);
        ERRLOG("Exception caught within CLogSerializer::loadAckedLogs: %s",errorStr.str());
        ex->Release();
    }
    catch(...)
    {
        DBGLOG("Unknown Exception thrown in CLogSerializer::loadAckedLogs");
    }
    Close();
}

 inline bool fileRead(const char *filename, MemoryBuffer &buff)
 {
     Owned<IFile> fi=createIFile(filename);
     if (fi)
     {
         Owned<IFileIO> fio=fi->open(IFOread);
         if (fio)
         {
             offset_t len=fio->size();
             if (fio->read(0, len, buff.reserveTruncate(len))==len)
                 return true;
         }
     }
     buff.clear();
     return false;
 }

    virtual void process()
    {
        CChooseSetsActivityMaster::process();

        IHThorChooseSetsArg *helper = (IHThorChooseSetsArg *)queryHelper();
        unsigned numSets = helper->getNumSets();
        unsigned nslaves = container.queryJob().querySlaves();

        MemoryBuffer countMb;
        rowcount_t *counts = (rowcount_t *)countMb.reserveTruncate((numSets*(nslaves+2)) * sizeof(rowcount_t));
        rowcount_t *totals = counts + nslaves*numSets;
        rowcount_t *tallies = totals + numSets;
        memset(counts, 0, countMb.length());

        unsigned s=nslaves;
        CMessageBuffer msg;
        while (s--)
        {
            msg.clear();
            rank_t sender;
            if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
                return;
            assertex(msg.length() == numSets*sizeof(rowcount_t));
            unsigned set = (unsigned)sender - 1;
            memcpy(&counts[set*numSets], msg.toByteArray(), numSets*sizeof(rowcount_t));
        }
        for (s=0; s<nslaves; s++)
        {
            unsigned i=0;
            for (; i<numSets; i++)
                totals[i] += counts[s * numSets + i];
        }
        msg.clear();
        msg.append(numSets*sizeof(rowcount_t), totals);
        unsigned endTotalsPos = msg.length();
        for (s=0; s<nslaves; s++)
        {
            msg.rewrite(endTotalsPos);
            msg.append(numSets*sizeof(rowcount_t), tallies);
            container.queryJob().queryJobComm().send(msg, s+1, mpTag);
            unsigned i=0;
            for (; i<numSets; i++)
                tallies[i] += counts[s * numSets + i];
        }
    }

size32_t aesDecrypt(MemoryBuffer &out, size32_t inSz, const void *inBytes, size32_t keyLen, const char *key, const char *iv)
{
    if (0 == inSz)
        return 0;
    OwnedEVPCipherCtx ctx(EVP_CIPHER_CTX_new());
    if (!ctx)
        throw makeEVPException(0, "Failed EVP_CIPHER_CTX_new");

    const size32_t cipherBlockSz = 128;
    // from man page - "should have sufficient room for (inl + cipher_block_size) bytes unless the cipher block size is 1 in which case inl bytes is sufficient"
    size32_t outMaxSz = (cipherBlockSz==1) ? inSz : (inSz + cipherBlockSz/8);
    size32_t startSz = out.length();
    byte *outPtr = (byte *)out.reserveTruncate(outMaxSz);

    /* Initialise the decryption operation. IMPORTANT - ensure you use a key
     * and IV size appropriate for your cipher
     * In this example we are using 256 bit AES (i.e. a 256 bit key). The
     * IV size for *most* modes is the same as the block size. For AES this
     * is 128 bits
     * */
    if (!iv) iv = staticAesIV;
    if (1 != EVP_DecryptInit_ex(ctx, getAesCipher(keyLen), nullptr, (const unsigned char *)key, (const unsigned char *)iv))
        throw makeEVPException(0, "Failed EVP_DecryptInit_ex");

    /* Provide the message to be decrypted, and obtain the plaintext output.
     * EVP_DecryptUpdate can be called multiple times if necessary
     */
    int outSz;
    if (1 != EVP_DecryptUpdate(ctx, outPtr, &outSz, (const unsigned char *)inBytes, inSz))
        throw makeEVPException(0, "Failed EVP_DecryptUpdate");
    int plaintext_len = outSz;

    /* Finalise the decryption. Further plaintext bytes may be written at
     * this stage.
     */
    if (1 != EVP_DecryptFinal_ex(ctx, outPtr + outSz, &outSz))
        throw makeEVPException(0, "Failed EVP_DecryptFinal_ex");

    plaintext_len += outSz;
    out.setLength(startSz+plaintext_len); // truncate length of 'out' to final size
    return (size32_t)plaintext_len;
}

void CJHTreeNode::unpack(const void *node, bool needCopy)
{
    memcpy(&hdr, node, sizeof(hdr));
    SwapBigEndian(hdr);
    __int64 maxsib = keyHdr->getHdrStruct()->phyrec;
    if (!hdr.isValid(keyHdr->getNodeSize()))
    {
        PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag);
        PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib);
        PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib);
        PROGLOG("maxsib=%" I64F "d",maxsib);
        PROGLOG("nodeSize=%d", keyHdr->getNodeSize());
        PROGLOG("keyBytes=%d",(int)hdr.keyBytes);
        PrintStackReport();
        throw MakeStringException(0, "Htree: Corrupt key node detected");
    }
    if (!hdr.leafFlag)
        keyLen = keyHdr->getNodeKeyLength();
    keyRecLen = keyLen + sizeof(offset_t);
    char *keys = ((char *) node) + sizeof(hdr);
    if (hdr.crc32)
    {
        unsigned crc = crc32(keys, hdr.keyBytes, 0);
        if (hdr.crc32 != crc)
            throw MakeStringException(0, "CRC error on key node");
    }
    if (hdr.leafFlag==1)
    {
        firstSequence = *(unsigned __int64 *) keys;
        keys += sizeof(unsigned __int64);
        _WINREV(firstSequence);
    }
    if(isMetadata())
    {
        unsigned short len = *reinterpret_cast<unsigned short *>(keys);
        _WINREV(len);
        expandedSize = len;
        keyBuf = (char *) allocMem(len);
        memcpy(keyBuf, keys+sizeof(unsigned short), len);
    }
    else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY))
    {
        {
            MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
            expandedSize = keyHdr->getNodeSize();
            bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
#ifndef _OLD_VERSION
            keyBuf = NULL;
            if (quick)
                rowexp.setown(expandQuickKeys(keys, needCopy));
            if (!quick||!rowexp.get())
#endif
            {
                keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
            }
        }
        assertex(keyBuf||rowexp.get());
    }
    else
    {
        int i;
        if (keyType & COL_PREFIX)
        {
            MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand");
            
            if (hdr.numKeys) {
                bool handleVariable = isVariable && isLeaf();
                KEYRECSIZE_T workRecLen;
                MemoryBuffer keyBufMb;
                const char *source = keys;
                char *target;
                // do first row
                if (handleVariable) {
                    memcpy(&workRecLen, source, sizeof(workRecLen));
                    _WINREV(workRecLen);
                    size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t);
                    target = (char *)keyBufMb.reserve(tmpSz+workRecLen);
                    memcpy(target, source, tmpSz);
                    source += tmpSz;
                    target += tmpSz;
                }
                else {
                    target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen);
                    workRecLen = keyRecLen - sizeof(offset_t);
                    memcpy(target, source, sizeof(offset_t));
                    source += sizeof(offset_t);
                    target += sizeof(offset_t);
                }

                // this is where next row gets data from
                const char *prev, *next = NULL;
                unsigned prevOffset = 0;
                if (handleVariable)
                    prevOffset = target-((char *)keyBufMb.bufferBase());
                else
                    next = target;

                unsigned char pack1 = *source++;
#ifdef _DEBUG
                assertex(0==pack1); // 1st time will be always be 0
//.........这里部分代码省略.........

int readResults(ISocket * socket, bool readBlocked, bool useHTTP, StringBuffer &result, const char *query, size32_t queryLen)
{
    if (readBlocked)
        socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,60*1000);

    MemoryBuffer remoteReadCursorMb;
    unsigned len;
    bool is_status;
    bool isBlockedResult;
    for (;;)
    {
        if (delay)
            MilliSleep(delay);
        is_status = false;
        isBlockedResult = false;
        try
        {
            if (useHTTP)
                len = 0x10000;
            else if (readBlocked)
                len = socket->receive_block_size();
            else
            {
                socket->read(&len, sizeof(len));
                _WINREV(len);                    
            }
        }
        catch(IException * e)
        {
            if (manyResults)
                showMessage("End of result multiple set\n");
            else
                pexception("failed to read len data", e);
            e->Release();
            return 1;
        }

        if (len == 0)
        {
            if (manyResults)
            {
                showMessage("----End of result set----\n");
                continue;
            }
            break;
        }

        bool isSpecial = false;
        bool pluginRequest = false;
        bool dataBlockRequest = false;
        bool remoteReadRequest = false;
        if (len & 0x80000000)
        {
            unsigned char flag;
            isSpecial = true;
            socket->read(&flag, sizeof(flag));
            switch (flag)
            {
            case '-':
                if (echoResults)
                    fputs("Error:", stdout);
                if (saveResults && trace != NULL)
                    fputs("Error:", trace);
                break;
            case 'D':
                showMessage("request for datablock\n");
                dataBlockRequest = true;
                break;
            case 'P':
                showMessage("request for plugin\n");
                pluginRequest = true;
                break;
            case 'S':
                 if (showStatus)
                 showMessage("Status:");
                 is_status=true;
                 break;
            case 'T':
                 showMessage("Timing:\n");
                 break;
            case 'X':
                showMessage("---Compound query finished---\n");
                return 1;
            case 'R':
                isBlockedResult = true;
                break;
            case 'J':
                remoteReadRequest = true;
                break;
            }
            len &= 0x7FFFFFFF;
            len--;      // flag already read
        }

        MemoryBuffer mb;
        mb.setEndian(__BIG_ENDIAN);
        char *mem = (char *)mb.reserveTruncate(len+1);
        char * t = mem;
        size32_t sendlen = len;
        t[len]=0;
//.........这里部分代码省略.........

bool CLogSerializer::readLogRequest(CLogRequestInFile* logRequestInFile, StringBuffer& logRequest)
{
    //Open the file if exists.
    StringBuffer fileName(logRequestInFile->getFileName());
    Owned<IFile> file = createIFile(fileName);
    Owned<IFileIO> fileIO = file->open(IFOread);
    if (!fileIO)
    {
        //The file may be renamed from .log to .old.
        fileName.replaceString(logFileExt, rolloverFileExt);
        file.setown(createIFile(fileName));
        fileIO.setown(file->open(IFOread));
        if (!fileIO)
        {
            ERRLOG("Unable to open logging file %s", fileName.str());
            return false;
        }
    }

    //Read data size
    char dataSize[9];
    memset(dataSize, 0, 9);
    offset_t finger = logRequestInFile->getPos();
    size32_t bytesRead = fileIO->read(finger, 9, dataSize);
    if (bytesRead < 9)
    {
        ERRLOG("Failed to read logging file %s: not enough data for dataSize", fileName.str());
        return false;
    }
    if (dataSize[8] != '\t')
    {
        ERRLOG("Failed to read logging file %s: incorrect data format for dataSize.", fileName.str());
        return false;
    }
    dataSize[8] = 0;

    char* eptr = nullptr;
    int dataLen = (int)strtol(dataSize, &eptr, 10);
    if (*eptr != '\0')
    {
        ERRLOG("Failed to read logging file %s: incorrect data format for dataSize.", fileName.str());
        return false;
    }

    if (dataLen + 9 != logRequestInFile->getSize())
    {
        ERRLOG("Failed to read logging file %s: incorrect dataSize", fileName.str());
        return false;
    }

    //Read other data
    MemoryBuffer data;
    finger += 9;
    bytesRead = fileIO->read(finger, dataLen, data.reserveTruncate(dataLen));
    if (bytesRead < dataLen)
    {
        ERRLOG("Failed to read logging file %s: dataSize = %d, bytesRead = %d", fileName.str(), dataLen, bytesRead);
        return false;
    }

    //Find GUID and log request
    StringBuffer GUID;
    splitLogRecord(data, GUID, logRequest);
    if (strieq(GUID, logRequestInFile->getGUID()))
        return true;

    ERRLOG("Failed to read logging file %s: GUID read (%s) is not same as GUID (%s)", fileName.str(), GUID.str(), logRequestInFile->getGUID());
    return false;
}

int CMCastRecvServer::run()
{
    SocketEndpoint ackEp(broadcastRoot);
    ackEp.port = ackPort;
    StringBuffer s;
    ackEp.getIpText(s);
    ackSock.setown(ISocket::udp_connect(ackEp.port, s.str()));
    ackSock->set_send_buffer_size(UDP_SEND_SIZE);

    StringBuffer ipStr;
    mcastEp.getIpText(ipStr);
    sock.setown(ISocket::multicast_create(mcastEp.port, ipStr.str()));
    sock->set_receive_buffer_size(UDP_RECV_SIZE);
    SocketEndpoint ep(ipStr.str());
    sock->join_multicast_group(ep);

    
    MemoryBuffer mbAck;
    MCAckPacketHeader *ackPacket = (MCAckPacketHeader *)mbAck.reserveTruncate(MC_ACK_PACKET_SIZE);

    ackPacket->node = groupMember;

    LOG(MCdebugProgress(10), unknownJob, "Running as client %d connected to server %s", groupMember, broadcastRoot.get());

    unsigned *nackList = (unsigned *)(((byte *)ackPacket)+sizeof(MCAckPacketHeader));
    const unsigned *nackUpper = (unsigned *)((byte *)ackPacket)+MC_ACK_PACKET_SIZE-sizeof(unsigned);

    Owned<CDataPacket> dataPacket = new CDataPacket();

    CTimeMon logTm(10000), logTmCons(5000), logTmPoll(5000), logTmOld(5000), logTmNoRecv(5000);
    loop
    {
        try
        {
            unsigned startTime = msTick();
            loop
            {
                try
                {
                    size32_t szRead;
                    sock->read(dataPacket->header, sizeof(MCPacketHeader), MC_PACKET_SIZE, szRead, 5000);
                    break;
                }
                catch (IException *e)
                {
                    if (JSOCKERR_timeout_expired != e->errorCode())
                        throw;
                    else e->Release();
                    LOG(MCdebugProgress(1), unknownJob, "Waiting on packet read socket (waited=%d)", msTick()-startTime);
                }
            }
            if (stopped) break;
            if (MCPacket_Stop == dataPacket->header->cmd)
            {
                stopped = true;
                break;
            }
            ackPacket->tag = dataPacket->header->tag;
            ackPacket->jobId = dataPacket->header->jobId;

            if (oldJobIds.find(dataPacket->header->jobId))
            {
                if (MCPacket_Poll == dataPacket->header->cmd)
                {
                    ackPacket->ackDone = true;
                    MilliSleep(MAX_POLL_REPLY_DELAY/(groupMember+1));
                    ackSock->write(ackPacket, sizeof(MCAckPacketHeader));
                }

                if (tracingPeriod && logTmOld.timedout())
                {
                    LOG(MCdebugProgress(1), unknownJob, "Old job polled=%s", MCPacket_Poll == dataPacket->header->cmd?"true":"false");
                    logTmOld.reset(tracingPeriod);
                }
            }
            else
            {
                CMCastReceiver *receiver = getReceiver(dataPacket->header->tag);
                if (receiver)
                {
                    if (MCPacket_Poll == dataPacket->header->cmd)
                    {
                        size32_t sz;
                        bool res = receiver->buildNack(ackPacket, sz, dataPacket->header->total);
                        MilliSleep(MAX_POLL_REPLY_DELAY/(groupMember+1));
                        ackSock->write(ackPacket, sz);
                        if (tracingPeriod && logTmPoll.timedout())
                        {
                            LOG(MCdebugProgress(1), unknownJob, "Send nack back sz=%d, res=%s, done=%s", sz, res?"true":"false", ackPacket->ackDone?"true":"false");
                            logTmPoll.reset(tracingPeriod);
                        }
                    }
                    else
                    {
                        unsigned total = dataPacket->header->total;
                        bool done;
                        if (receiver->packetReceived(*dataPacket, done)) // if true, packet consumed
                        {
                            unsigned level;
                            if (tracingPeriod && logTmCons.timedout())
//.........这里部分代码省略.........