C++ Index::acquireReader方法代码示例

void IndexTestCase::testRefreshIndexWithMultiBarrel()
{
    DocumentSchema schema;
    schema.addTextField("BODY");
    
    stringstream ss;
    const size_t NUM_DOCS = 1000;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << "body" << i << " hot;";
    }
    buildIndex(schema, ss.str());
    buildIndex(schema, ss.str(), true);

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pIndexReader1 = index.acquireReader();
    checkDocFreq(pIndexReader1, "BODY", "hot", 2 * NUM_DOCS);

    buildIndex(schema, ss.str(), true);
    IndexReaderPtr pIndexReader2 = index.acquireReader(true);
    CPPUNIT_ASSERT(pIndexReader1 != pIndexReader2);
    
    checkDocFreq(pIndexReader2, "BODY", "hot", 3 * NUM_DOCS);
}

void IndexTestCase::testKeywordIndex()
{
    DocumentSchema schema;
    schema.addUnIndexedField("PATH");
    schema.addField("Keyword", "KEYWORD", false);

    buildIndex(schema, "file1.txt, hello world.");

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);
    TermIteratorPtr pTermIterator = pTermReader->termIterator("Keyword");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    while (pTermIterator->hasNext())
    {
        TermIterator::TermEntry entry = pTermIterator->next();
        const TermMeta& termMeta = entry.postingIterator->getTermMeta();
        CPPUNIT_ASSERT_EQUAL((df_t)1, termMeta.getDocFreq());
        CPPUNIT_ASSERT_EQUAL((ctf_t)1, termMeta.getCTF());
    }

    Term term("Keyword", "hello world.");
    TermPostingIteratorPtr pPost = pTermReader->seek(&term);
    CPPUNIT_ASSERT(pPost);
    docid_t docId = pPost->skipTo(0);
    CPPUNIT_ASSERT_EQUAL((docid_t)0, docId);
    docId = pPost->skipTo(++docId);
    CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);
}

void DateTimeIndexTestCase::testCTF()
{
    const size_t NUM_DOCS = 100;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << 2009 << "-" << i % 12 + 1 << "-" << i % 27 + 1 << " " 
           << i % 24 << ":" << i % 60 << ":" << i % 60 << ";";
    }

    buildDateTimeIndex(ss.str());

    Index index;
    index.open(getIndexPath(), Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);
    TermIteratorPtr pTermIterator = pTermReader->termIterator("DateTime1");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    df_t ttf = 0;
    while (pTermIterator->hasNext())
    {
        TermIterator::TermEntry entry = pTermIterator->next();
        const TermMeta& termMeta = entry.postingIterator->getTermMeta();
        CPPUNIT_ASSERT_EQUAL((df_t)1, termMeta.getDocFreq());
        CPPUNIT_ASSERT_EQUAL((ctf_t)1, termMeta.getCTF());
        ttf +=  termMeta.getDocFreq();
    }

    CPPUNIT_ASSERT_EQUAL((df_t)100, ttf);
}

void DateTimeIndexTestCase::testQuery()
{
    const size_t NUM_DOCS = 12;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << 2009 << "-" << i % 12 + 1 << "-" << i % 27 + 1 << " " 
           << i % 24 << ":" << i % 60 << ":" << i % 60 << ";";
        if (i % 10 == 0)
        {
            ss << 2009 << "-" << i % 12 + 1 << "-" << i % 27 + 1 << " " 
               << i % 24 << ":" << i % 60 << ":" << i % 60 << ";";
        }
    }

    buildDateTimeIndex(ss.str());

    Index index;
    index.open(getIndexPath(), Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    IndexSearcher se(pReader);

    QueryParser queryParser(pReader->getAnalyzerMapper(), 
                    "DateTime1", QueryParser::OP_AND);
    QueryHitsPtr pHits = se.search("query=\'2009-2-2 1:1:1\'", queryParser);
    CPPUNIT_ASSERT(pHits);
    uint64_t uTotalHits =  pHits->getTotalHits();
    CPPUNIT_ASSERT_EQUAL((uint64_t)1, uTotalHits);

    pHits = se.search("query=\'2009-11-11 10:10:10\'", queryParser);
    CPPUNIT_ASSERT(pHits);
    uTotalHits =  pHits->getTotalHits();
    CPPUNIT_ASSERT_EQUAL((uint64_t)2, uTotalHits);
}

void IndexContentTestCase::testIndexContent_DL()
{
    Index* pIndex;
    IndexReaderPtr pReader;

    const Term* pTerm;
    TermIteratorPtr pTermIter;
    int	docCount = 0;
    int	termCount = 0;
    uint32_t i;
    uint32_t indexTermId;
    string fileName;

    //Check posting list
    Path indexPath = TestHelper::getTestDataPath();
    indexPath.makeDirectory();
    indexPath.pushDirectory(_T("test_dlindex"));    
    pIndex = new Index(indexPath.toString().c_str(), Index::READ, NULL);
    auto_ptr<Index> indexPtr(pIndex);
    pReader = pIndex->acquireReader();
    TermReaderPtr pTermReader = pReader->termReader();

    pTermIter = pTermReader->termIterator("BODY");

    StoredFieldsReaderPtr pDocReader = pReader->createStoredFieldsReader();
    //Iterator all terms
    while(pTermIter->next())
    {
        pTerm = pTermIter->term();
		
        CPPUNIT_ASSERT(pTermReader->seek(pTerm));
				
        indexTermId = (pTerm->cast<int32_t>())->getValue();
        docCount = 0;
        TermPostingIteratorPtr pTermDocFreqs = pTermReader->termPostings();
        while(pTermDocFreqs->nextDoc())
        {
            DocumentPtr pDoc = pDocReader->document(pTermDocFreqs->doc());
            docCount++;
            // 获取文件路径
            fileName.assign(pDoc->getField("PATH")->getValue().c_str());

            TermList* pTermIdList = m_pDocScanner->getTermListOfFile(fileName);
            CPPUNIT_ASSERT(pTermIdList != NULL);

            for(i = 0, termCount = 0; i < pTermIdList->getSize(); i++)
            {
                if(indexTermId == pTermIdList->getValue(i))
                {
                    termCount++;
                }
            }
			
            CPPUNIT_ASSERT_EQUAL((tf_t)termCount, pTermDocFreqs->freq());

        }//end while nextDoc()
        CPPUNIT_ASSERT_EQUAL((df_t)docCount, pTermDocFreqs->getDocFreq());
    }
    CPPUNIT_ASSERT(m_pDocScanner->getTotalTermCount() == pReader->getNumTerms());
}

void IndexTestCase::testInt32ForwardIndex()
{
    DocumentSchema schema;
    schema.addSortableField("Int32Id", FieldType::INT32, false);

    const static size_t NUM_DOCS = 1000;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << i << ";";
    }

    GLOBAL_CONF().Build.buildThreadCount = 1;
    buildIndex(schema, ss.str());

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    ForwardIndexIteratorPtr pForIndexIt = pReader->forwardIndexReader("Int32Id");
    CPPUNIT_ASSERT(pForIndexIt != NULL);
    Int32ForwardIndexIteratorPtr pInt32ForIndexIt =
        pForIndexIt.cast<Int32ForwardIndexIterator>();
    CPPUNIT_ASSERT(pInt32ForIndexIt != NULL);
    
    int32_t value = 0;
    docid_t docId = 0;
    for (; docId < (docid_t)NUM_DOCS; ++docId)
    {
        CPPUNIT_ASSERT(pInt32ForIndexIt->seek(docId, value));
        CPPUNIT_ASSERT_EQUAL((int32_t)docId, value);
    }
    CPPUNIT_ASSERT(!pInt32ForIndexIt->seek(docId, value));
}

void KeywordIndexTestCase::testCTF()
{
    GLOBAL_CONF().Build.buildThreadCount = 2;
    buildKeywordIndex("1, 2; 3, 4; 5, 6; 7, 8; 9, 10; "
                      "1, 2; 3, 4; 5, 6; 7, 8; 9, 10");

    Index index;
    index.open(getIndexPath(), Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);
    TermIteratorPtr pTermIterator = pTermReader->termIterator("Keyword1");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    try 
    {
        while (pTermIterator->hasNext())
        {
            TermIterator::TermEntry entry = pTermIterator->next();
            const TermMeta& termMeta = entry.postingIterator->getTermMeta();
            CPPUNIT_ASSERT_EQUAL((df_t)2, termMeta.getDocFreq());
            CPPUNIT_ASSERT_EQUAL((ctf_t)2, termMeta.getCTF());
        }
    }
    catch(const FirteXException& e)
    {
        cout << "ERROR:" << e.what() << endl;
        CPPUNIT_ASSERT(false);
    } 
}

void IndexTestCase::testInt32_IF()
{
    DocumentSchema schema;
    schema.addField("Int32", "INT32_IF", false);

    const static size_t NUM_DOCS = 1000;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << (i % 100) << ";";
    }

    GLOBAL_CONF().Build.buildThreadCount = 1;
    buildIndex(schema, ss.str());

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);
    TermIteratorPtr pTermIterator = pTermReader->termIterator("Int32");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    while (pTermIterator->hasNext())
    {
        TermIterator::TermEntry entry = pTermIterator->next();
        const TermMeta& termMeta = entry.postingIterator->getTermMeta();
        CPPUNIT_ASSERT_EQUAL((df_t)10, termMeta.getDocFreq());
        CPPUNIT_ASSERT_EQUAL((ctf_t)10, termMeta.getCTF());
    }

    Term term("Int32", "0");
    TermPostingIteratorPtr pPost = pTermReader->seek(&term);
    CPPUNIT_ASSERT(pPost);
    docid_t docId = pPost->skipTo(0);
    CPPUNIT_ASSERT_EQUAL((docid_t)0, docId);
    docId = pPost->skipTo(901);
    CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);

    ForwardIndexIteratorPtr pForIndexIt = pReader->forwardIndexReader("Int32");
    CPPUNIT_ASSERT(pForIndexIt != NULL);
    Int32ForwardIndexIteratorPtr pInt32ForIndexIt =
        pForIndexIt.cast<Int32ForwardIndexIterator>();
    CPPUNIT_ASSERT(pInt32ForIndexIt != NULL);
    
    int32_t value = 0;
    docId = 0;
    for (; docId < (docid_t)NUM_DOCS; ++docId)
    {
        CPPUNIT_ASSERT(pInt32ForIndexIt->seek(docId, value));
        CPPUNIT_ASSERT_EQUAL((int32_t)(docId % 100), value);
    }
    CPPUNIT_ASSERT(!pInt32ForIndexIt->seek(docId, value));
}

void IndexTestCase::testIndexOptimize()
{
    GLOBAL_CONF().Merge.maxAllowedOpenFiles = 16;
    DocumentSchema schema;
    schema.addTextField("BODY");
    
    stringstream ss;
    const size_t NUM_DOCS = 1000;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << "body" << i << " hot;";
    }

    try
    {
        buildIndex(schema, ss.str());
        buildIndex(schema, ss.str(), true);
    }
    catch (const FirteXException& e)
    {
        cout << "ERROR: " << e.what() << endl;
        CPPUNIT_ASSERT(false);
    }

    tstring str = getTestPath();

    {
        Index index;
        index.open(str, Index::APPEND, NULL); 
        IndexWriterPtr pIndexWriter = index.acquireWriter();
        CPPUNIT_ASSERT(pIndexWriter != NULL);
    
        try
        {
            pIndexWriter->optimizeIndex();
        }
        catch (const FirteXException& e)
        {
            cout << "ERROR: " << e.what() << endl;
            CPPUNIT_ASSERT(false);
        }
    }
    Index index;
    try
    {
        index.open(str, Index::READ, NULL);
        IndexReaderPtr pIndexReader = index.acquireReader();
        checkDocFreq(pIndexReader, "BODY", "hot", 2 * NUM_DOCS);
    }
    catch (const FirteXException& e)
    {
        cout << "ERROR: " << e.what() << endl;
        CPPUNIT_ASSERT(false);
    }
}

void DateTimeIndexTestCase::testPosting()
{
    const size_t NUM_DOCS = 100;
    vector<TimeAndDoc> timeVec;

    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        int y = 2011;
        int m = (i % 12) + 1;
        int d = (i % 27) + 1;
        int h = i % 24;
        int min = i % 60;
        int s = i % 60;
        ss << y << "-" << m << "-" << d << " " 
           << h << ":" << min << ":" << s << ";";
        TimeAndDoc td;
        td.nTime = DateTimeAnalyzer::makeTime(y, m, d, h, min, s);
        td.docId = (docid_t)i;
        timeVec .push_back(td);
    }
    sort(timeVec.begin(), timeVec.end(), timeLess);

    GLOBAL_CONF().Build.buildThreadCount = 1;
    buildDateTimeIndex(ss.str());

    Index index;
    index.open(getIndexPath(), Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    TermReaderPtr pTermReader = pReader->termReader();
    TermIteratorPtr pIterator = pTermReader->termIterator("DateTime1");
    CPPUNIT_ASSERT(pIterator);

    size_t j = 0;
    while (pIterator->hasNext())
    {
        TermIterator::TermEntry entry = pIterator->next();
        const Term* pTerm = entry.term;
        const Int64Term* pTermX = pTerm->cast<int64_t>();
        int64_t nCurTerm = pTermX->getValue();
        int64_t nExpTerm = timeVec[j].nTime;
        CPPUNIT_ASSERT_EQUAL(nExpTerm, nCurTerm);

        TermPostingIteratorPtr pPostingIter = entry.postingIterator;
        CPPUNIT_ASSERT(pPostingIter != NULL);
        docid_t docId = pPostingIter->skipTo(0);
        CPPUNIT_ASSERT_EQUAL(timeVec[j].docId, docId);
        docId = pPostingIter->skipTo(docId + 1);
        CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);
        
        j++;
    }
}

void IndexTestCase::testRefreshIndex()
{
    DocumentSchema schema;
    schema.addTextField("BODY");
    
    stringstream ss;
    const size_t NUM_DOCS = 1000;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << "body" << i << " hot;";
    }
    buildIndex(schema, ss.str());

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);

    IndexReaderPtr pIndexReader1 = index.acquireReader();
    checkDocFreq(pIndexReader1, "BODY", "hot", NUM_DOCS);

    IndexBarrelKeeperPtr pKeeper = index.getIndexBarrelKeeper();
    CPPUNIT_ASSERT_EQUAL((size_t)1, pKeeper->getHeldCommitCount());

    buildIndex(schema, ss.str(), true);
    IndexReaderPtr pIndexReader2 = index.acquireReader(true);
    CPPUNIT_ASSERT(pIndexReader1 != pIndexReader2);

    checkDocFreq(pIndexReader2, "BODY", "hot", 2 * NUM_DOCS);

    CPPUNIT_ASSERT_EQUAL((size_t)2, pKeeper->getHeldCommitCount());

    pIndexReader1.reset();
    pIndexReader2.reset();
    buildIndex(schema, ss.str(), true);

    IndexReaderPtr pIndexReader3 = index.acquireReader(true);
    CPPUNIT_ASSERT_EQUAL((size_t)1, pKeeper->getHeldCommitCount());
}

void IndexTestCase::testPrimaryKeyIndex()
{
    DocumentSchema schema;
    schema.addField("PK", "PRIMARY_KEY", false);

    const static size_t NUM_DOCS = 1000;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << i << ";";
    }

    GLOBAL_CONF().Build.buildThreadCount = 1;

    buildIndex(schema, ss.str());

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);
    TermIteratorPtr pTermIterator = pTermReader->termIterator("PK");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    while (pTermIterator->hasNext())
    {
        TermIterator::TermEntry entry = pTermIterator->next();
        const TermMeta& termMeta = entry.postingIterator->getTermMeta();
        CPPUNIT_ASSERT_EQUAL((df_t)1, termMeta.getDocFreq());
        CPPUNIT_ASSERT_EQUAL((ctf_t)1, termMeta.getCTF());
    }

    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        stringstream ss2;
        ss2 << i;
        Term  term("PK", ss2.str());
        TermPostingIteratorPtr pPost = pTermReader->seek(&term);
        CPPUNIT_ASSERT(pPost);
        docid_t docId = pPost->skipTo(0);
        CPPUNIT_ASSERT_EQUAL((docid_t)i, docId);
        docId = pPost->skipTo(++docId);
        CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);
    }
}

void StandardCollectionTestCase::checkIndex()
{
    string sIndex = TestHelper::getTestDataPath() + "/standard_collection/files_index";
    Index index;
    try
    {
        index.open(sIndex, Index::READ, NULL);

        IndexReaderPtr pReader = index.acquireReader();
        CPPUNIT_ASSERT(pReader != NULL);        
    }
    catch(const FirteXException& e)
    {
        cout << "===ERROR: " << e.what() << endl;
        CPPUNIT_ASSERT(false);
    }
}

void DateTimeIndexTestCase::testRangeQuery()
{
    const size_t NUM_DOCS = 12;
    stringstream ss;
    for (size_t i = 0; i < NUM_DOCS; ++i)
    {
        ss << 2009 << "-" << i % 12 + 1 << "-" << i % 27 + 1 << " " 
           << i % 24 << "/" << i % 60 << "/" << i % 60 << ";";
        /*  if (i % 10 == 0)
            {
            ss << 2009 << "-" << i % 12 + 1 << "-" << i % 27 + 1 << " " 
            << i % 24 << "/" << i % 60 << "/" << i % 60 << ";";
            }*/
    }

    buildDateTimeIndex(ss.str());

    Index index;
    index.open(getIndexPath(), Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();

    // test case for [13 TO 18]
    {   
        IndexSearcher se(pReader);
        // QueryHits* pHits = se.search("[2009-11-11 10/10/10 TO 2009-12-12 11/11/11]",
        //         "DateTime1", 100);
        // CPPUNIT_ASSERT(pHits != NULL);
        // df_t numHits =  pHits->getHitNum();
        // CPPUNIT_ASSERT_EQUAL((df_t)2, numHits);

        // QueryHits::Iterator it = pHits->iterator();
    
        // df_t n = 0;
        // while (n < numHits)
        // {
        //     CPPUNIT_ASSERT(it.hasNext());
        //     QueryHits::HitDoc* hitDoc = it.next();
        //     CPPUNIT_ASSERT(hitDoc != NULL);
        //     CPPUNIT_ASSERT_EQUAL((docid_t)10 + n, hitDoc->getDocID());
        //     n++;
        // }

        // delete pHits;    
    }
}

void IndexTestCase::testTextIndex()
{
    DocumentSchema schema;
    schema.addUnIndexedField("PATH");
    schema.addTextField("CONTENT");

    buildIndex(schema, "file1.txt, hello world.");

    tstring str = getTestPath();
    Index index;
    index.open(str, Index::READ, NULL);
    IndexReaderPtr pReader = index.acquireReader();
    CPPUNIT_ASSERT(pReader != NULL);
    TermReaderPtr pTermReader = pReader->termReader();
    CPPUNIT_ASSERT(pTermReader);

    TermIteratorPtr pTermIterator = pTermReader->termIterator("CONTENT");
    CPPUNIT_ASSERT(pTermIterator != NULL);

    while (pTermIterator->hasNext())
    {
        TermIterator::TermEntry entry = pTermIterator->next();
        const TermMeta& termMeta = entry.postingIterator->getTermMeta();
        CPPUNIT_ASSERT_EQUAL((df_t)1, termMeta.getDocFreq());
        CPPUNIT_ASSERT_EQUAL((ctf_t)1, termMeta.getCTF());
    }

    Term term("CONTENT", "hello");
    TermPostingIteratorPtr pPost = pTermReader->seek(&term);

    CPPUNIT_ASSERT(pPost);
    docid_t docId = pPost->skipTo(0);
    CPPUNIT_ASSERT_EQUAL((docid_t)0, docId);
    docId = pPost->skipTo(++docId);
    CPPUNIT_ASSERT_EQUAL((docid_t)INVALID_DOCID, docId);

    StoredFieldsReaderPtr pDocReader = pReader->createStoredFieldsReader();
    CPPUNIT_ASSERT(pDocReader);
    FieldSelector selector(pReader->getDocSchema(), true, false);
    ResultDoc resultDoc(0);
    bool ret = pDocReader->getDocument(selector, resultDoc);
    CPPUNIT_ASSERT(ret);
    CPPUNIT_ASSERT(resultDoc.size() > 0);
}