C++ Prediction::size方法代码示例

/** SQLite callback function
    Builds prediction from query results.

*/
int buildPrediction( void* callbackDataPtr,
		     int argc,
		     char** argv,
		     char** column )
{
	// cast pointer to void back to pointer to CallbackData object
	CallbackData* dataPtr = static_cast<CallbackData*>(callbackDataPtr);

	Prediction* predictionPtr = dataPtr->predPtr;
	size_t maxPredictionSize = dataPtr->predSize;

	if (predictionPtr->size() > maxPredictionSize) {
		return 1;
	} else {

		if( argc == 2 &&
		    strcmp( "word", column[ 0 ] ) == 0 &&
		    strcmp( "count", column[ 1 ] ) == 0 ) {
			
			predictionPtr->addSuggestion( 
				Suggestion( argv[ argc - 2 ],
					    atof( argv[ argc - 1 ] )
					)
				);
			
		} else {
			std::cerr << "Invalid invocation of buildPrediction method!"
				  << std::endl;
			exit( 1 );
		}
	}
	return 0;
}

void SelectorTest::testSelect(TestDataSuite* tds)
{
    while (tds->hasMoreTestData()) {
	std::cerr << "Updating strstream: " << strstream->str() << '|' << std::endl
		  << " with: " << tds->getUpdateString() << '|' << std::endl;
	*strstream << tds->getUpdateString();
	std::vector<std::string> selectedTokens;
	selectedTokens = selector->select(tds->getInputPrediction());

	Prediction expected = tds->getOutputPrediction();
	CPPUNIT_ASSERT_EQUAL( (size_t)expected.size(), selectedTokens.size() );

	std::vector<std::string>::const_iterator actual_it = selectedTokens.begin();
	int index = 0;
	while (actual_it != selectedTokens.end()) {
	    std::cerr << "[expected] " << expected.getSuggestion(index).getWord()
		      << "  [actual] " << *actual_it << std::endl;
	    CPPUNIT_ASSERT_EQUAL(expected.getSuggestion(index).getWord(),
				 *actual_it);
	    
	    index++;
	    actual_it++;
	}

	contextTracker->update();
	tds->nextTestData();
    }
}

Prediction RecencyPredictor::predict (const size_t max, const char** filter) const
{
    Prediction result;

    std::string prefix = contextTracker->getPrefix();
    logger << INFO << "prefix: " << prefix << endl;
    if (!prefix.empty()) {
        // Only build recency prediction if prefix is not empty: when
        // prefix is empty, all previosly seen tokens are candidates
        // for prediction. This is not desirable, because it means
        // that recency prediction reduces to repetion of max previous
        // tokens (i.e. the prediction would contain the most recent
        // tokens in reverse order).
        //
        Suggestion  suggestion;
        size_t      index = 1;
        std::string token = contextTracker->getToken(index);
	double      prob = 0;
        while (!token.empty()                // context history exhausted
	       && result.size() < max        // need only max suggestions
	       && index <= cutoff_threshold  // look back only as far as cutoff
	    ) {
	    logger << INFO << "token: " << token << endl;

            if (token.find(prefix) == 0) { // if token starts with prefix

		if (token_satisfies_filter (token, prefix, filter)) {
		    // compute probability according to exponential decay
		    // formula
		    //
		    prob = n_0 * exp(-(lambda * (index - 1)));
		    logger << INFO << "probability: " << prob << endl;
		    suggestion.setWord(token);
		    suggestion.setProbability(prob);
		    result.addSuggestion(suggestion);
		}

            }

            index++;
            token = contextTracker->getToken(index);
        }
    }

    return result;
}

bool Prediction::operator== (const Prediction& right) const
{
    // same instance is obviously equal to itself
    if (&right == this) {
	return true;
    } else {
	if (size() != right.size()) {
	    return false;
	} else {
	    // need to compare each suggestion
	    bool result = true;
	    size_t i = 0;
	    while (i < size() && result) {
		if (getSuggestion(i) != right.getSuggestion(i)) {
		    result = false;
		}
		i++;
	    }
	    return result;
	}
    }
}

void PredictorRegistryTest::testNext()
{
    ContextTracker* pointer = static_cast<ContextTracker*>((void*)0xdeadbeef);
    registry->setContextTracker(pointer);

    PredictorRegistry::Iterator it = registry->iterator();
    Predictor* predictor = 0;

    while (it.hasNext()) {
	predictor = it.next();
    }

    // since we've iterated till the end of the predictors list, predictor
    // is now pointing to the DummyPredictor, so let's test we got the
    // dummy prediction back
    Prediction prediction = predictor->predict(20, 0);

    CPPUNIT_ASSERT(predictor != 0);
    size_t expected_size = 18;
    CPPUNIT_ASSERT_EQUAL(expected_size, prediction.size());
    CPPUNIT_ASSERT_EQUAL(Suggestion("foo1", 0.99), prediction.getSuggestion(0));
    CPPUNIT_ASSERT_EQUAL(Suggestion("foobar6", 0.74), prediction.getSuggestion(17));
}

void NewSmoothedNgramPluginTest::testLearning()
{
    // get pointer to plugin
    Plugin* plugin = pluginRegistry->iterator().next();

    {
	*stream << "f";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(0), actual.size());
    }

    {
	*stream << "o";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(0), actual.size());
    }

    {
	*stream << "o ";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(1), actual.size());
	CPPUNIT_ASSERT_EQUAL(std::string("foo"), actual.getSuggestion(0).getWord());
	ct->update();
    }

    {
	*stream << "bar";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
    }

    {
	*stream << " ";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(2), actual.size());
	CPPUNIT_ASSERT_EQUAL(std::string("foo"), actual.getSuggestion(0).getWord());
	CPPUNIT_ASSERT_EQUAL(std::string("bar"), actual.getSuggestion(1).getWord());
    }

    {
	*stream << "foobar ";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(3), actual.size());
	CPPUNIT_ASSERT_EQUAL(std::string("foobar"), actual.getSuggestion(0).getWord());
	CPPUNIT_ASSERT_EQUAL(std::string("foo"), actual.getSuggestion(1).getWord());
	CPPUNIT_ASSERT_EQUAL(std::string("bar"), actual.getSuggestion(2).getWord());
    }

    {
	*stream << "f";
	ct->update();
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(2), actual.size());
	CPPUNIT_ASSERT_EQUAL(std::string("foobar"), actual.getSuggestion(0).getWord());
	CPPUNIT_ASSERT_EQUAL(std::string("foo"), actual.getSuggestion(1).getWord());
    }
}

void NewSmoothedNgramPluginTest::testFilter()
{
    // get pointer to plugin
    Plugin* plugin = pluginRegistry->iterator().next();

    std::vector<std::string> change;
    change.push_back("foo");
    change.push_back("bar");
    change.push_back("foobar");
    change.push_back("foz");
    change.push_back("baz");
    change.push_back("fozbaz");
    change.push_back("roo");
    change.push_back("rar");
    change.push_back("roobar");

    // Learn some context so that we have data to create non-empty
    // predictions
    // 
    plugin->learn(change);

    // Alternatively, plugin could have learnt thus...
    //    *stream << "foo bar foobar foz baz fozbaz roo rar roobar ";
    //    ct->update();

    {
	Prediction actual = plugin->predict(SIZE, 0);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(9), actual.size());
    }

    {
	const char* filters[] = {"f", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(4), actual.size());
    }

    {
	const char* filters[] = {"b", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(2), actual.size());
    }

    {
	const char* filters[] = {"r", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(3), actual.size());
    }

    {
	const char* filters[] = {"f", "b", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(6), actual.size());
    }

    {
	const char* filters[] = {"f", "r", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(7), actual.size());
    }

    {
	const char* filters[] = {"f", "b", "r", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(9), actual.size());
    }

    {
	const char* filters[] = {"fo", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(4), actual.size());
    }

    {
	const char* filters[] = {"foo", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(2), actual.size());
    }

    {
	const char* filters[] = {"fo", "ba", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(6), actual.size());
    }

    {
	const char* filters[] = {"fo", "ba", "ro", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(8), actual.size());
    }

    {
	const char* filters[] = {"foo", "bar", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(3), actual.size());
    }

    {
	const char* filters[] = {"foobar", "fozba", "roo", 0};
	Prediction actual = plugin->predict(SIZE, filters);
	CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(4), actual.size());
//.........这里部分代码省略.........

Prediction SmoothedCountPlugin::predict(const size_t max_partial_predictions_size, const char** filter) const
{
    // get w_2, w_1, and prefix from HistoryTracker object
    std::string prefix = strtolower( contextTracker->getPrefix() );
    std::string word_1 = strtolower( contextTracker->getToken(1) );
    std::string word_2 = strtolower( contextTracker->getToken(2) );
    
    std::string query; // string used to build sql query
    int result;        // database interrogation diagnostic
    CallbackData data; // data to pass through to callback function
	

    // get most likely unigrams whose w contains prefix
    Prediction predUnigrams;
    
    data.predPtr = &predUnigrams;
    data.predSize = MAX_PARTIAL_PREDICTION_SIZE;
    
    query = 
	"SELECT word, count "
	"FROM _1_gram "
	"WHERE word LIKE \"" + prefix + "%\" "
	"ORDER BY count DESC;";
    
#if defined(HAVE_SQLITE3_H)
    result = sqlite3_exec(
#elif defined(HAVE_SQLITE_H)
    result = sqlite_exec(
#endif
	db,
	query.c_str(),
	buildPrediction,
	&data,
	NULL
    );
    assert(result == SQLITE_OK);


    // get most likely bigrams having matching w_1 whose w contains prefix
    Prediction predBigrams;
    
    data.predPtr = &predBigrams;
    
    query = 
    "SELECT word, count "
    "FROM _2_gram "
    "WHERE word_1 = \"" + word_1 + "\" "
    "AND word LIKE \"" + prefix + "\" "
    "ORDER BY count DESC;";
    
#if defined(HAVE_SQLITE3_H)
    result = sqlite3_exec(
#elif defined(HAVE_SQLITE_H)
    result = sqlite_exec(
#endif
	db,
	query.c_str(),
	buildPrediction,
	&data,
	NULL
    );
    assert(result == SQLITE_OK);


    // get most likely trigrams having matching w_2, w_1 whose w contains prefix
    Prediction predTrigrams;
    
    data.predPtr = &predTrigrams;
    
    query = 
    "SELECT word, count "
    "FROM _3_gram "
    "WHERE word_2 = \"" + word_2 + "\" "
    "AND word_1 = \"" + word_1 + "\" "
    "AND word LIKE \"" + prefix + "\" "
    "ORDER BY count DESC;";
    
#if defined(HAVE_SQLITE3_H)
    result = sqlite3_exec(
#elif defined(HAVE_SQLITE_H)
    result = sqlite_exec(
#endif
	db,
	query.c_str(),
	buildPrediction,
	&data,
	NULL
    );
    assert(result == SQLITE_OK);
	
    
    Prediction p;     // combined result of uni/bi/tri gram predictions
    std::string word; // pivot unigram word (used in next for loop)
    double ccount;    // combined count
    
    // compute smoothed probability estimation
    
    // TODO !!!!!!!! Everything should be scaled down to probabilities!!!
    // TODO That means that counts should be scaled down to values between
    // TODO 0 and 1. We need total word count to do that.
//.........这里部分代码省略.........