本文整理汇总了C++中Vocab::LoadVocab方法的典型用法代码示例。如果您正苦于以下问题：C++ Vocab::LoadVocab方法的具体用法？C++ Vocab::LoadVocab怎么用？C++ Vocab::LoadVocab使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Vocab的用法示例。
在下文中一共展示了Vocab::LoadVocab方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C++代码示例。
示例1: ModelTrain
void ModelTrain()
{
Vocab vocab;
vocab.LoadVocab("l3g.txt");
cout << "vocab Size " << vocab.VocabSize << endl;
vector < tuple <int *, int > > src_batch, tgt_batch;
extractBinaryfromStream("data//train_data_40k.tsv", vocab, src_batch, tgt_batch, 1, 0);
int sampleSize = src_batch.size();
cout << "train sample size" << sampleSize << endl;
int iteration = 30;
int miniBatchSize = 1024;
int featureDim = vocab.VocabSize;
int batchNum = sampleSize / miniBatchSize;
int nTrial = 4;
vector <int> shuff(sampleSize);
RunnerBehavior rb;
rb.RunMode = RUNMODE_TRAIN;
rb.Device = DEVICE_GPU;
cout<<"init cuda computation ...."<<endl;
rb.ComputeLib = new CudaOperationManager(true, true);
cout<<"init cuda computation done"<<endl;
int hiddenDim1 = 128;
int hiddenDim2 = 128;
SparseIndexMatrixStat srcMiniBatchInfo;
srcMiniBatchInfo.MAX_ROW_SIZE = miniBatchSize;
srcMiniBatchInfo.MAX_COL_SIZE = featureDim;
srcMiniBatchInfo.TOTAL_BATCH_NUM = batchNum;
srcMiniBatchInfo.TOTAL_SAMPLE_NUM = sampleSize;
srcMiniBatchInfo.MAX_ELEMENT_SIZE = miniBatchSize * 256;
SparseIndexMatrixStat tgtMiniBatchInfo;
tgtMiniBatchInfo.MAX_ROW_SIZE = miniBatchSize;
tgtMiniBatchInfo.MAX_COL_SIZE = featureDim;
tgtMiniBatchInfo.TOTAL_BATCH_NUM = batchNum;
tgtMiniBatchInfo.TOTAL_SAMPLE_NUM = sampleSize;
tgtMiniBatchInfo.MAX_ELEMENT_SIZE = miniBatchSize * 256;
DenseMatrixStat OutputLayer1Info;
OutputLayer1Info.MAX_ROW_SIZE = miniBatchSize;
OutputLayer1Info.MAX_COL_SIZE = hiddenDim1;
OutputLayer1Info.TOTAL_BATCH_NUM = batchNum;
OutputLayer1Info.TOTAL_SAMPLE_NUM = sampleSize;
DenseMatrixStat OutputLayer2Info;
OutputLayer2Info.MAX_ROW_SIZE = miniBatchSize;
OutputLayer2Info.MAX_COL_SIZE = hiddenDim2;
OutputLayer2Info.TOTAL_BATCH_NUM = batchNum;
OutputLayer2Info.TOTAL_SAMPLE_NUM = sampleSize;
FullyConnectedLayer srcLayer1(featureDim, hiddenDim1, &rb);
FullyConnectedLayer srcLayer2(hiddenDim1, hiddenDim2, &rb);
FullyConnectedLayer tgtLayer1(featureDim, hiddenDim1, &rb);
FullyConnectedLayer tgtLayer2(hiddenDim1, hiddenDim2, &rb);
DenseMatrixStat OutputSimInfo;
OutputSimInfo.MAX_ROW_SIZE = miniBatchSize;
OutputSimInfo.MAX_COL_SIZE = 1 + nTrial;
OutputSimInfo.TOTAL_BATCH_NUM = batchNum;
OutputSimInfo.TOTAL_SAMPLE_NUM = sampleSize;
SparseIndexMatrix srcBatch(&srcMiniBatchInfo, rb.Device);
HiddenDenseMatrix srcLayer1Data(&OutputLayer1Info, rb.Device);
HiddenDenseMatrix srcLayer2Data(&OutputLayer2Info, rb.Device);
SparseIndexMatrix tgtBatch(&tgtMiniBatchInfo, rb.Device);
HiddenDenseMatrix tgtLayer1Data(&OutputLayer1Info, rb.Device);
HiddenDenseMatrix tgtLayer2Data(&OutputLayer2Info, rb.Device);
BiMatchData biMatchData(miniBatchSize, nTrial, rb.Device);
SimilarityRunner similarityRunner(10, &rb);
HiddenDenseMatrix simOutput(&OutputSimInfo, rb.Device);
HiddenDenseMatrix probOutput(&OutputSimInfo, rb.Device);
probOutput.Deriv->Data->Zero();
//iteration = 1;
cout<<"start training iteration"<<endl;
double train_time = 0;
double io_time = 0;
struct timeval train_start, train_end;
struct timeval io_start, io_end;
gettimeofday(&train_start, 0);
for (int iter = 0; iter<iteration; iter++)
{
for (int i = 0; i<sampleSize; i++) shuff[i] = i;
//.........这里部分代码省略.........
示例2: ModelPredict
void ModelPredict()
{
Vocab vocab;
vocab.LoadVocab("l3g.txt");
cout << "vocab Size " << vocab.VocabSize << endl;
vector < tuple <int *, int > > src_batch, tgt_batch;
extractBinaryfromStream("data//test_data_clean.tsv", vocab, src_batch, tgt_batch, 0, 0);
int sampleSize = src_batch.size();
cout << "test sample size" << sampleSize << endl;
int miniBatchSize = 1024;
int featureDim = vocab.VocabSize;
int batchNum = (sampleSize - 1) / miniBatchSize + 1;
RunnerBehavior rb;
rb.RunMode = RUNMODE_PREDICT;
rb.Device = DEVICE_GPU;
rb.ComputeLib = new CudaOperationManager(true, true);
int hiddenDim1 = 128;
int hiddenDim2 = 128;
SparseIndexMatrixStat srcMiniBatchInfo;
srcMiniBatchInfo.MAX_ROW_SIZE = miniBatchSize;
srcMiniBatchInfo.MAX_COL_SIZE = featureDim;
srcMiniBatchInfo.TOTAL_BATCH_NUM = batchNum;
srcMiniBatchInfo.TOTAL_SAMPLE_NUM = sampleSize;
srcMiniBatchInfo.MAX_ELEMENT_SIZE = miniBatchSize * 256;
SparseIndexMatrixStat tgtMiniBatchInfo;
tgtMiniBatchInfo.MAX_ROW_SIZE = miniBatchSize;
tgtMiniBatchInfo.MAX_COL_SIZE = featureDim;
tgtMiniBatchInfo.TOTAL_BATCH_NUM = batchNum;
tgtMiniBatchInfo.TOTAL_SAMPLE_NUM = sampleSize;
tgtMiniBatchInfo.MAX_ELEMENT_SIZE = miniBatchSize * 256;
DenseMatrixStat OutputLayer1Info;
OutputLayer1Info.MAX_ROW_SIZE = miniBatchSize;
OutputLayer1Info.MAX_COL_SIZE = hiddenDim1;
OutputLayer1Info.TOTAL_BATCH_NUM = batchNum;
OutputLayer1Info.TOTAL_SAMPLE_NUM = sampleSize;
DenseMatrixStat OutputLayer2Info;
OutputLayer2Info.MAX_ROW_SIZE = miniBatchSize;
OutputLayer2Info.MAX_COL_SIZE = hiddenDim2;
OutputLayer2Info.TOTAL_BATCH_NUM = batchNum;
OutputLayer2Info.TOTAL_SAMPLE_NUM = sampleSize;
ifstream modelReader;
modelReader.open("model//dssm.v2.model", ofstream::binary);
FullyConnectedLayer srcLayer1(modelReader, &rb);
FullyConnectedLayer srcLayer2(modelReader, &rb);
FullyConnectedLayer tgtLayer1(modelReader, &rb);
FullyConnectedLayer tgtLayer2(modelReader, &rb);
modelReader.close();
DenseMatrixStat OutputSimInfo;
OutputSimInfo.MAX_ROW_SIZE = miniBatchSize;
OutputSimInfo.MAX_COL_SIZE = 1;
OutputSimInfo.TOTAL_BATCH_NUM = batchNum;
OutputSimInfo.TOTAL_SAMPLE_NUM = sampleSize;
SparseIndexMatrix srcBatch(&srcMiniBatchInfo, rb.Device);
HiddenDenseMatrix srcLayer1Data(&OutputLayer1Info, rb.Device);
HiddenDenseMatrix srcLayer2Data(&OutputLayer2Info, rb.Device);
SparseIndexMatrix tgtBatch(&tgtMiniBatchInfo, rb.Device);
HiddenDenseMatrix tgtLayer1Data(&OutputLayer1Info, rb.Device);
HiddenDenseMatrix tgtLayer2Data(&OutputLayer2Info, rb.Device);
BiMatchData biMatchData(miniBatchSize, 0, rb.Device);
SimilarityRunner similarityRunner(10, &rb);
HiddenDenseMatrix simOutput(&OutputSimInfo, rb.Device);
HiddenDenseMatrix probOutput(&OutputSimInfo, rb.Device);
ofstream outfile;
outfile.open("data//test_data.v2.result", ofstream::out);
int smpIdx = 0;
for (int b = 0; b<batchNum; b++)
{
srcBatch.Refresh();
tgtBatch.Refresh();
while (smpIdx < sampleSize && srcBatch.RowSize < miniBatchSize && tgtBatch.RowSize < miniBatchSize)
{
srcBatch.PushSample(get<0>(src_batch[smpIdx]), get<1>(src_batch[smpIdx]));
tgtBatch.PushSample(get<0>(tgt_batch[smpIdx]), get<1>(tgt_batch[smpIdx]));
smpIdx++;
}
srcLayer1.Forward(&srcBatch, srcLayer1Data.Output);
srcLayer2.Forward(srcLayer1Data.Output, srcLayer2Data.Output);
tgtLayer1.Forward(&tgtBatch, tgtLayer1Data.Output);
tgtLayer2.Forward(tgtLayer1Data.Output, tgtLayer2Data.Output);
//.........这里部分代码省略.........