C++ GMatrix::loadArff方法代码示例

GMatrixFactorization* GRecommenderLib::InstantiateMatrixFactorization(GArgReader& args)
{
	if(args.size() < 1)
		throw Ex("The number of intrinsic dims must be specified for this algorithm");
	size_t intrinsicDims = args.pop_uint();
	GMatrixFactorization* pModel = new GMatrixFactorization(intrinsicDims);
	while(args.next_is_flag())
	{
		if(args.if_pop("-regularize"))
			pModel->setRegularizer(args.pop_double());
		else if(args.if_pop("-miniters"))
			pModel->setMinIters(args.pop_uint());
		else if(args.if_pop("-decayrate"))
			pModel->setDecayRate(args.pop_double());
		else if(args.if_pop("-nonneg"))
			pModel->nonNegative();
		else if(args.if_pop("-clampusers"))
		{
			GMatrix tmp;
			tmp.loadArff(args.pop_string());
			size_t offset = args.pop_uint();
			pModel->clampUsers(tmp, offset);
		}
		else if(args.if_pop("-clampitems"))
		{
			GMatrix tmp;
			tmp.loadArff(args.pop_string());
			size_t offset = args.pop_uint();
			pModel->clampItems(tmp, offset);
		}
		else
			throw Ex("Invalid option: ", args.peek());
	}
	return pModel;
}

void Loader::loadLaborData2(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	GMatrix raw;
	raw.loadArff("data/labor_stats.arff");
	
	size_t dims = 1;
	size_t offset = 0;
	size_t train_size = 258;
	size_t test_size = 96;
	
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		*x = double(i) / train_size;
		*y = raw[offset + i][0];
	}
}

void loadData(GMatrix& m, const char* szFilename)
{
	// Load the dataset by extension
	PathData pd;
	GFile::parsePath(szFilename, &pd);
	if(_stricmp(szFilename + pd.extStart, ".arff") == 0)
		m.loadArff(szFilename);
	else if(_stricmp(szFilename + pd.extStart, ".csv") == 0)
	{
		GCSVParser parser;
		parser.parse(m, szFilename);
		cerr << "\nParsing Report:\n";
		for(size_t i = 0; i < m.cols(); i++)
			cerr << to_str(i) << ") " << parser.report(i) << "\n";
	}
	else if(_stricmp(szFilename + pd.extStart, ".dat") == 0)
	{
		GCSVParser parser;
		parser.setSeparator('\0');
		parser.parse(m, szFilename);
		cerr << "\nParsing Report:\n";
		for(size_t i = 0; i < m.cols(); i++)
			cerr << to_str(i) << ") " << parser.report(i) << "\n";
	}
	else
		throw Ex("Unsupported file format: ", szFilename + pd.extStart);
}

void Loader::loadMackeyGlassData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	GMatrix raw;
	raw.loadArff("data/mackey_glass.arff");
	
	size_t dims = 1;
	size_t offset = 30;
	size_t train_size = 300;
	size_t test_size = 600;
	
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		*x = double(i) / train_size;
		*y = raw[offset + i][1];
	}
}

void GRecommenderLib::loadData(GMatrix& data, const char* szFilename)
{
	PathData pd;
	GFile::parsePath(szFilename, &pd);
	if(_stricmp(szFilename + pd.extStart, ".sparse") == 0)
	{
		GDom doc;
		doc.loadJson(szFilename);
		GSparseMatrix sm(doc.root());
		data.resize(0, 3);
		for(size_t i = 0; i < sm.rows(); i++)
		{
			GSparseMatrix::Iter rowEnd = sm.rowEnd(i);
			for(GSparseMatrix::Iter it = sm.rowBegin(i); it != rowEnd; it++)
			{
				GVec& vec = data.newRow();
				vec[0] = (double)i;
				vec[1] = (double)it->first;
				vec[2] = it->second;
			}
		}
	}
	else if(_stricmp(szFilename + pd.extStart, ".arff") == 0)
		data.loadArff(szFilename);
	else
		throw Ex("Unsupported file format: ", szFilename + pd.extStart);
}

void Loader::loadTemperatureData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	// TODO: determine optimal granularity
	
	GMatrix raw;
	raw.loadArff("data/melbourne-temperature.arff");
	
	size_t dims = 2;
	size_t offset = 0;
	size_t granularity = 7;
	
	double *x, *y;
	
	trainFeat.resize(TRAIN_SIZE, 1);
	trainLab.resize(TRAIN_SIZE, dims);
	testFeat.resize(TEST_SIZE, 1);
	testLab.resize(TEST_SIZE, dims);
	
	for(size_t i = 0; i < TRAIN_SIZE + TEST_SIZE; i++)
	{
		if(i < TRAIN_SIZE)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - TRAIN_SIZE];
			y = testLab[i - TRAIN_SIZE];
		}
		
		x[0] = double(i) / TRAIN_SIZE;
		
		for(size_t j = 0; j < dims; j++)
		{
			y[j] = 0.0;
			for(size_t k = granularity; k < granularity * 2; k++)
			{
				y[j] += raw[offset + i * granularity + k][j + 1];
			}
			y[j] /= double(granularity);
			y[j] *= 0.05;
		}
	}
	
	// Smooth input data
	
	for(size_t i = 1; i < TRAIN_SIZE - 1; i++)
	{
		for(size_t j = 0; j < dims; j++)
		{
			trainLab[i][j] = (trainLab[i-1][j] + trainLab[i][j] + trainLab[i+1][j]) / 3.0;
		}
	}
}

void LoadData(GArgReader &args, std::unique_ptr<GMatrix> &hOutput)
{
	// Load the dataset by extension
	if(args.size() < 1)
		throw Ex("Expected the filename of a datset. (Found end of arguments.)");
	const char* szFilename = args.pop_string();
	PathData pd;
	GFile::parsePath(szFilename, &pd);
	GMatrix data;
	vector<size_t> abortedCols;
	vector<size_t> ambiguousCols;
	const char *input_type;
	if (args.next_is_flag() && args.if_pop("-input_type")) {
		input_type = args.pop_string();
	} else { /* deduce it from extension (if any) */
		input_type = szFilename + pd.extStart;
		if (*input_type != '.') /* no extension - assume ARFF */
			input_type = "arff";
		else
			input_type++;
	}
	
	// Now load the data
	if(_stricmp(input_type, "arff") == 0)
	{
		data.loadArff(szFilename);
	}
	else if(_stricmp(input_type, "csv") == 0)
	{
		GCSVParser parser;
		parser.parse(data, szFilename);
		cerr << "\nParsing Report:\n";
		for(size_t i = 0; i < data.cols(); i++)
			cerr << to_str(i) << ") " << parser.report(i) << "\n";
	}
	else if(_stricmp(input_type, "dat") == 0)
	{
		GCSVParser parser;
		parser.setSeparator('\0');
		parser.parse(data, szFilename);
		cerr << "\nParsing Report:\n";
		for(size_t i = 0; i < data.cols(); i++)
			cerr << to_str(i) << ") " << parser.report(i) << "\n";
	}
	else
	{
		throw Ex("Unsupported file format: ", szFilename + pd.extStart);
	}
	
	// Split data into a feature matrix and a label matrix
	GMatrix* pFeatures = data.cloneSub(0, 0, data.rows(), data.cols());
	hOutput.reset(pFeatures);
}

GMatrix* loadData(const char* szFilename)
{
	// Load the dataset by extension
	PathData pd;
	GFile::parsePath(szFilename, &pd);
	GMatrix* pData = new GMatrix();
	if(_stricmp(szFilename + pd.extStart, ".arff") == 0)
		pData->loadArff(szFilename);
	else if(_stricmp(szFilename + pd.extStart, ".csv") == 0)
		pData->loadCsv(szFilename, ',', false, false);
	else if(_stricmp(szFilename + pd.extStart, ".dat") == 0)
		pData->loadCsv(szFilename, '\0', false, false);
	else
		throw Ex("Unsupported file format: ", szFilename + pd.extStart);
	return pData;
}

void Loader::loadPrecipitationData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	// Only 87 samples, so TEST_SIZE should be 23 if TRAIN_SIZE is 64
	
	GMatrix raw;
	raw.loadArff("data/precipitation.arff");
	
	size_t dims = 1;
	double *x, *y;
	
	trainFeat.resize(TRAIN_SIZE, 1);
	trainLab.resize(TRAIN_SIZE, dims);
	testFeat.resize(TEST_SIZE, 1);
	testLab.resize(TEST_SIZE, dims);
	
	for(size_t i = 0; i < TRAIN_SIZE + TEST_SIZE; i++)
	{
		if(i < TRAIN_SIZE)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - TRAIN_SIZE];
			y = testLab[i - TRAIN_SIZE];
		}
		
		x[0] = double(i) / TRAIN_SIZE;
		y[0] = raw[i][1] * 0.05;
		// y[1] = raw[i][2] * 0.05;
		// y[2] = raw[i][3] * 0.05;
		// y[3] = raw[i][4] * 0.05;
	}
	
	for(size_t i = 1; i < TRAIN_SIZE - 1; i++)
	{
		for(size_t j = 0; j < dims; j++)
		{
			trainLab[i][j] = (trainLab[i-1][j] + 2 * trainLab[i][j] + trainLab[i+1][j]) * 0.25;
		}
	}
}

void Loader::loadSunSpotData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	GMatrix raw;
	raw.loadArff("data/sunspots.arff");
	
	size_t dims = 1;
	size_t offset = 0;
	size_t granularity = 6;
	size_t train_size = 240;
	size_t test_size = 240;
	
	double scale = 0.01;
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		*x = double(i) / train_size;
		
		*y = 0.0;
		for(size_t j = 0; j < granularity; j++)
		{
			*y += scale * raw[offset + granularity * i + j][3];
		}
		*y /= granularity;
	}
}

void Loader::loadStockData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	// 3   - AAPL
	// 108 - MSFT
	// 157 - DJI
	
	GMatrix raw;
	raw.loadArff("data/stocks.arff");
	
	size_t dims = 1;
	size_t offset = 600;
	size_t train_size = 500;
	size_t test_size = 300;
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	double log_10 = log(10);
	double vert_offset = 3.9;
	double scale = 20;
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		x[0] = double(i) / train_size;
		y[0] = ((log(raw[offset + i][157]) / log_10) - vert_offset) * scale;
	}
}

void Loader::loadAirPassengerData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	GMatrix raw;
	raw.loadArff("data/air_passengers.arff");
	
	size_t dims = 1;
	size_t offset = 0;
	size_t train_size = 72;
	size_t test_size = 72;
	
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	double log_10 = log(10);
	double vert_offset = 2;
	double scale = 10;//0.1;
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		*x = double(i) / train_size;
		*y = ((log(scale * raw[offset + i][0]) / log_10) - vert_offset);
	}
	
	trainLab.saveArff("out/train.arff");
	testLab.saveArff("out/test.arff");
}

GSparseMatrix* GRecommenderLib::loadSparseData(const char* szFilename)
{
	// Load the dataset by extension
	PathData pd;
	GFile::parsePath(szFilename, &pd);
	if(_stricmp(szFilename + pd.extStart, ".arff") == 0)
	{
		// Convert a 3-column dense ARFF file to a sparse matrix
		GMatrix data;
		data.loadArff(szFilename);
		if(data.cols() != 3)
			throw Ex("Expected 3 columns: 0) user or row-index, 1) item or col-index, 2) value or rating");
		double m0 = data.columnMin(0);
		double r0 = data.columnMax(0) - m0;
		double m1 = data.columnMin(1);
		double r1 = data.columnMax(1) - m1;
		if(m0 < 0 || m0 > 1e10 || r0 < 2 || r0 > 1e10)
			throw Ex("Invalid row indexes");
		if(m1 < 0 || m1 > 1e10 || r1 < 2 || r1 > 1e10)
			throw Ex("Invalid col indexes");
		GSparseMatrix* pMatrix = new GSparseMatrix(size_t(m0 + r0) + 1, size_t(m1 + r1) + 1, UNKNOWN_REAL_VALUE);
		std::unique_ptr<GSparseMatrix> hMatrix(pMatrix);
		for(size_t i = 0; i < data.rows(); i++)
		{
			GVec& row = data.row(i);
			pMatrix->set(size_t(row[0]), size_t(row[1]), row[2]);
		}
		return hMatrix.release();
	}
	else if(_stricmp(szFilename + pd.extStart, ".sparse") == 0)
	{
		GDom doc;
		doc.loadJson(szFilename);
		return new GSparseMatrix(doc.root());
	}
	throw Ex("Unsupported file format: ", szFilename + pd.extStart);
	return NULL;
}

void Loader::loadOzoneData(GMatrix &trainFeat, GMatrix &trainLab, GMatrix &testFeat, GMatrix &testLab)
{
	GMatrix raw;
	raw.loadArff("data/mhsets_monthly-ozone.arff");
	
	size_t dims = 1;
	size_t offset = 0;
	size_t train_size = 108;
	size_t test_size = 44;
	
	double *x, *y;
	
	trainFeat.resize(train_size, 1);
	trainLab.resize(train_size, dims);
	testFeat.resize(test_size, 1);
	testLab.resize(test_size, dims);
	
	for(size_t i = 0; i < train_size + test_size; i++)
	{
		if(i < train_size)
		{
			x = trainFeat[i];
			y = trainLab[i];
		}
		else
		{
			x = testFeat[i - train_size];
			y = testLab[i - train_size];
		}
		
		*x = double(i) / train_size;
		*y = log(raw[offset + i][0]) / log(10);
	}
	
	trainLab.saveArff("out/train.arff");
	testLab.saveArff("out/test.arff");
}

void doit()
{
	// Load the data
	GMatrix trainLab;
	GMatrix testLab;
	if (chdir("../bin") != 0)
	{
	}
	trainLab.loadArff("train.arff");
	testLab.loadArff("test.arff");
	double dataMin = trainLab.columnMin(0);
	double dataMax = trainLab.columnMax(0);
	trainLab.normalizeColumn(0, dataMin, dataMax, -5.0, 5.0);
	testLab.normalizeColumn(0, dataMin, dataMax, -5.0, 5.0);
	GMatrix trainFeat(trainLab.rows(), 1);
	for(size_t i = 0; i < trainLab.rows(); i++)
		trainFeat[i][0] = (double)i / trainLab.rows() - 0.5;
	GMatrix testFeat(testLab.rows(), 1);
	for(size_t i = 0; i < testLab.rows(); i++)
		testFeat[i][0] = (double)(i + trainLab.rows()) / trainLab.rows() - 0.5;

	// Make a neural network
	GNeuralNet nn;
	GUniformRelation relOne(1);
	nn.beginIncrementalLearning(relOne, relOne);

	// Initialize the weights of the sine units to match the frequencies used by the Fourier transform.
	GLayerClassic* pSine2 = new GLayerClassic(1, 64, new GActivationSin());
	GMatrix& wSin = pSine2->weights();
	GVec& bSin = pSine2->bias();
	for(size_t i = 0; i < pSine2->outputs() / 2; i++)
	{
		wSin[0][2 * i] = 2.0 * M_PI * (i + 1);
		bSin[2 * i] = 0.5 * M_PI;
		wSin[0][2 * i + 1] = 2.0 * M_PI * (i + 1);
		bSin[2 * i + 1] = M_PI;
	}

	// Make the hidden layer
	GLayerMixed* pMix2 = new GLayerMixed();
	pSine2->resize(1, pSine2->outputs(), &nn.rand(), PERTURBATION);
	pMix2->addComponent(pSine2);
	GLayerClassic* pSoftPlus2 = new GLayerClassic(1, SOFTPLUS_NODES, new GActivationSoftPlus());
	pMix2->addComponent(pSoftPlus2);
	GLayerClassic* pIdentity2 = new GLayerClassic(1, IDENTITY_NODES, new GActivationIdentity());
	pMix2->addComponent(pIdentity2);
	nn.addLayer(pMix2);

	// Make the output layer
	GLayerClassic* pIdentity3 = new GLayerClassic(FLEXIBLE_SIZE, trainLab.cols(), new GActivationIdentity());
	pIdentity3->resize(pMix2->outputs(), pIdentity3->outputs(), &nn.rand(), PERTURBATION);
	nn.addLayer(pIdentity3);

	// Initialize all the non-periodic nodes to approximate the identity function, then perturb a little bit
	pSoftPlus2->setWeightsToIdentity();
	for(size_t i = 0; i < SOFTPLUS_NODES; i++)
	{
		pSoftPlus2->bias()[i] += SOFTPLUS_SHIFT;
		pIdentity3->renormalizeInput(pSine2->outputs() + i, 0.0, 1.0, SOFTPLUS_SHIFT, SOFTPLUS_SHIFT + 1.0);
	}
	pIdentity2->setWeightsToIdentity();
	pSoftPlus2->perturbWeights(nn.rand(), PERTURBATION);
	pIdentity2->perturbWeights(nn.rand(), PERTURBATION);

	// Randomly initialize the weights on the output layer
	pIdentity3->weights().setAll(0.0);
	pIdentity3->perturbWeights(nn.rand(), PERTURBATION);

	// Open Firefox to view the progress
	GApp::systemCall("firefox ./view.html#progress.svg", false, true);

	// Do some training
	GRandomIndexIterator ii(trainLab.rows(), nn.rand());
	nn.setLearningRate(LEARNING_RATE);
	for(size_t epoch = 0; epoch < TRAINING_EPOCHS; epoch++)
	{
		// Visit each sample in random order
		ii.reset();
		size_t i;
		while(ii.next(i))
		{
			// Regularize
			pIdentity3->scaleWeights(1.0 - nn.learningRate() * REGULARIZATION_TERM, true);
			pIdentity3->diminishWeights(nn.learningRate() * REGULARIZATION_TERM, true);

			// Train
			nn.trainIncremental(trainFeat[i], trainLab[i]); // One iteration of stochastic gradient descent
		}

		// Report progress
		double rmse = sqrt(nn.sumSquaredError(trainFeat, trainLab) / trainLab.rows());
		if(epoch % (TRAINING_EPOCHS / 100) == 0)
		{
			double val = sqrt(nn.sumSquaredError(testFeat, testLab) / testLab.rows());
			cout << "prog=" << to_str((double)epoch * 100.0 / TRAINING_EPOCHS) << "%	rmse=" << to_str(rmse) << "	val=" << to_str(val) << "\n";
			plot_it("progress.svg", nn, trainFeat, trainLab, testFeat, testLab);
		}
	}
}