Java SloppyMath类代码示例

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * Calculate the log probabilities of each class, for the given datum
 * (feature bundle).
 */
public <F, L> double[] getLogProbabilities(EncodedDatum datum,
		double[] weights, Encoding<F, L> encoding,
		IndexLinearizer indexLinearizer) {
	// Compute unnormalized log probabilities
	int numSubLabels = encoding.getNumSubLabels();
	double[] logProbabilities = DoubleArrays.constantArray(0.0,
			numSubLabels);
	for (int i = 0; i < datum.getNumActiveFeatures(); i++) {
		int featureIndex = datum.getFeatureIndex(i);
		double featureCount = datum.getFeatureCount(i);
		for (int j = 0; j < numSubLabels; j++) {
			int index = indexLinearizer.getLinearIndex(featureIndex, j);
			double weight = weights[index];
			logProbabilities[j] += weight * featureCount;
		}
	}
	// Normalize
	double logNormalizer = SloppyMath.logAdd(logProbabilities);
	for (int i = 0; i < numSubLabels; i++) {
		logProbabilities[i] -= logNormalizer;
	}

	return logProbabilities;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
@Override
public void delinearizeLexicon(double[] logProbs) {
	int nDangerous = 0;
	for (short tag = 0; tag < lexicon.hierarchicalScores.length; tag++) {
		for (int word = 0; word < lexicon.hierarchicalScores[tag].length; word++) {
			int index = linearIndex[tag][word];
			double[] vals = lexicon.getLastLevel(tag, word);
			for (int substate = 0; substate < vals.length; substate++) {
				double val = logProbs[index++];
				if (SloppyMath.isVeryDangerous(val)) {
					nDangerous++;
					continue;
				}
				vals[substate] = val;
			}
		}
	}
	if (nDangerous > 0)
		System.out
				.println("Left "
						+ nDangerous
						+ " lexicon weights unchanged since the proposed weight was dangerous.");
	lexicon.explicitlyComputeScores(finalLevel);
	// System.out.println(lexicon);
	// return lexicon;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * @param goldLL
 * @param allLL
 * @param stateSetTree
 * @return
 */
protected boolean sanityCheckLLs(double goldLL, double allLL,
		Tree<StateSet> stateSetTree) {
	if (SloppyMath.isVeryDangerous(allLL)
			|| SloppyMath.isVeryDangerous(goldLL)) {
		unparsableTrees++;
		return false;
	}

	if (goldLL - allLL > 1.0e-4) {
		System.out.println("Something is wrong! The gold LL is "
				+ goldLL + " and the all LL is " + allLL);// +"\n"+sentence+"\n"+stateSetTree);
		System.out.println(stateSetTree);
		incorrectLLTrees++;
		return false;
	}
	return true;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * Calculate the log probabilities of each class, for the given datum
 * (feature bundle). Note that the weighted votes (refered to as
 * activations) are *almost* log probabilities, but need to be normalized.
 */
private static <F, L> double[] getLogProbabilities(EncodedDatum datum,
		double[] weights, Encoding<F, L> encoding,
		IndexLinearizer indexLinearizer) {

	double[] logProbabilities = new double[encoding.getNumLabels()];
	for (int labelIndex = 0; labelIndex < encoding.getNumLabels(); ++labelIndex) {
		for (int num = 0; num < datum.getNumActiveFeatures(); ++num) {
			int featureIndex = datum.getFeatureIndex(num);
			double featureCount = datum.getFeatureCount(num);
			int linearFeatureIndex = indexLinearizer.getLinearIndex(
					featureIndex, labelIndex);
			logProbabilities[labelIndex] += weights[linearFeatureIndex]
					* featureCount;
		}
	}

	double logSumProb = SloppyMath.logAdd(logProbabilities);
	for (int labelIndex = 0; labelIndex < encoding.getNumLabels(); ++labelIndex) {
		logProbabilities[labelIndex] -= logSumProb;
	}

	return logProbabilities;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public static void scaleArrayToScale(double[] scores, int previousScale,
		int newScale) {
	int scaleDiff = previousScale - newScale;
	if (scaleDiff == 0)
		return; // nothing to do
	double max = DoubleArrays.max(scores);
	if (SloppyMath.isDangerous(max))
		return;

	double scale = calcScaleFactor(scaleDiff);

	if (Math.abs(scaleDiff) >= 800) {
		// under-/overflow...
		Arrays.fill(scores, 0.0);
		return;
	}

	for (int i = 0; i < scores.length; i++) {
		scores[i] *= scale;
	}
	// if (SloppyMath.isDangerous(ArrayMath.max(scores))){
	// System.out.println("Undeflow when scaling scores!");
	// }
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public static double scaleToScale(double score, int previousScale,
		int newScale) {
	int scaleDiff = previousScale - newScale;
	if (scaleDiff == 0)
		return score; // nothing to do
	double max = score;

	if (SloppyMath.isDangerous(max))
		return 0;

	double scale = calcScaleFactor(scaleDiff);

	if (Math.abs(scaleDiff) >= 800) {
		// under-/overflow...
		return 0;
	}

	score *= scale;

	return score;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public double l2_regularize(double objective, double[] derivatives) {
	// Incorporate penalty terms (regularization) into the objective and
	// derivatives
	if (SloppyMath.isVeryDangerous(objective))
		return objective;
	double sigma2 = sigma * sigma;
	double penalty = 0.0;
	for (int index = 0; index < x.length; index++) {
		// if (lastX[index]==10000 || Double.isInfinite(lastX[index]))
		// continue;
		penalty += x[index] * x[index];
	}
	// System.out.print(" penalty="+penalty);
	objective -= penalty / (2 * sigma2);

	for (int index = 0; index < x.length; index++) {
		// 'x' and 'derivatives' have same layout
		// if (lastX[index]==10000 || Double.isInfinite(lastX[index]))
		// continue;
		derivatives[index] -= x[index] / sigma2;
		if (SloppyMath.isVeryDangerous(derivatives[index])) {
			System.out
					.println("Setting regularized derivative to zero because it is Inf.");
			derivatives[index] = 0;
		}
	}
	return objective;

}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public void delinearizeLexicon(double[] logProbs) {
	int nDangerous = 0;
	for (short tag = 0; tag < lexicon.scores.length; tag++) {
		for (int word = 0; word < lexicon.scores[tag][0].length; word++) {
			int index = linearIndex[tag][word];
			for (int substate = 0; substate < lexicon.numSubStates[tag]; substate++) {
				double val = Math.exp(logProbs[index++]);
				if (SloppyMath.isVeryDangerous(val)) {
					System.out
							.println("dangerous value when delinearizng lexicon "
									+ lexicon.scores[tag][substate][word]);
					System.out.println("Word "
							+ lexicon.wordIndexer
									.get(lexicon.tagWordIndexer[tag]
											.get(word)) + " tag "
							+ logProbs[index - 1]);
					val = 0;
					nDangerous++;
					// continue;
				}
				lexicon.scores[tag][substate][word] = val;
			}
		}
	}
	if (nDangerous > 0)
		System.out
				.println("Left "
						+ nDangerous
						+ " lexicon weights unchanged since the proposed weight was dangerous.");
	// return lexicon;
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * 
 * @param logScores
 * @param <K>
 */
public static <K> void makeProbsFromLogScoresInPlace(Counter<K> logScores) {
	double logSum = SloppyMath.logAdd(logScores);
	for (Map.Entry<K, Double> entry : logScores.entrySet()) {
		double logScore = entry.getValue();
		double prob = Math.exp(logScore - logSum);
		entry.setValue(prob);
	}
	logScores.setDirty(true);
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public static double approxLogSum(double[] logInputs, int leng) {

		if (leng == 0) {
			throw new IllegalArgumentException();
		}
		int maxIdx = 0;
		double max = logInputs[0];
		for (int i = 1; i < leng; i++) {
			if (logInputs[i] > max) {
				maxIdx = i;
				max = logInputs[i];
			}
		}
		boolean haveTerms = false;
		double intermediate = 0.0;
		double cutoff = max - SloppyMath.LOGTOLERANCE;
		// we avoid rearranging the array and so test indices each time!
		for (int i = 0; i < leng; i++) {
			if (i != maxIdx && logInputs[i] > cutoff) {
				haveTerms = true;
				intermediate += SloppyMath.approxExp(logInputs[i] - max);
			}
		}
		if (haveTerms) {
			return max + SloppyMath.approxLog(1.0 + intermediate);
		} else {
			return max;
		}
	}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public static double logSum(double[] logInputs, int leng) {

		if (leng == 0) {
			throw new IllegalArgumentException();
		}
		int maxIdx = 0;
		double max = logInputs[0];
		for (int i = 1; i < leng; i++) {
			if (logInputs[i] > max) {
				maxIdx = i;
				max = logInputs[i];
			}
		}
		boolean haveTerms = false;
		double intermediate = 0.0;
		double cutoff = max - SloppyMath.LOGTOLERANCE;
		// we avoid rearranging the array and so test indices each time!
		for (int i = 0; i < leng; i++) {
			if (i != maxIdx && logInputs[i] > cutoff) {
				haveTerms = true;
				intermediate += Math.exp(logInputs[i] - max);
			}
		}
		if (haveTerms) {
			return max + Math.log(1.0 + intermediate);
		} else {
			return max;
		}
	}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
public void checkScores(Tree<StateSet> tree) {
	StateSet node = tree.getLabel();
	int state = node.getState();
	int from = node.from, to = node.to;
	int oldS = iScale[from][to][state];
	int newS = ScalingTools.scaleArray(iScorePostU[from][to][state], oldS);
	if (oldS > newS) {
		System.out.println("why?? iscale");
	}

	oldS = oScale[from][to][state];
	newS = ScalingTools.scaleArray(oScorePostU[from][to][state], oldS);
	if (oldS > newS) {
		ScalingTools.scaleArrayToScale(oScorePostU[from][to][state], newS,
				oldS);
		System.out.println("why?? oscale");
	}
	for (int substate = 0; substate < numSubStatesArray[state]; substate++) {
		if ((node.getIScale() == iScale[from][to][state])
				&& (!SloppyMath.isGreater(
						iScorePostU[from][to][state][substate],
						node.getIScore(substate)))) {
			if (!allowedSubStates[from][to][state][substate])
				System.out.println("This state was pruned!");
			else {
				System.out.println("Gold iScore is higher for state "
						+ state + " from " + from + " to " + to + "!");
				System.out.println("Gold " + node.getIScore(substate)
						+ " all " + iScorePostU[from][to][state][substate]);
			}
		}
		double tmpA = node.getOScore(substate);
		double tmpB = oScorePostU[from][to][state][substate];
		if ((node.getOScale() == oScale[from][to][state])
				&& (!SloppyMath.isGreater(tmpB, tmpA))) {
			if (!allowedSubStates[from][to][state][substate])
				System.out.println("This state was pruned!");
			else {
				System.out.println("Gold oScore is higher for state "
						+ state + " from " + from + " to " + to + "!");
				System.out.println("Gold " + node.getOScore(substate)
						+ " all " + oScorePostU[from][to][state][substate]);
			}
		}
	}
	for (Tree<StateSet> child : tree.getChildren()) {
		if (!child.isLeaf())
			checkScores(child);
	}
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * Computes the longest common substring of s and t. The longest common
 * substring of a and b is the longest run of characters that appear in
 * order inside both a and b. Both a and b may have other extraneous
 * characters along the way. This is like edit distance but with no
 * substitution and a higher number means more similar. For example, the LCS
 * of "abcD" and "aXbc" is 3 (abc).
 */
public static int longestCommonSubstring(String s, String t) {
	int d[][]; // matrix
	int n; // length of s
	int m; // length of t
	int i; // iterates through s
	int j; // iterates through t
	char s_i; // ith character of s
	char t_j; // jth character of t
	// Step 1
	n = s.length();
	m = t.length();
	if (n == 0) {
		return 0;
	}
	if (m == 0) {
		return 0;
	}
	d = new int[n + 1][m + 1];
	// Step 2
	for (i = 0; i <= n; i++) {
		d[i][0] = 0;
	}
	for (j = 0; j <= m; j++) {
		d[0][j] = 0;
	}
	// Step 3
	for (i = 1; i <= n; i++) {
		s_i = s.charAt(i - 1);
		// Step 4
		for (j = 1; j <= m; j++) {
			t_j = t.charAt(j - 1);
			// Step 5
			// js: if the chars match, you can get an extra point
			// otherwise you have to skip an insertion or deletion (no subs)
			if (s_i == t_j) {
				d[i][j] = SloppyMath.max(d[i - 1][j], d[i][j - 1],
						d[i - 1][j - 1] + 1);
			} else {
				d[i][j] = Math.max(d[i - 1][j], d[i][j - 1]);
			}
		}
	}
	if (false) {
		// num chars needed to display longest num
		int numChars = (int) Math.ceil(Math.log(d[n][m]) / Math.log(10));
		for (i = 0; i < numChars + 3; i++) {
			System.err.print(' ');
		}
		for (j = 0; j < m; j++) {
			System.err.print("" + t.charAt(j) + " ");
		}
		System.err.println();
		for (i = 0; i <= n; i++) {
			System.err.print((i == 0 ? ' ' : s.charAt(i - 1)) + " ");
			for (j = 0; j <= m; j++) {
				System.err.print("" + d[i][j] + " ");
			}
			System.err.println();
		}
	}
	// Step 7
	return d[n][m];
}
 

import edu.berkeley.nlp.math.SloppyMath; //导入依赖的package包/类
/**
 * Computes the Levenshtein (edit) distance of the two given Strings.
 */
public static int editDistance(String s, String t) {
	int d[][]; // matrix
	int n; // length of s
	int m; // length of t
	int i; // iterates through s
	int j; // iterates through t
	char s_i; // ith character of s
	char t_j; // jth character of t
	int cost; // cost
	// Step 1
	n = s.length();
	m = t.length();
	if (n == 0) {
		return m;
	}
	if (m == 0) {
		return n;
	}
	d = new int[n + 1][m + 1];
	// Step 2
	for (i = 0; i <= n; i++) {
		d[i][0] = i;
	}
	for (j = 0; j <= m; j++) {
		d[0][j] = j;
	}
	// Step 3
	for (i = 1; i <= n; i++) {
		s_i = s.charAt(i - 1);
		// Step 4
		for (j = 1; j <= m; j++) {
			t_j = t.charAt(j - 1);
			// Step 5
			if (s_i == t_j) {
				cost = 0;
			} else {
				cost = 1;
			}
			// Step 6
			d[i][j] = SloppyMath.min(d[i - 1][j] + 1, d[i][j - 1] + 1,
					d[i - 1][j - 1] + cost);
		}
	}

	// Step 7
	return d[n][m];
}