Java OpenBitSet.clone方法代码示例

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
/**
 * Checks whether all subsets of X (with length of X - 1) are part of the last level.
 * Only if this check return true X is added to the new level.
 *
 * @param X
 * @return
 */
private boolean checkSubsets(OpenBitSet X) {
    boolean xIsValid = true;

    // clone of X for usage in the following loop
    OpenBitSet Xclone = (OpenBitSet) X.clone();

    for (int l = X.nextSetBit(0); l >= 0; l = X.nextSetBit(l + 1)) {
        Xclone.clear(l);
        if (!level0.containsKey(Xclone)) {
            xIsValid = false;
            break;
        }
        Xclone.set(l);
    }

    return xIsValid;
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private OpenBitSet extendWith(OpenBitSet ucc, int extensionAttr) {
		if (ucc.get(extensionAttr))
			return null;
		
		OpenBitSet childUCC = ucc.clone();
		childUCC.set(extensionAttr);
		
		if (this.posCover.containsUCCOrGeneralization(childUCC))
			return null;
		
//		if (this.negCover.containsUCCOrSpecialization(childUCC)) // TODO: May be needed?
//			return null;
		
		return childUCC;
	}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
public void runNext(UCCList newNonUCCs, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	OpenBitSet equalAttrs = new OpenBitSet(this.posCover.getNumAttributes());
	
	int previousNewNonFdsSize = newNonUCCs.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			
			this.sampler.match(equalAttrs, compressedRecords[recordId], compressedRecords[partnerRecordId]);
			
			if (!this.negCover.contains(equalAttrs)) {
				OpenBitSet equalAttrsCopy = equalAttrs.clone();
				this.negCover.add(equalAttrsCopy);
				newNonUCCs.add(equalAttrsCopy);
				
				this.memoryGuardian.memoryChanged(1);
				this.memoryGuardian.match(this.negCover, this.posCover, newNonUCCs);
			}
			numComparisons++;
		}
	}
	numNewNonFds = newNonUCCs.size() - previousNewNonFdsSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private void recurseTerms(List<List<TermBitSet>> bitSets, int index, String[] curValues, OpenBitSet curBits,
	MatrixResults results, IndexReader reader, boolean countOnly)
{
	List<TermBitSet> termBitSetList = bitSets.get(index);
	boolean last = index == curValues.length - 1;
	for( TermBitSet termBitSet : termBitSetList )
	{
		OpenBitSet termBits = termBitSet.bitSet;
		Term term = termBitSet.term;
		// if we don't intersect there's no point in recursing further in
		if( curBits.intersects(termBits) )
		{
			// Collect current term's value into the value array
			curValues[index] = term.text();
			OpenBitSet docBits = (OpenBitSet) curBits.clone();
			docBits.and(termBits);
			if( last )
			{
				int count;
				List<ItemIdKey> ids = null;
				ArrayList<String> vals = new ArrayList<String>(Arrays.asList(curValues));
				if( !countOnly )
				{
					ids = getIdsForBitset(docBits, reader);
					count = ids.size();
				}
				else
				{
					count = (int) docBits.cardinality();
				}
				results.addEntry(new MatrixResults.MatrixEntry(vals, ids, count));
			}
			else
			{
				recurseTerms(bitSets, index + 1, curValues, docBits, results, reader, countOnly);
			}
		}
	}
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
public void updatePositiveCover(FDList nonFds) {
/*		if (nonFds.isEmpty())
			return;
		
		// Sort the negative cover
		Logger.getInstance().writeln("Sorting FD-violations ...");
		Collections.sort(nonFds, new Comparator<OpenBitSet>() {
			@Override
			public int compare(OpenBitSet o1, OpenBitSet o2) {
				return (int)(o1.cardinality() - o2.cardinality());
			}
		});
*/		// THE SORTING IS NOT NEEDED AS THE UCCSet SORTS THE NONUCCS BY LEVEL ALREADY
		
		Logger.getInstance().writeln("Inducing FD candidates ...");
		for (int i = nonFds.getFdLevels().size() - 1; i >= 0; i--) {
			if (i >= nonFds.getFdLevels().size()) // If this level has been trimmed during iteration
				continue;
			
			List<OpenBitSet> nonFdLevel = nonFds.getFdLevels().get(i);
			for (OpenBitSet lhs : nonFdLevel) {
				
				OpenBitSet fullRhs = lhs.clone();
				fullRhs.flip(0, this.posCover.getNumAttributes());
				
				for (int rhs = fullRhs.nextSetBit(0); rhs >= 0; rhs = fullRhs.nextSetBit(rhs + 1))
					this.specializePositiveCover(lhs, rhs, nonFds);
			}
			nonFdLevel.clear();
		}
	}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
public void runNext(FDList newNonFds, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	OpenBitSet equalAttrs = new OpenBitSet(this.posCover.getNumAttributes());
	
	int previousNegCoverSize = newNonFds.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			
			this.sampler.match(equalAttrs, compressedRecords[recordId], compressedRecords[partnerRecordId]);
			
			if (!this.negCover.contains(equalAttrs)) {
				OpenBitSet equalAttrsCopy = equalAttrs.clone();
				this.negCover.add(equalAttrsCopy);
				newNonFds.add(equalAttrsCopy);
				
				this.memoryGuardian.memoryChanged(1);
				this.memoryGuardian.match(this.negCover, this.posCover, newNonFds);
			}
			numComparisons++;
		}
	}
	numNewNonFds = newNonFds.size() - previousNegCoverSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
public void grow(OpenBitSet lhs, FDTree fdTree) {
	// Add specializations of all nodes an mark them as isFD, but if specialization exists, then it is invalid and should not be marked; only add specializations of nodes not marked as isFD!
	OpenBitSet rhs = this.rhsAttributes;
	
	OpenBitSet invalidRhs = rhs.clone();
	invalidRhs.remove(this.rhsFds);
	
	// Add specializations that are not invalid
	if (invalidRhs.cardinality() > 0) {
		for (int extensionAttr = 0; extensionAttr < this.numAttributes; extensionAttr++) {
			if (lhs.get(extensionAttr) || rhs.get(extensionAttr))
				continue;
			
			lhs.set(extensionAttr);
			fdTree.addFunctionalDependencyIfNotInvalid(lhs, invalidRhs);
			lhs.clear(extensionAttr);
		}
	}
	
	// Traverse children and let them add their specializations
	if (this.children != null) {
		for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
			FDTreeElement element = this.children[childAttr];
			if (element != null) {
				lhs.set(childAttr);
				element.grow(lhs, fdTree);
				lhs.clear(childAttr);
			}
		}
	}
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private void computeDependencies(int l) throws AlgorithmExecutionException {
    initializeCplusForLevel();

    // iterate through the combinations of the level
    for (OpenBitSet X : level1.keySet()) {
        // Build the intersection between X and C_plus(X)
        OpenBitSet C_plus = level1.get(X).getRhsCandidates();
        OpenBitSet intersection = (OpenBitSet) X.clone();
        intersection.intersect(C_plus);

        // clone of X for usage in the following loop
        OpenBitSet Xclone = (OpenBitSet) X.clone();

        // iterate through all elements (A) of the intersection
        for (int A = intersection.nextSetBit(0); A >= 0; A = intersection.nextSetBit(A + 1)) {
            Xclone.clear(A);

            // check if X\A -> A is valid
            StrippedPartition spXwithoutA = level0.get(Xclone).getPartition();
            StrippedPartition spX = level1.get(X).getPartition();

            if (spX.getError() == spXwithoutA.getError()) {
                // add fd to FDTree. Filter the tree at the end of the algorithm.
                dependencies.addFunctionalDependency(Xclone, A);


                // remove A from C_plus(X)
                level1.get(X).getRhsCandidates().clear(A);

                // remove all B in R\X from C_plus(X)
                OpenBitSet RwithoutX = new OpenBitSet();
                // set to R
                RwithoutX.set(1, numberAttributes + 1);
                // remove X
                RwithoutX.andNot(X);

                for (int i = RwithoutX.nextSetBit(0); i >= 0; i = RwithoutX.nextSetBit(i + 1)) {
                    level1.get(X).getRhsCandidates().clear(i);
                }

            }
            Xclone.set(A);
        }
    }
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
/**
 * Computes the dependencies for the current level (level1).
 *
 * @throws AlgorithmExecutionException
 */
private void computeDependencies() throws AlgorithmExecutionException {
    initializeCplusForLevel();

    // iterate through the combinations of the level
    for (OpenBitSet X : level1.keySet()) {
        if (level1.get(X).isValid()) {
            // Build the intersection between X and C_plus(X)
            OpenBitSet C_plus = level1.get(X).getRhsCandidates();
            OpenBitSet intersection = (OpenBitSet) X.clone();
            intersection.intersect(C_plus);

            // clone of X for usage in the following loop
            OpenBitSet Xclone = (OpenBitSet) X.clone();

            // iterate through all elements (A) of the intersection
            for (int A = intersection.nextSetBit(0); A >= 0; A = intersection.nextSetBit(A + 1)) {
                Xclone.clear(A);

                // check if X\A -> A is valid
                StrippedPartition spXwithoutA = level0.get(Xclone).getPartition();
                StrippedPartition spX = level1.get(X).getPartition();

                if (spX.getError() == spXwithoutA.getError()) {
                    // found Dependency
                    OpenBitSet XwithoutA = (OpenBitSet) Xclone.clone();
                    processFunctionalDependency(XwithoutA, A);

                    // remove A from C_plus(X)
                    level1.get(X).getRhsCandidates().clear(A);

                    // remove all B in R\X from C_plus(X)
                    OpenBitSet RwithoutX = new OpenBitSet();
                    // set to R
                    RwithoutX.set(1, numberAttributes + 1);
                    // remove X
                    RwithoutX.andNot(X);

                    for (int i = RwithoutX.nextSetBit(0); i >= 0; i = RwithoutX.nextSetBit(i + 1)) {
                        level1.get(X).getRhsCandidates().clear(i);
                    }

                }
                Xclone.set(A);
            }
        }
    }
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private void computeDependencies(int l) throws AlgorithmExecutionException {
    initializeCplusForLevel();

    // iterate through the combinations of the level
    for (OpenBitSet X : level1.keySet()) {
        // Build the intersection between X and C_plus(X)
        OpenBitSet C_plus = level1.get(X).getRhsCandidates();
        OpenBitSet intersection = (OpenBitSet) X.clone();
        intersection.intersect(C_plus);

        // clone of X for usage in the following loop
        OpenBitSet Xclone = (OpenBitSet) X.clone();

        // iterate through all elements (A) of the intersection
        for (int A = intersection.nextSetBit(0); A >= 0; A = intersection.nextSetBit(A + 1)) {
            Xclone.clear(A);

            // check if X\A -> A is valid
            StrippedPartition spXwithoutA = level0.get(Xclone).getPartition();
            StrippedPartition spX = level1.get(X).getPartition();

            if (spX.getError() == spXwithoutA.getError()) {
                // Add dependency to the result receiver as well as to the FDTree (dependencies),
                // which is used to check, whether a more general dependency already exists.
                OpenBitSet XwithoutA = (OpenBitSet) Xclone.clone();
                addDependencyToResultReceiver(XwithoutA, A);
                dependencies.addFunctionalDependency(Xclone, A);


                // remove A from C_plus(X)
                level1.get(X).getRhsCandidates().clear(A);

                // remove all B in R\X from C_plus(X)
                OpenBitSet RwithoutX = new OpenBitSet();
                // set to R
                RwithoutX.set(1, numberAttributes + 1);
                // remove X
                RwithoutX.andNot(X);

                for (int i = RwithoutX.nextSetBit(0); i >= 0; i = RwithoutX.nextSetBit(i + 1)) {
                    level1.get(X).getRhsCandidates().clear(i);
                }

            }
            Xclone.set(A);
        }
    }
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private boolean checkIfSetIsSuperSetOf(OpenBitSet set, OpenBitSet set2) {
    OpenBitSet setCopy = set.clone();
    setCopy.intersect(set2);
    return setCopy.equals(set2);
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
private FDTree calculatePositiveCover(ArrayList<OpenBitSet> negCover) {
		FDTree posCover = new FDTree(this.numAttributes, this.maxLhsSize);
		posCover.addMostGeneralDependencies();
		
		//
		//int bitsetCounter = 0;
		//long t = System.currentTimeMillis();
		//
		
//		OpenBitSet previous1Lhs = null;
//		OpenBitSet previous2Lhs = null;
		for (int i = negCover.size() - 1; i >= 0; i--) {
			OpenBitSet lhs = negCover.remove(i);

			//
			//bitsetCounter++;
			//if (bitsetCounter % 1 == 0) {
			//	System.out.println("\t\t" + bitsetCounter + "\\" + negCover.size() + " bitsets; " + "- fds; " + (System.currentTimeMillis() - t) + " time");
			//	t = System.currentTimeMillis();
			//}
			//
			
			OpenBitSet fullRhs = lhs.clone();
			fullRhs.flip(0, fullRhs.size());
			
			for (int rhs = fullRhs.nextSetBit(0); rhs >= 0; rhs = fullRhs.nextSetBit(rhs + 1)) {
				// If one of the previous lhs subsumes this lhs with the same rhs, then we can skip it here
//				if ((previous1Lhs != null) && this.subsumes(lhs, previous1Lhs, rhs))
//					continue;
//				if ((previous2Lhs != null) && this.subsumes(lhs, previous2Lhs, rhs))
//					continue;
				
				this.memoryGuardian.memoryChanged(this.specializePositiveCover(posCover, lhs, rhs));
			}
			
			// If dynamic memory management is enabled, frequently check the memory consumption and trim the positive cover if it does not fit anymore
		//	this.memoryGuardian.match(posCover);
			
//			previous2Lhs = previous1Lhs;
//			previous1Lhs = lhs;
		}
		return posCover;
	}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
protected void maximizeNegativeRecursive(PositionListIndex currentPli, OpenBitSet currentLhs, int numAttributes, int[][] rhsPlis, FDTree invalidFds) {
	PositionListIndex[] childPlis = new PositionListIndex[numAttributes];
	
	// Traverse the tree depth-first, left-first; generate plis for children and pass them over; store the child plis locally to reuse them for the checking afterwards
	if (this.children != null) {
		for (int attr = 0; attr < numAttributes; attr++) {
			if (this.children[attr] != null) {
				childPlis[attr] = currentPli.intersect(rhsPlis[attr]);
				
				currentLhs.set(attr);
				this.children[attr].maximizeNegativeRecursive(childPlis[attr], currentLhs, numAttributes, rhsPlis, invalidFds);
				currentLhs.clear(attr);
			}
		}
	}
	
	// On the way back, check all rhs-FDs that all their possible supersets are valid FDs; check with refines or, if available, with previously calculated plis
	//     which supersets to consider: add all attributes A with A notIn lhs and A notequal rhs; 
	//         for newLhs->rhs check that no FdOrSpecialization exists, because it is invalid then; this check might be slower than the FD check on high levels but faster on low levels in particular in the root! this check is faster on the negative cover, because we look for a non-FD
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		OpenBitSet extensions = currentLhs.clone();
		extensions.flip(0, numAttributes);
		extensions.clear(rhs);
		
		for (int extensionAttr = extensions.nextSetBit(0); extensionAttr >= 0; extensionAttr = extensions.nextSetBit(extensionAttr + 1)) {
			currentLhs.set(extensionAttr);
			if (childPlis[extensionAttr] == null)
				childPlis[extensionAttr] = currentPli.intersect(rhsPlis[extensionAttr]);
			
			// If a superset is a non-FD, mark this as not rhsFD, add the superset as a new node, filterGeneralizations() of the new node, call maximizeNegative() on the new supersets node
			//     if the superset node is in a right node of the tree, it will be checked anyways later; hence, only check supersets that are left or in the same tree path
			if (!childPlis[extensionAttr].refines(rhsPlis[rhs])) {
				this.rhsFds.clear(rhs);

				FDTreeElement newElement = invalidFds.addFunctionalDependency(currentLhs, rhs);
				//invalidFds.filterGeneralizations(currentLhs, rhs); // TODO: test effect
				newElement.maximizeNegativeRecursive(childPlis[extensionAttr], currentLhs, numAttributes, rhsPlis, invalidFds);
			}
			currentLhs.clear(extensionAttr);
		}
	}
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
public void maximizeNegative(List<PositionListIndex> plis, int[][] invertedPlis, int numRecords) {
	// Maximizing negative cover is better than maximizing positive cover, because we do not need to check minimality; inversion does this automatically, i.e., generating a non-FD that creates a valid, non-minimal FD is not possible
	int numAttributes = plis.size();
	OpenBitSet currentLhs = new OpenBitSet(numAttributes);
	
	// Traverse the tree depth-first, left-first
	if (this.getChildren() != null) {
		for (int attr = 0; attr < numAttributes; attr++) {
			if (this.getChildren()[attr] != null) {
				currentLhs.set(attr);
				this.getChildren()[attr].maximizeNegativeRecursive(plis.get(attr), currentLhs, numAttributes, invertedPlis, this);
				currentLhs.clear(attr);
			}
		}
	}

	// Add invalid root FDs {} -/-> rhs to negative cover, because these are seeds for not yet discovered non-FDs
	this.addInvalidRootFDs(plis, numRecords); // TODO: These FDs make the search complex again :-(

	// On the way back, check all rhs-FDs that all their possible supersets are valid FDs; check with refines or, if available, with previously calculated plis
	//     which supersets to consider: add all attributes A with A notIn lhs and A notequal rhs; 
	//         for newLhs->rhs check that no FdOrSpecialization exists, because it is invalid then; this check might be slower than the FD check on high levels but faster on low levels in particular in the root! this check is faster on the negative cover, because we look for a non-FD
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		OpenBitSet extensions = currentLhs.clone();
		extensions.flip(0, numAttributes);
		extensions.clear(rhs);
		
		// If a superset is a non-FD, mark this as not rhsFD, add the superset as a new node, filterGeneralizations() of the new node, call maximizeNegative() on the new supersets node
		//     if the superset node is in a right node of the tree, it will be checked anyways later; hence, only check supersets that are left or in the same tree path
		for (int extensionAttr = extensions.nextSetBit(0); extensionAttr >= 0; extensionAttr = extensions.nextSetBit(extensionAttr + 1)) {
			currentLhs.set(extensionAttr);
			
			if (this.containsFdOrSpecialization(currentLhs, rhs) || !plis.get(extensionAttr).refines(invertedPlis[rhs])) { // Otherwise, it must be false and a specialization is already contained; Only needed in root node, because we already filtered generalizations of other nodes and use a depth-first search that always removes generalizations when a new node comes in!
				this.rhsFds.clear(rhs);
				
				FDTreeElement newElement = this.addFunctionalDependency(currentLhs, rhs);
				//this.filterGeneralizations(currentLhs, rhs); // TODO: test effect
				newElement.maximizeNegativeRecursive(plis.get(extensionAttr), currentLhs, numAttributes, invertedPlis, this);
			}
			currentLhs.clear(extensionAttr);
		}
	}
}
 

import org.apache.lucene.util.OpenBitSet; //导入方法依赖的package包/类
/**
 * Get prefix of OpenBitSet by copying it and removing the last Bit.
 *
 * @param bitset
 * @return
 */
private OpenBitSet getPrefix(OpenBitSet bitset) {
    OpenBitSet prefix = (OpenBitSet) bitset.clone();
    prefix.clear(getLastSetBitIndex(prefix));
    return prefix;
}