Java BloomFilter.readFields方法代码示例

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
@Override
protected void setup(Context context) throws IOException,
		InterruptedException {

	// TODO Create a FileSystem object
	FileSystem fs = FileSystem.get(context.getConfiguration());

	// TODO get the cache files from the context
	URI[] uris = context.getCacheFiles();

	if (uris.length > 0) {
		// TODO create a new Bloom filter
		filter = new BloomFilter();
		
		// TODO call the filter's readFields method, passing in an FSDataInputStream
		filter.readFields(fs.open(new Path(uris[0].toString())));
	} else {
		throw new IOException(
				"Bloom filter file not in DistributedCache");
	}
}
 

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
private void init() throws IOException {
    filter = new BloomFilter();
    String dir = "./" + getFilenameFromPath(bloomFile);
    String[] partFiles = new File(dir)
            .list(new FilenameFilter() {
                @Override
                public boolean accept(File current, String name) {
                    return name.startsWith("part");
                }
            });

    String dcFile = dir + "/" + partFiles[0];
    DataInputStream dis = new DataInputStream(new FileInputStream(dcFile));
    try {
        filter.readFields(dis);
    } finally {
        dis.close();
    }
}
 

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
/**
 * For testing only, do not use directly.
 */
public void setFilter(DataByteArray dba) throws IOException {
    DataInputStream dis = new DataInputStream(new
        ByteArrayInputStream(dba.get()));
    filter = new BloomFilter();
    filter.readFields(dis);
}
 

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
protected BloomFilter bloomIn(DataByteArray b) throws IOException {
    DataInputStream dis = new DataInputStream(new
        ByteArrayInputStream(b.get()));
    BloomFilter f = new BloomFilter();
    f.readFields(dis);
    return f;
}
 

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
public void redisMembershipTestWithFilter(Path input, Path bloom)
		throws Exception {

	System.out.println("Testing Redis set membership of " + input
			+ " using a Bloom filter " + bloom);
	
	// TODO create a fileSystem object
	FileSystem fs = FileSystem.get(getConf());
	
	// TODO connect to Redis at localhost, port 6379
	jedis = new Jedis("localhost", 6379);
	jedis.connect();
	
	// TODO Create a new BloomFilter object
	BloomFilter filter = new BloomFilter();
	
	// TODO call readFields with an FSDataInputStream from the file
	filter.readFields(fs.open(bloom));
	
	// TODO Open the testing file for read
	String line = null;
	int numBFhits = 0, numhits = 0, numlines = 0;
	BufferedReader rdr = new BufferedReader(new InputStreamReader(
			fs.open(input)));
	
	// TODO create a new Key to re-use
	Key key = new Key();

	long start = System.currentTimeMillis();
	while ((line = rdr.readLine()) != null) {
		// TODO increment numlines
		++numlines;
		
		// TODO set the bytes of the key to line's bytes with a weight of 1.0
		key.set(line.getBytes(), 1.0);
		
		// TODO membership test the key
		if (filter.membershipTest(key)) {
			// TODO increment numBFhits
			++numBFhits;

			// TODO test jedis using sismember
			if (jedis.sismember(REDIS_SET_KEY, line)) {
				// TODO increment numhits
				++numhits;
			}
		}
	}
	long finish = System.currentTimeMillis();

	// TODO close the file reader and Redis client
	rdr.close();
	jedis.disconnect();

	System.out.println("Took " + (finish - start) + " ms to check Redis "
			+ numlines + " times for " + numhits
			+ " successful tests.  Bloom filter hits: " + numBFhits
			+ " False postives: " + (numBFhits - numhits));
}
 

import org.apache.hadoop.util.bloom.BloomFilter; //导入方法依赖的package包/类
private void init() throws IOException {
    filter = new BloomFilter();
    String dcFile = "./" + getFilenameFromPath(bloomFile) +
        "/part-r-00000";
    filter.readFields(new DataInputStream(new FileInputStream(dcFile)));
}