1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import java.io.ByteArrayOutputStream;
24 import java.io.DataOutputStream;
25 import java.nio.ByteBuffer;
26
27 import junit.framework.TestCase;
28
29 public class TestByteBloomFilter extends TestCase {
30
31 public void testBasicBloom() throws Exception {
32 ByteBloomFilter bf1 = new ByteBloomFilter(1000, (float)0.01, Hash.MURMUR_HASH, 0);
33 ByteBloomFilter bf2 = new ByteBloomFilter(1000, (float)0.01, Hash.MURMUR_HASH, 0);
34 bf1.allocBloom();
35 bf2.allocBloom();
36
37
38 byte[] key1 = {1,2,3,4,5,6,7,8,9};
39 byte[] key2 = {1,2,3,4,5,6,7,8,7};
40
41 bf1.add(key1);
42 bf2.add(key2);
43
44 assertTrue(bf1.contains(key1));
45 assertFalse(bf1.contains(key2));
46 assertFalse(bf2.contains(key1));
47 assertTrue(bf2.contains(key2));
48
49 byte [] bkey = {1,2,3,4};
50 byte [] bval = "this is a much larger byte array".getBytes();
51
52 bf1.add(bkey);
53 bf1.add(bval, 1, bval.length-1);
54
55 assertTrue( bf1.contains(bkey) );
56 assertTrue( bf1.contains(bval, 1, bval.length-1) );
57 assertFalse( bf1.contains(bval) );
58 assertFalse( bf1.contains(bval) );
59
60
61
62 ByteArrayOutputStream bOut = new ByteArrayOutputStream();
63 bf1.writeBloom(new DataOutputStream(bOut));
64 ByteBuffer bb = ByteBuffer.wrap(bOut.toByteArray());
65 ByteBloomFilter newBf1 = new ByteBloomFilter(1000, (float)0.01,
66 Hash.MURMUR_HASH, 0);
67 assertTrue(newBf1.contains(key1, bb));
68 assertFalse(newBf1.contains(key2, bb));
69 assertTrue( newBf1.contains(bkey, bb) );
70 assertTrue( newBf1.contains(bval, 1, bval.length-1, bb) );
71 assertFalse( newBf1.contains(bval, bb) );
72 assertFalse( newBf1.contains(bval, bb) );
73
74 System.out.println("Serialized as " + bOut.size() + " bytes");
75 assertTrue(bOut.size() - bf1.byteSize < 10);
76 }
77
78 public void testBloomFold() throws Exception {
79
80 ByteBloomFilter b = new ByteBloomFilter(1003, (float) 0.01,
81 Hash.MURMUR_HASH, 2);
82 b.allocBloom();
83 long origSize = b.getByteSize();
84 assertEquals(1204, origSize);
85 for (int i = 0; i < 12; ++i) {
86 b.add(Bytes.toBytes(i));
87 }
88 b.compactBloom();
89 assertEquals(origSize>>2, b.getByteSize());
90 int falsePositives = 0;
91 for (int i = 0; i < 25; ++i) {
92 if (b.contains(Bytes.toBytes(i))) {
93 if(i >= 12) falsePositives++;
94 } else {
95 assertFalse(i < 12);
96 }
97 }
98 assertTrue(falsePositives <= 1);
99
100
101 }
102
103 public void testBloomPerf() throws Exception {
104
105 float err = (float)0.01;
106 ByteBloomFilter b = new ByteBloomFilter(10*1000*1000, (float)err, Hash.MURMUR_HASH, 3);
107 b.allocBloom();
108 long startTime = System.currentTimeMillis();
109 long origSize = b.getByteSize();
110 for (int i = 0; i < 1*1000*1000; ++i) {
111 b.add(Bytes.toBytes(i));
112 }
113 long endTime = System.currentTimeMillis();
114 System.out.println("Total Add time = " + (endTime - startTime) + "ms");
115
116
117 startTime = System.currentTimeMillis();
118 b.compactBloom();
119 endTime = System.currentTimeMillis();
120 System.out.println("Total Fold time = " + (endTime - startTime) + "ms");
121 assertTrue(origSize >= b.getByteSize()<<3);
122
123
124 startTime = System.currentTimeMillis();
125 int falsePositives = 0;
126 for (int i = 0; i < 2*1000*1000; ++i) {
127
128 if (b.contains(Bytes.toBytes(i))) {
129 if(i >= 1*1000*1000) falsePositives++;
130 } else {
131 assertFalse(i < 1*1000*1000);
132 }
133 }
134 endTime = System.currentTimeMillis();
135 System.out.println("Total Contains time = " + (endTime - startTime) + "ms");
136 System.out.println("False Positive = " + falsePositives);
137 assertTrue(falsePositives <= (1*1000*1000)*err);
138
139
140 }
141
142 public void testSizing() {
143 int bitSize = 8 * 128 * 1024;
144 double errorRate = 0.025;
145
146
147
148 long maxKeys = ByteBloomFilter.idealMaxKeys(bitSize, errorRate);
149 assertEquals(136570, maxKeys);
150
151
152
153 long bitSize2 = ByteBloomFilter.computeBitSize(maxKeys, errorRate);
154
155
156 assertTrue(Math.abs(bitSize2 - bitSize) * 1.0 / bitSize < 1e-5);
157 }
158
159 public void testFoldableByteSize() {
160 assertEquals(128, ByteBloomFilter.computeFoldableByteSize(1000, 5));
161 assertEquals(640, ByteBloomFilter.computeFoldableByteSize(5001, 4));
162 }
163
164 }