1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import java.io.DataInput;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26
27 import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer;
28 import org.apache.hadoop.hbase.io.hfile.HFile;
29 import org.apache.hadoop.hbase.io.hfile.HFileBlock;
30 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex;
31 import org.apache.hadoop.io.RawComparator;
32
33
34
35
36
37
38
39
40 public class CompoundBloomFilter extends CompoundBloomFilterBase
41 implements BloomFilter {
42
43
44 private HFile.Reader reader;
45
46 private HFileBlockIndex.BlockIndexReader index;
47
48 private int hashCount;
49 private Hash hash;
50
51 private long[] numQueriesPerChunk;
52 private long[] numPositivesPerChunk;
53
54
55
56
57
58
59
60
61 public CompoundBloomFilter(DataInput meta, HFile.Reader reader)
62 throws IOException {
63 this.reader = reader;
64
65 totalByteSize = meta.readLong();
66 hashCount = meta.readInt();
67 hashType = meta.readInt();
68 totalKeyCount = meta.readLong();
69 totalMaxKeys = meta.readLong();
70 numChunks = meta.readInt();
71 comparator = FixedFileTrailer.createComparator(
72 Bytes.toString(Bytes.readByteArray(meta)));
73
74 hash = Hash.getInstance(hashType);
75 if (hash == null) {
76 throw new IllegalArgumentException("Invalid hash type: " + hashType);
77 }
78
79 index = new HFileBlockIndex.BlockIndexReader(comparator, 1);
80 index.readRootIndex(meta, numChunks);
81 }
82
83 @Override
84 public boolean contains(byte[] key, int keyOffset, int keyLength,
85 ByteBuffer bloom) {
86
87
88 boolean result;
89
90 int block = index.rootBlockContainingKey(key, keyOffset, keyLength);
91 if (block < 0) {
92 result = false;
93 } else {
94 HFileBlock bloomBlock;
95 try {
96
97 bloomBlock = reader.readBlock(index.getRootBlockOffset(block),
98 index.getRootBlockDataSize(block), true, true, false);
99 } catch (IOException ex) {
100
101 throw new IllegalArgumentException(
102 "Failed to load Bloom block for key "
103 + Bytes.toStringBinary(key, keyOffset, keyLength), ex);
104 }
105
106 ByteBuffer bloomBuf = bloomBlock.getBufferReadOnly();
107 result = ByteBloomFilter.contains(key, keyOffset, keyLength,
108 bloomBuf.array(), bloomBuf.arrayOffset() + HFileBlock.HEADER_SIZE,
109 bloomBlock.getUncompressedSizeWithoutHeader(), hash, hashCount);
110 }
111
112 if (numQueriesPerChunk != null && block >= 0) {
113
114 ++numQueriesPerChunk[block];
115 if (result)
116 ++numPositivesPerChunk[block];
117 }
118
119 return result;
120 }
121
122 public boolean supportsAutoLoading() {
123 return true;
124 }
125
126 public int getNumChunks() {
127 return numChunks;
128 }
129
130 @Override
131 public RawComparator<byte[]> getComparator() {
132 return comparator;
133 }
134
135 public void enableTestingStats() {
136 numQueriesPerChunk = new long[numChunks];
137 numPositivesPerChunk = new long[numChunks];
138 }
139
140 public String formatTestingStats() {
141 StringBuilder sb = new StringBuilder();
142 for (int i = 0; i < numChunks; ++i) {
143 sb.append("chunk #");
144 sb.append(i);
145 sb.append(": queries=");
146 sb.append(numQueriesPerChunk[i]);
147 sb.append(", positives=");
148 sb.append(numPositivesPerChunk[i]);
149 sb.append(", positiveRatio=");
150 sb.append(numPositivesPerChunk[i] * 1.0 / numQueriesPerChunk[i]);
151 sb.append(";\n");
152 }
153 return sb.toString();
154 }
155
156 public long getNumQueriesForTesting(int chunk) {
157 return numQueriesPerChunk[chunk];
158 }
159
160 public long getNumPositivesForTesting(int chunk) {
161 return numPositivesPerChunk[chunk];
162 }
163
164 @Override
165 public String toString() {
166 StringBuilder sb = new StringBuilder();
167 sb.append(ByteBloomFilter.formatStats(this));
168 sb.append(ByteBloomFilter.STATS_RECORD_SEP +
169 "Number of chunks: " + numChunks);
170 sb.append(ByteBloomFilter.STATS_RECORD_SEP +
171 "Comparator: " + comparator.getClass().getSimpleName());
172 return sb.toString();
173 }
174
175 }