1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import java.io.DataInput;
24 import java.io.IOException;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
31 import org.apache.hadoop.hbase.io.hfile.HFile;
32 import org.apache.hadoop.hbase.regionserver.StoreFile;
33 import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
34
35
36
37
38
39 public final class BloomFilterFactory {
40
41 private static final Log LOG =
42 LogFactory.getLog(BloomFilterFactory.class.getName());
43
44
45 private BloomFilterFactory() {}
46
47
48
49
50
51 public static final String IO_STOREFILE_BLOOM_ERROR_RATE =
52 "io.storefile.bloom.error.rate";
53
54
55
56
57
58 public static final String IO_STOREFILE_BLOOM_MAX_FOLD =
59 "io.storefile.bloom.max.fold";
60
61
62
63
64
65 public static final String IO_STOREFILE_BLOOM_MAX_KEYS =
66 "io.storefile.bloom.max.keys";
67
68
69 public static final String IO_STOREFILE_BLOOM_ENABLED =
70 "io.storefile.bloom.enabled";
71
72
73
74
75
76 public static final String IO_STOREFILE_BLOOM_BLOCK_SIZE =
77 "io.storefile.bloom.block.size";
78
79
80 private static final int MAX_ALLOWED_FOLD_FACTOR = 7;
81
82
83
84
85
86
87
88
89
90
91
92
93 public static BloomFilter
94 createFromMeta(DataInput meta, HFile.Reader reader)
95 throws IllegalArgumentException, IOException {
96 int version = meta.readInt();
97 switch (version) {
98 case ByteBloomFilter.VERSION:
99
100
101
102 return new ByteBloomFilter(meta);
103
104 case CompoundBloomFilterBase.VERSION:
105 return new CompoundBloomFilter(meta, reader);
106
107 default:
108 throw new IllegalArgumentException(
109 "Bad bloom filter format version " + version
110 );
111 }
112 }
113
114
115
116
117 public static boolean isBloomEnabled(Configuration conf) {
118 return conf.getBoolean(IO_STOREFILE_BLOOM_ENABLED, true);
119 }
120
121 public static float getErrorRate(Configuration conf) {
122 return conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
123 }
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138 public static BloomFilterWriter createBloomAtWrite(Configuration conf,
139 CacheConfig cacheConf, BloomType bloomType, int maxKeys,
140 HFile.Writer writer) {
141 if (!isBloomEnabled(conf)) {
142 if (LOG.isTraceEnabled()) {
143 LOG.trace("Bloom filters are disabled by configuration for "
144 + writer.getPath()
145 + (conf == null ? " (configuration is null)" : ""));
146 }
147 return null;
148 } else if (bloomType == BloomType.NONE) {
149 if (LOG.isTraceEnabled()) {
150 LOG.debug("Bloom filter is turned off for the column family");
151 }
152 return null;
153 }
154
155 float err = getErrorRate(conf);
156
157
158
159
160
161 if (bloomType == BloomType.ROWCOL) {
162 err = (float) (1 - Math.sqrt(1 - err));
163 }
164
165 int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD,
166 MAX_ALLOWED_FOLD_FACTOR);
167
168 if (HFile.getFormatVersion(conf) > HFile.MIN_FORMAT_VERSION) {
169
170 CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(
171 getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold,
172 cacheConf.shouldCacheBloomsOnWrite(), bloomType == BloomType.ROWCOL
173 ? KeyValue.KEY_COMPARATOR : Bytes.BYTES_RAWCOMPARATOR);
174 writer.addInlineBlockWriter(bloomWriter);
175 return bloomWriter;
176 } else {
177
178
179 int tooBig = conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS,
180 128 * 1000 * 1000);
181
182 if (maxKeys <= 0) {
183 LOG.warn("Invalid maximum number of keys specified: " + maxKeys
184 + ", not using Bloom filter");
185 return null;
186 } else if (maxKeys < tooBig) {
187 BloomFilterWriter bloom = new ByteBloomFilter((int) maxKeys, err,
188 Hash.getHashType(conf), maxFold);
189 bloom.allocBloom();
190 return bloom;
191 } else {
192 LOG.debug("Skipping bloom filter because max keysize too large: "
193 + maxKeys);
194 }
195 }
196 return null;
197 }
198
199
200 public static int getBloomBlockSize(Configuration conf) {
201 return conf.getInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, 128 * 1024);
202 }
203 };