1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.io.hfile;
19
20 import java.lang.management.ManagementFactory;
21 import java.lang.management.MemoryUsage;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.HColumnDescriptor;
27 import org.apache.hadoop.hbase.regionserver.StoreFile;
28 import org.apache.hadoop.hbase.util.DirectMemoryUtils;
29 import org.apache.hadoop.util.StringUtils;
30
31
32
33
34 public class CacheConfig {
35 private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
36
37
38
39
40 public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
41 "hfile.block.cache.size";
42
43
44
45
46
47 public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48 "hbase.rs.cacheblocksonwrite";
49
50
51
52
53
54 public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55 "hfile.block.index.cacheonwrite";
56
57
58
59
60 public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61 "hfile.block.bloom.cacheonwrite";
62
63
64
65
66
67 public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
68 "hbase.rs.blockcache.cachedatacompressed";
69
70
71
72
73
74 public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
75 "hbase.rs.evictblocksonclose";
76
77
78
79 public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
80 public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
81 public static final boolean DEFAULT_IN_MEMORY = false;
82 public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
83 public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
84 public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
85 public static final boolean DEFAULT_COMPRESSED_CACHE = false;
86
87
88 private final BlockCache blockCache;
89
90
91
92
93
94 private boolean cacheDataOnRead;
95
96
97 private final boolean inMemory;
98
99
100 private boolean cacheDataOnWrite;
101
102
103 private final boolean cacheIndexesOnWrite;
104
105
106 private final boolean cacheBloomsOnWrite;
107
108
109 private boolean evictOnClose;
110
111
112 private final boolean cacheCompressed;
113
114
115
116
117
118
119
120 public CacheConfig(Configuration conf, HColumnDescriptor family) {
121 this(CacheConfig.instantiateBlockCache(conf),
122 family.isBlockCacheEnabled(), family.isInMemory(),
123 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
124 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
125 DEFAULT_CACHE_INDEXES_ON_WRITE),
126 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
127 DEFAULT_CACHE_BLOOMS_ON_WRITE),
128 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
129 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE)
130 );
131 }
132
133
134
135
136
137
138 public CacheConfig(Configuration conf) {
139 this(CacheConfig.instantiateBlockCache(conf),
140 DEFAULT_CACHE_DATA_ON_READ,
141 DEFAULT_IN_MEMORY,
142
143 conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
144 conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
145 DEFAULT_CACHE_INDEXES_ON_WRITE),
146 conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
147 DEFAULT_CACHE_BLOOMS_ON_WRITE),
148 conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
149 conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY,
150 DEFAULT_COMPRESSED_CACHE)
151 );
152 }
153
154
155
156
157
158
159
160
161
162
163
164
165
166 CacheConfig(final BlockCache blockCache,
167 final boolean cacheDataOnRead, final boolean inMemory,
168 final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
169 final boolean cacheBloomsOnWrite, final boolean evictOnClose,
170 final boolean cacheCompressed) {
171 this.blockCache = blockCache;
172 this.cacheDataOnRead = cacheDataOnRead;
173 this.inMemory = inMemory;
174 this.cacheDataOnWrite = cacheDataOnWrite;
175 this.cacheIndexesOnWrite = cacheIndexesOnWrite;
176 this.cacheBloomsOnWrite = cacheBloomsOnWrite;
177 this.evictOnClose = evictOnClose;
178 this.cacheCompressed = cacheCompressed;
179 }
180
181
182
183
184
185 public CacheConfig(CacheConfig cacheConf) {
186 this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
187 cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
188 cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
189 cacheConf.cacheCompressed);
190 }
191
192
193
194
195 public boolean isBlockCacheEnabled() {
196 return this.blockCache != null;
197 }
198
199
200
201
202
203 public BlockCache getBlockCache() {
204 return this.blockCache;
205 }
206
207
208
209
210
211 public boolean shouldCacheDataOnRead() {
212 return isBlockCacheEnabled() && cacheDataOnRead;
213 }
214
215
216
217
218 public boolean isInMemory() {
219 return isBlockCacheEnabled() && this.inMemory;
220 }
221
222
223
224
225
226 public boolean shouldCacheDataOnWrite() {
227 return isBlockCacheEnabled() && this.cacheDataOnWrite;
228 }
229
230
231
232
233
234
235 public void forTestsOnly_setCacheDataOnWrite(boolean cacheDataOnWrite) {
236 this.cacheDataOnWrite = cacheDataOnWrite;
237 }
238
239
240
241
242
243 public boolean shouldCacheIndexesOnWrite() {
244 return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
245 }
246
247
248
249
250
251 public boolean shouldCacheBloomsOnWrite() {
252 return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
253 }
254
255
256
257
258
259 public boolean shouldEvictOnClose() {
260 return isBlockCacheEnabled() && this.evictOnClose;
261 }
262
263
264
265
266
267
268 public void forTestsOnly_setEvictOnClose(boolean evictOnClose) {
269 this.evictOnClose = evictOnClose;
270 }
271
272
273
274
275 public boolean shouldCacheCompressed() {
276 return isBlockCacheEnabled() && this.cacheCompressed;
277 }
278
279 @Override
280 public String toString() {
281 if (!isBlockCacheEnabled()) {
282 return "CacheConfig:disabled";
283 }
284 return "CacheConfig:enabled " +
285 "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
286 "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
287 "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
288 "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
289 "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
290 "[cacheCompressed=" + shouldCacheCompressed() + "]";
291 }
292
293
294
295
296
297
298
299 private static BlockCache globalBlockCache;
300
301
302 private static boolean blockCacheDisabled = false;
303
304
305
306
307
308
309
310 private static synchronized BlockCache instantiateBlockCache(
311 Configuration conf) {
312 if (globalBlockCache != null) return globalBlockCache;
313 if (blockCacheDisabled) return null;
314
315 float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f);
316 if (cachePercentage == 0L) {
317 blockCacheDisabled = true;
318 return null;
319 }
320 if (cachePercentage > 1.0) {
321 throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY +
322 " must be between 0.0 and 1.0, not > 1.0");
323 }
324
325
326 MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
327 long cacheSize = (long)(mu.getMax() * cachePercentage);
328 int blockSize = conf.getInt("hbase.offheapcache.minblocksize",
329 HFile.DEFAULT_BLOCKSIZE);
330 long offHeapCacheSize =
331 (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0.95) *
332 DirectMemoryUtils.getDirectMemorySize());
333 LOG.info("Allocating LruBlockCache with maximum size " +
334 StringUtils.humanReadableInt(cacheSize));
335 if (offHeapCacheSize <= 0) {
336 globalBlockCache = new LruBlockCache(cacheSize,
337 StoreFile.DEFAULT_BLOCKSIZE_SMALL);
338 } else {
339 globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
340 StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
341 }
342 return globalBlockCache;
343 }
344 }