1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.Closeable;
23 import java.io.DataInput;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.ArrayList;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.atomic.AtomicLong;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FSDataOutputStream;
36 import org.apache.hadoop.fs.FileStatus;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.fs.PathFilter;
40 import org.apache.hadoop.hbase.KeyValue;
41 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
42 import org.apache.hadoop.hbase.io.HbaseMapWritable;
43 import org.apache.hadoop.hbase.util.BloomFilterWriter;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSUtils;
46 import org.apache.hadoop.io.RawComparator;
47 import org.apache.hadoop.io.Writable;
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112 public class HFile {
113 static final Log LOG = LogFactory.getLog(HFile.class);
114
115
116
117
118 public final static int MAXIMUM_KEY_LENGTH = Integer.MAX_VALUE;
119
120
121
122
123 public final static int DEFAULT_BLOCKSIZE = 64 * 1024;
124
125
126
127
128 public final static Compression.Algorithm DEFAULT_COMPRESSION_ALGORITHM =
129 Compression.Algorithm.NONE;
130
131
132 public static final int MIN_FORMAT_VERSION = 1;
133
134
135 public static final int MAX_FORMAT_VERSION = 2;
136
137
138 public final static String DEFAULT_COMPRESSION =
139 DEFAULT_COMPRESSION_ALGORITHM.getName();
140
141
142 static final char CACHE_KEY_SEPARATOR = '_';
143
144
145 static volatile AtomicLong readOps = new AtomicLong();
146 static volatile AtomicLong readTimeNano = new AtomicLong();
147 static volatile AtomicLong writeOps = new AtomicLong();
148 static volatile AtomicLong writeTimeNano = new AtomicLong();
149
150 public static final long getReadOps() {
151 return readOps.getAndSet(0);
152 }
153
154 public static final long getReadTimeMs() {
155 return readTimeNano.getAndSet(0) / 1000000;
156 }
157
158 public static final long getWriteOps() {
159 return writeOps.getAndSet(0);
160 }
161
162 public static final long getWriteTimeMs() {
163 return writeTimeNano.getAndSet(0) / 1000000;
164 }
165
166
167 public interface Writer extends Closeable {
168
169
170 void appendFileInfo(byte[] key, byte[] value) throws IOException;
171
172 void append(KeyValue kv) throws IOException;
173
174 void append(byte[] key, byte[] value) throws IOException;
175
176
177 Path getPath();
178
179 void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
180
181
182
183
184
185 void addInlineBlockWriter(InlineBlockWriter bloomWriter);
186
187
188
189
190
191
192 void addBloomFilter(BloomFilterWriter bfw);
193 }
194
195
196
197
198
199 public static abstract class WriterFactory {
200 protected Configuration conf;
201 protected CacheConfig cacheConf;
202
203 WriterFactory(Configuration conf, CacheConfig cacheConf) {
204 this.conf = conf;
205 this.cacheConf = cacheConf;
206 }
207
208 public abstract Writer createWriter(FileSystem fs, Path path)
209 throws IOException;
210
211 public abstract Writer createWriter(FileSystem fs, Path path,
212 int blockSize, Compression.Algorithm compress,
213 final KeyComparator comparator) throws IOException;
214
215 public abstract Writer createWriter(FileSystem fs, Path path,
216 int blockSize, String compress,
217 final KeyComparator comparator) throws IOException;
218
219 public abstract Writer createWriter(final FSDataOutputStream ostream,
220 final int blockSize, final String compress,
221 final KeyComparator comparator) throws IOException;
222
223 public abstract Writer createWriter(final FSDataOutputStream ostream,
224 final int blockSize, final Compression.Algorithm compress,
225 final KeyComparator c) throws IOException;
226 }
227
228
229 public static final String FORMAT_VERSION_KEY = "hfile.format.version";
230
231 public static int getFormatVersion(Configuration conf) {
232 int version = conf.getInt(FORMAT_VERSION_KEY, MAX_FORMAT_VERSION);
233 checkFormatVersion(version);
234 return version;
235 }
236
237
238
239
240 public static final WriterFactory getWriterFactory(Configuration conf) {
241 return HFile.getWriterFactory(conf, new CacheConfig(conf));
242 }
243
244
245
246
247 public static final WriterFactory getWriterFactory(Configuration conf,
248 CacheConfig cacheConf) {
249 int version = getFormatVersion(conf);
250 switch (version) {
251 case 1:
252 return new HFileWriterV1.WriterFactoryV1(conf, cacheConf);
253 case 2:
254 return new HFileWriterV2.WriterFactoryV2(conf, cacheConf);
255 default:
256 throw new IllegalArgumentException("Cannot create writer for HFile " +
257 "format version " + version);
258 }
259 }
260
261
262 public interface CachingBlockReader {
263 HFileBlock readBlock(long offset, long onDiskBlockSize,
264 boolean cacheBlock, final boolean pread, final boolean isCompaction)
265 throws IOException;
266 }
267
268
269 public interface Reader extends Closeable, CachingBlockReader {
270
271
272
273
274
275 String getName();
276
277 String getColumnFamilyName();
278
279 RawComparator<byte []> getComparator();
280
281 HFileScanner getScanner(boolean cacheBlocks,
282 final boolean pread, final boolean isCompaction);
283
284 ByteBuffer getMetaBlock(String metaBlockName,
285 boolean cacheBlock) throws IOException;
286
287 Map<byte[], byte[]> loadFileInfo() throws IOException;
288
289 byte[] getLastKey();
290
291 byte[] midkey() throws IOException;
292
293 long length();
294
295 long getEntries();
296
297 byte[] getFirstKey();
298
299 long indexSize();
300
301 byte[] getFirstRowKey();
302
303 byte[] getLastRowKey();
304
305 FixedFileTrailer getTrailer();
306
307 HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
308
309 HFileScanner getScanner(boolean cacheBlocks, boolean pread);
310
311 Compression.Algorithm getCompressionAlgorithm();
312
313
314
315
316
317 DataInput getBloomFilterMetadata() throws IOException;
318
319 Path getPath();
320
321
322 void close(boolean evictOnClose) throws IOException;
323 }
324
325 private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,
326 long size, boolean closeIStream, CacheConfig cacheConf)
327 throws IOException {
328 FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, size);
329 switch (trailer.getVersion()) {
330 case 1:
331 return new HFileReaderV1(path, trailer, fsdis, size, closeIStream,
332 cacheConf);
333 case 2:
334 return new HFileReaderV2(path, trailer, fsdis, size, closeIStream,
335 cacheConf);
336 default:
337 throw new IOException("Cannot instantiate reader for HFile version " +
338 trailer.getVersion());
339 }
340 }
341
342 public static Reader createReader(FileSystem fs, Path path,
343 CacheConfig cacheConf) throws IOException {
344 return pickReaderVersion(path, fs.open(path),
345 fs.getFileStatus(path).getLen(), true, cacheConf);
346 }
347
348 public static Reader createReader(Path path, FSDataInputStream fsdis,
349 long size, CacheConfig cacheConf)
350 throws IOException {
351 return pickReaderVersion(path, fsdis, size, false, cacheConf);
352 }
353
354
355
356
357 static class FileInfo extends HbaseMapWritable<byte [], byte []> {
358 static final String RESERVED_PREFIX = "hfile.";
359 static final byte[] RESERVED_PREFIX_BYTES = Bytes.toBytes(RESERVED_PREFIX);
360 static final byte [] LASTKEY = Bytes.toBytes(RESERVED_PREFIX + "LASTKEY");
361 static final byte [] AVG_KEY_LEN =
362 Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
363 static final byte [] AVG_VALUE_LEN =
364 Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
365 static final byte [] COMPARATOR =
366 Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
367
368
369
370
371
372
373
374
375
376
377
378
379 public FileInfo append(final byte[] k, final byte[] v,
380 final boolean checkPrefix) throws IOException {
381 if (k == null || v == null) {
382 throw new NullPointerException("Key nor value may be null");
383 }
384 if (checkPrefix && isReservedFileInfoKey(k)) {
385 throw new IOException("Keys with a " + FileInfo.RESERVED_PREFIX
386 + " are reserved");
387 }
388 put(k, v);
389 return this;
390 }
391
392 }
393
394
395 public static boolean isReservedFileInfoKey(byte[] key) {
396 return Bytes.startsWith(key, FileInfo.RESERVED_PREFIX_BYTES);
397 }
398
399
400
401
402
403
404
405
406
407
408
409
410
411 public static String[] getSupportedCompressionAlgorithms() {
412 return Compression.getSupportedAlgorithms();
413 }
414
415
416
417
418
419
420 static int longToInt(final long l) {
421
422
423 return (int)(l & 0x00000000ffffffffL);
424 }
425
426
427
428
429
430
431
432
433
434
435 static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
436 throws IOException {
437 List<Path> res = new ArrayList<Path>();
438 PathFilter dirFilter = new FSUtils.DirFilter(fs);
439 FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
440 for(FileStatus dir : familyDirs) {
441 FileStatus[] files = fs.listStatus(dir.getPath());
442 for (FileStatus file : files) {
443 if (!file.isDir()) {
444 res.add(file.getPath());
445 }
446 }
447 }
448 return res;
449 }
450
451 public static void main(String[] args) throws IOException {
452 HFilePrettyPrinter prettyPrinter = new HFilePrettyPrinter();
453 System.exit(prettyPrinter.run(args));
454 }
455
456 public static String getBlockCacheKey(String hfileName, long offset) {
457 return hfileName + CACHE_KEY_SEPARATOR + offset;
458 }
459
460
461
462
463
464
465
466
467
468
469 public static void checkFormatVersion(int version)
470 throws IllegalArgumentException {
471 if (version < MIN_FORMAT_VERSION || version > MAX_FORMAT_VERSION) {
472 throw new IllegalArgumentException("Invalid HFile version: " + version
473 + " (expected to be " + "between " + MIN_FORMAT_VERSION + " and "
474 + MAX_FORMAT_VERSION + ")");
475 }
476 }
477
478 }