1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.io.hfile;
22
23 import java.io.DataOutput;
24 import java.io.DataOutputStream;
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.List;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FSDataOutputStream;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.KeyValue;
36 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
37 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
38 import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable;
39 import org.apache.hadoop.hbase.util.BloomFilterWriter;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.io.Writable;
42 import org.apache.hadoop.io.WritableUtils;
43
44
45
46
47 public class HFileWriterV2 extends AbstractHFileWriter {
48 static final Log LOG = LogFactory.getLog(HFileWriterV2.class);
49
50
51 public static final byte [] MAX_MEMSTORE_TS_KEY = Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
52
53 public static final byte [] KEY_VALUE_VERSION = Bytes.toBytes("KEY_VALUE_VERSION");
54
55 public static final int KEY_VALUE_VER_WITH_MEMSTORE = 1;
56
57
58 private List<InlineBlockWriter> inlineBlockWriters =
59 new ArrayList<InlineBlockWriter>();
60
61
62 private HFileBlock.Writer fsBlockWriter;
63
64 private HFileBlockIndex.BlockIndexWriter dataBlockIndexWriter;
65 private HFileBlockIndex.BlockIndexWriter metaBlockIndexWriter;
66
67
68 private long firstDataBlockOffset = -1;
69
70
71 private long lastDataBlockOffset;
72
73
74 private List<BlockWritable> additionalLoadOnOpenData =
75 new ArrayList<BlockWritable>();
76
77 private final boolean includeMemstoreTS = true;
78 private long maxMemstoreTS = 0;
79
80 static class WriterFactoryV2 extends HFile.WriterFactory {
81
82 WriterFactoryV2(Configuration conf, CacheConfig cacheConf) {
83 super(conf, cacheConf);
84 }
85
86 @Override
87 public Writer createWriter(FileSystem fs, Path path)
88 throws IOException {
89 return new HFileWriterV2(conf, cacheConf, fs, path);
90 }
91
92 @Override
93 public Writer createWriter(FileSystem fs, Path path, int blockSize,
94 Compression.Algorithm compress,
95 final KeyComparator comparator) throws IOException {
96 return new HFileWriterV2(conf, cacheConf, fs, path, blockSize,
97 compress, comparator);
98 }
99
100 @Override
101 public Writer createWriter(FileSystem fs, Path path, int blockSize,
102 String compress, final KeyComparator comparator)
103 throws IOException {
104 return new HFileWriterV2(conf, cacheConf, fs, path, blockSize,
105 compress, comparator);
106 }
107
108 @Override
109 public Writer createWriter(final FSDataOutputStream ostream,
110 final int blockSize, final String compress,
111 final KeyComparator comparator) throws IOException {
112 return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress,
113 comparator);
114 }
115
116 @Override
117 public Writer createWriter(final FSDataOutputStream ostream,
118 final int blockSize, final Compression.Algorithm compress,
119 final KeyComparator c) throws IOException {
120 return new HFileWriterV2(conf, cacheConf, ostream, blockSize, compress,
121 c);
122 }
123 }
124
125
126 public HFileWriterV2(Configuration conf, CacheConfig cacheConf,
127 FileSystem fs, Path path)
128 throws IOException {
129 this(conf, cacheConf, fs, path, HFile.DEFAULT_BLOCKSIZE,
130 HFile.DEFAULT_COMPRESSION_ALGORITHM, null);
131 }
132
133
134
135
136
137 public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs,
138 Path path, int blockSize, String compressAlgoName,
139 final KeyComparator comparator) throws IOException {
140 this(conf, cacheConf, fs, path, blockSize,
141 compressionByName(compressAlgoName), comparator);
142 }
143
144
145 public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs,
146 Path path, int blockSize, Compression.Algorithm compressAlgo,
147 final KeyComparator comparator) throws IOException {
148 super(cacheConf, createOutputStream(conf, fs, path), path,
149 blockSize, compressAlgo, comparator);
150 finishInit(conf);
151 }
152
153
154 public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf,
155 final FSDataOutputStream outputStream, final int blockSize,
156 final String compressAlgoName, final KeyComparator comparator)
157 throws IOException {
158 this(conf, cacheConf, outputStream, blockSize,
159 Compression.getCompressionAlgorithmByName(compressAlgoName),
160 comparator);
161 }
162
163
164 public HFileWriterV2(final Configuration conf, final CacheConfig cacheConf,
165 final FSDataOutputStream outputStream, final int blockSize,
166 final Compression.Algorithm compress, final KeyComparator comparator)
167 throws IOException {
168 super(cacheConf, outputStream, null, blockSize, compress, comparator);
169 finishInit(conf);
170 }
171
172
173 private void finishInit(final Configuration conf) {
174 if (fsBlockWriter != null)
175 throw new IllegalStateException("finishInit called twice");
176
177
178 fsBlockWriter = new HFileBlock.Writer(compressAlgo);
179
180
181 boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite();
182 dataBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter(fsBlockWriter,
183 cacheIndexesOnWrite ? cacheConf.getBlockCache(): null,
184 cacheIndexesOnWrite ? name : null);
185 dataBlockIndexWriter.setMaxChunkSize(
186 HFileBlockIndex.getMaxChunkSize(conf));
187 inlineBlockWriters.add(dataBlockIndexWriter);
188
189
190 metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
191
192 LOG.debug("Initialized with " + cacheConf);
193 }
194
195
196
197
198
199
200 private void checkBlockBoundary() throws IOException {
201 if (fsBlockWriter.blockSizeWritten() < blockSize)
202 return;
203
204 finishBlock();
205 writeInlineBlocks(false);
206 newBlock();
207 }
208
209
210 private void finishBlock() throws IOException {
211 if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0)
212 return;
213
214 long startTimeNs = System.nanoTime();
215
216
217 if (firstDataBlockOffset == -1)
218 firstDataBlockOffset = outputStream.getPos();
219
220
221 lastDataBlockOffset = outputStream.getPos();
222
223 fsBlockWriter.writeHeaderAndData(outputStream);
224
225 int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
226 dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset,
227 onDiskSize);
228 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
229
230 HFile.writeTimeNano.addAndGet(System.nanoTime() - startTimeNs);
231 HFile.writeOps.incrementAndGet();
232
233 if (cacheConf.shouldCacheDataOnWrite()) {
234 cacheConf.getBlockCache().cacheBlock(
235 HFile.getBlockCacheKey(name, lastDataBlockOffset),
236 fsBlockWriter.getBlockForCaching());
237 }
238 }
239
240
241 private void writeInlineBlocks(boolean closing) throws IOException {
242 for (InlineBlockWriter ibw : inlineBlockWriters) {
243 while (ibw.shouldWriteBlock(closing)) {
244 long offset = outputStream.getPos();
245 boolean cacheThisBlock = ibw.cacheOnWrite();
246 ibw.writeInlineBlock(fsBlockWriter.startWriting(
247 ibw.getInlineBlockType(), cacheThisBlock));
248 fsBlockWriter.writeHeaderAndData(outputStream);
249 ibw.blockWritten(offset, fsBlockWriter.getOnDiskSizeWithHeader(),
250 fsBlockWriter.getUncompressedSizeWithoutHeader());
251 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
252
253 if (cacheThisBlock) {
254
255 cacheConf.getBlockCache().cacheBlock(
256 HFile.getBlockCacheKey(name, offset),
257 fsBlockWriter.getBlockForCaching());
258 }
259 }
260 }
261 }
262
263
264
265
266
267
268 private void newBlock() throws IOException {
269
270 fsBlockWriter.startWriting(BlockType.DATA,
271 cacheConf.shouldCacheDataOnWrite());
272 firstKeyInBlock = null;
273 }
274
275
276
277
278
279
280
281
282
283
284
285
286 @Override
287 public void appendMetaBlock(String metaBlockName, Writable content) {
288 byte[] key = Bytes.toBytes(metaBlockName);
289 int i;
290 for (i = 0; i < metaNames.size(); ++i) {
291
292 byte[] cur = metaNames.get(i);
293 if (Bytes.BYTES_RAWCOMPARATOR.compare(cur, 0, cur.length, key, 0,
294 key.length) > 0) {
295 break;
296 }
297 }
298 metaNames.add(i, key);
299 metaData.add(i, content);
300 }
301
302
303
304
305
306
307
308
309
310 @Override
311 public void append(final KeyValue kv) throws IOException {
312 append(kv.getMemstoreTS(), kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength(),
313 kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
314 this.maxMemstoreTS = Math.max(this.maxMemstoreTS, kv.getMemstoreTS());
315 }
316
317
318
319
320
321
322
323
324
325
326
327 @Override
328 public void append(final byte[] key, final byte[] value) throws IOException {
329 append(0, key, 0, key.length, value, 0, value.length);
330 }
331
332
333
334
335
336
337
338
339
340
341
342
343
344 private void append(final long memstoreTS, final byte[] key, final int koffset, final int klength,
345 final byte[] value, final int voffset, final int vlength)
346 throws IOException {
347 boolean dupKey = checkKey(key, koffset, klength);
348 checkValue(value, voffset, vlength);
349 if (!dupKey) {
350 checkBlockBoundary();
351 }
352
353 if (!fsBlockWriter.isWriting())
354 newBlock();
355
356
357
358 {
359 DataOutputStream out = fsBlockWriter.getUserDataStream();
360 out.writeInt(klength);
361 totalKeyLength += klength;
362 out.writeInt(vlength);
363 totalValueLength += vlength;
364 out.write(key, koffset, klength);
365 out.write(value, voffset, vlength);
366 if (this.includeMemstoreTS) {
367 WritableUtils.writeVLong(out, memstoreTS);
368 }
369 }
370
371
372 if (firstKeyInBlock == null) {
373
374 firstKeyInBlock = new byte[klength];
375 System.arraycopy(key, koffset, firstKeyInBlock, 0, klength);
376 }
377
378 lastKeyBuffer = key;
379 lastKeyOffset = koffset;
380 lastKeyLength = klength;
381 entryCount++;
382 }
383
384 @Override
385 public void close() throws IOException {
386 if (outputStream == null) {
387 return;
388 }
389
390
391
392 finishBlock();
393 writeInlineBlocks(true);
394
395 FixedFileTrailer trailer = new FixedFileTrailer(2);
396
397
398 if (!metaNames.isEmpty()) {
399 for (int i = 0; i < metaNames.size(); ++i) {
400
401 long offset = outputStream.getPos();
402
403 DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META,
404 cacheConf.shouldCacheDataOnWrite());
405 metaData.get(i).write(dos);
406
407 fsBlockWriter.writeHeaderAndData(outputStream);
408 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
409
410
411 metaBlockIndexWriter.addEntry(metaNames.get(i), offset,
412 fsBlockWriter.getOnDiskSizeWithHeader());
413 }
414 }
415
416
417
418
419
420
421
422
423
424
425 long rootIndexOffset = dataBlockIndexWriter.writeIndexBlocks(outputStream);
426 trailer.setLoadOnOpenOffset(rootIndexOffset);
427
428
429 metaBlockIndexWriter.writeSingleLevelIndex(fsBlockWriter.startWriting(
430 BlockType.ROOT_INDEX, false), "meta");
431 fsBlockWriter.writeHeaderAndData(outputStream);
432 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
433
434 if (this.includeMemstoreTS) {
435 appendFileInfo(MAX_MEMSTORE_TS_KEY, Bytes.toBytes(maxMemstoreTS));
436 appendFileInfo(KEY_VALUE_VERSION, Bytes.toBytes(KEY_VALUE_VER_WITH_MEMSTORE));
437 }
438
439
440 writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO,
441 false));
442 fsBlockWriter.writeHeaderAndData(outputStream);
443 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
444
445
446 for (BlockWritable w : additionalLoadOnOpenData){
447 fsBlockWriter.writeBlock(w, outputStream);
448 totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
449 }
450
451
452 trailer.setNumDataIndexLevels(dataBlockIndexWriter.getNumLevels());
453 trailer.setUncompressedDataIndexSize(
454 dataBlockIndexWriter.getTotalUncompressedSize());
455 trailer.setFirstDataBlockOffset(firstDataBlockOffset);
456 trailer.setLastDataBlockOffset(lastDataBlockOffset);
457 trailer.setComparatorClass(comparator.getClass());
458 trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
459
460
461 finishClose(trailer);
462
463 fsBlockWriter.releaseCompressor();
464 }
465
466 @Override
467 public void addInlineBlockWriter(InlineBlockWriter ibw) {
468 inlineBlockWriters.add(ibw);
469 }
470
471 @Override
472 public void addBloomFilter(final BloomFilterWriter bfw) {
473 if (bfw.getKeyCount() <= 0)
474 return;
475
476 additionalLoadOnOpenData.add(new BlockWritable() {
477 @Override
478 public BlockType getBlockType() {
479 return BlockType.BLOOM_META;
480 }
481
482 @Override
483 public void writeToBlock(DataOutput out) throws IOException {
484 bfw.getMetaWriter().write(out);
485 Writable dataWriter = bfw.getDataWriter();
486 if (dataWriter != null)
487 dataWriter.write(out);
488 }
489 });
490 }
491
492 }