1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.BufferedInputStream;
23 import java.io.ByteArrayInputStream;
24 import java.io.ByteArrayOutputStream;
25 import java.io.DataInputStream;
26 import java.io.DataOutput;
27 import java.io.DataOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.OutputStream;
31 import java.nio.ByteBuffer;
32 import org.apache.hadoop.fs.FSDataInputStream;
33 import org.apache.hadoop.fs.FSDataOutputStream;
34
35 import org.apache.hadoop.hbase.io.DoubleOutputStream;
36 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.apache.hadoop.hbase.util.ClassSize;
39 import org.apache.hadoop.hbase.util.CompoundBloomFilter;
40 import org.apache.hadoop.hbase.util.Writables;
41 import org.apache.hadoop.io.IOUtils;
42 import org.apache.hadoop.io.Writable;
43 import org.apache.hadoop.io.compress.Compressor;
44 import org.apache.hadoop.io.compress.Decompressor;
45
46 import com.google.common.base.Preconditions;
47
48 import static org.apache.hadoop.hbase.io.hfile.BlockType.MAGIC_LENGTH;
49 import static org.apache.hadoop.hbase.io.hfile.Compression.Algorithm.NONE;
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76 public class HFileBlock implements Cacheable {
77
78
79 public static final int HEADER_SIZE = MAGIC_LENGTH + 2 * Bytes.SIZEOF_INT
80 + Bytes.SIZEOF_LONG;
81
82
83 public static final byte[] DUMMY_HEADER = new byte[HEADER_SIZE];
84
85 public static final int BYTE_BUFFER_HEAP_SIZE = (int) ClassSize.estimateBase(
86 ByteBuffer.wrap(new byte[0], 0, 0).getClass(), false);
87
88 static final int EXTRA_SERIALIZATION_SPACE = Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;
89
90
91 private static final CacheableDeserializer<Cacheable> blockDeserializer =
92 new CacheableDeserializer<Cacheable>() {
93 public HFileBlock deserialize(ByteBuffer buf) throws IOException{
94 ByteBuffer newByteBuffer = ByteBuffer.allocate(buf.limit()
95 - HFileBlock.EXTRA_SERIALIZATION_SPACE);
96 buf.limit(buf.limit()
97 - HFileBlock.EXTRA_SERIALIZATION_SPACE).rewind();
98 newByteBuffer.put(buf);
99 HFileBlock ourBuffer = new HFileBlock(newByteBuffer);
100
101 buf.position(buf.limit());
102 buf.limit(buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE);
103 ourBuffer.offset = buf.getLong();
104 ourBuffer.nextBlockOnDiskSizeWithHeader = buf.getInt();
105 return ourBuffer;
106 }
107 };
108 private BlockType blockType;
109 private final int onDiskSizeWithoutHeader;
110 private final int uncompressedSizeWithoutHeader;
111 private final long prevBlockOffset;
112 private ByteBuffer buf;
113
114
115
116
117
118 private long offset = -1;
119
120
121
122
123
124
125 private int nextBlockOnDiskSizeWithHeader = -1;
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146 public HFileBlock(BlockType blockType, int onDiskSizeWithoutHeader,
147 int uncompressedSizeWithoutHeader, long prevBlockOffset, ByteBuffer buf,
148 boolean fillHeader, long offset) {
149 this.blockType = blockType;
150 this.onDiskSizeWithoutHeader = onDiskSizeWithoutHeader;
151 this.uncompressedSizeWithoutHeader = uncompressedSizeWithoutHeader;
152 this.prevBlockOffset = prevBlockOffset;
153 this.buf = buf;
154 if (fillHeader)
155 overwriteHeader();
156 this.offset = offset;
157 }
158
159
160
161
162
163
164
165 private HFileBlock(ByteBuffer b) throws IOException {
166 b.rewind();
167 blockType = BlockType.read(b);
168 onDiskSizeWithoutHeader = b.getInt();
169 uncompressedSizeWithoutHeader = b.getInt();
170 prevBlockOffset = b.getLong();
171 buf = b;
172 buf.rewind();
173 }
174
175 public BlockType getBlockType() {
176 return blockType;
177 }
178
179
180
181
182 public int getOnDiskSizeWithHeader() {
183 return onDiskSizeWithoutHeader + HEADER_SIZE;
184 }
185
186
187
188
189
190
191
192
193
194 public int getOnDiskSizeWithoutHeader() {
195 return onDiskSizeWithoutHeader;
196 }
197
198
199
200
201
202 public int getUncompressedSizeWithoutHeader() {
203 return uncompressedSizeWithoutHeader;
204 }
205
206
207
208
209
210 public long getPrevBlockOffset() {
211 return prevBlockOffset;
212 }
213
214
215
216
217
218 private void overwriteHeader() {
219 buf.rewind();
220 blockType.write(buf);
221 buf.putInt(onDiskSizeWithoutHeader);
222 buf.putInt(uncompressedSizeWithoutHeader);
223 buf.putLong(prevBlockOffset);
224 }
225
226
227
228
229
230
231
232
233 public ByteBuffer getBufferWithoutHeader() {
234 return ByteBuffer.wrap(buf.array(), buf.arrayOffset() + HEADER_SIZE,
235 buf.limit() - HEADER_SIZE).slice();
236 }
237
238
239
240
241
242
243
244
245
246 public ByteBuffer getBufferReadOnly() {
247 return buf;
248 }
249
250
251
252
253
254
255
256 public ByteBuffer getBufferWithHeader() {
257 ByteBuffer dupBuf = buf.duplicate();
258 dupBuf.rewind();
259 return dupBuf;
260 }
261
262
263
264
265
266 public void readInto(Writable w) throws IOException {
267 Preconditions.checkNotNull(w);
268
269 if (Writables.getWritable(buf.array(), buf.arrayOffset() + HEADER_SIZE,
270 buf.limit() - HEADER_SIZE, w) == null) {
271 throw new IOException("Failed to deserialize block " + this + " into a "
272 + w.getClass().getSimpleName());
273 }
274 }
275
276 private void sanityCheckAssertion(long valueFromBuf, long valueFromField,
277 String fieldName) throws IOException {
278 if (valueFromBuf != valueFromField) {
279 throw new AssertionError(fieldName + " in the buffer (" + valueFromBuf
280 + ") is different from that in the field (" + valueFromField + ")");
281 }
282 }
283
284
285
286
287
288
289
290 void sanityCheck() throws IOException {
291 buf.rewind();
292
293 {
294 BlockType blockTypeFromBuf = BlockType.read(buf);
295 if (blockTypeFromBuf != blockType) {
296 throw new IOException("Block type stored in the buffer: " +
297 blockTypeFromBuf + ", block type field: " + blockType);
298 }
299 }
300
301 sanityCheckAssertion(buf.getInt(), onDiskSizeWithoutHeader,
302 "onDiskSizeWithoutHeader");
303
304 sanityCheckAssertion(buf.getInt(), uncompressedSizeWithoutHeader,
305 "uncompressedSizeWithoutHeader");
306
307 sanityCheckAssertion(buf.getLong(), prevBlockOffset, "prevBlocKOffset");
308
309 int expectedBufLimit = uncompressedSizeWithoutHeader + HEADER_SIZE;
310 if (buf.limit() != expectedBufLimit) {
311 throw new AssertionError("Expected buffer limit " + expectedBufLimit
312 + ", got " + buf.limit());
313 }
314
315
316
317 if (buf.capacity() != uncompressedSizeWithoutHeader + HEADER_SIZE &&
318 buf.capacity() != uncompressedSizeWithoutHeader + 2 * HEADER_SIZE) {
319 throw new AssertionError("Invalid buffer capacity: " + buf.capacity() +
320 ", expected " + (uncompressedSizeWithoutHeader + HEADER_SIZE) +
321 " or " + (uncompressedSizeWithoutHeader + 2 * HEADER_SIZE));
322 }
323 }
324
325 @Override
326 public String toString() {
327 return "blockType="
328 + blockType
329 + ", onDiskSizeWithoutHeader="
330 + onDiskSizeWithoutHeader
331 + ", uncompressedSizeWithoutHeader="
332 + uncompressedSizeWithoutHeader
333 + ", prevBlockOffset="
334 + prevBlockOffset
335 + ", dataBeginsWith="
336 + Bytes.toStringBinary(buf.array(), buf.arrayOffset() + HEADER_SIZE,
337 Math.min(32, buf.limit() - buf.arrayOffset() - HEADER_SIZE))
338 + ", fileOffset=" + offset;
339 }
340
341 private void validateOnDiskSizeWithoutHeader(
342 int expectedOnDiskSizeWithoutHeader) throws IOException {
343 if (onDiskSizeWithoutHeader != expectedOnDiskSizeWithoutHeader) {
344 String blockInfoMsg =
345 "Block offset: " + offset + ", data starts with: "
346 + Bytes.toStringBinary(buf.array(), buf.arrayOffset(),
347 buf.arrayOffset() + Math.min(32, buf.limit()));
348 throw new IOException("On-disk size without header provided is "
349 + expectedOnDiskSizeWithoutHeader + ", but block "
350 + "header contains " + onDiskSizeWithoutHeader + ". " +
351 blockInfoMsg);
352 }
353 }
354
355
356
357
358
359
360
361
362 private void allocateBuffer(boolean extraBytes) {
363 int capacityNeeded = HEADER_SIZE + uncompressedSizeWithoutHeader +
364 (extraBytes ? HEADER_SIZE : 0);
365
366 ByteBuffer newBuf = ByteBuffer.allocate(capacityNeeded);
367
368
369 System.arraycopy(buf.array(), buf.arrayOffset(), newBuf.array(),
370 newBuf.arrayOffset(), HEADER_SIZE);
371
372 buf = newBuf;
373 buf.limit(HEADER_SIZE + uncompressedSizeWithoutHeader);
374 }
375
376
377 public void assumeUncompressed() throws IOException {
378 if (onDiskSizeWithoutHeader != uncompressedSizeWithoutHeader) {
379 throw new IOException("Using no compression but "
380 + "onDiskSizeWithoutHeader=" + onDiskSizeWithoutHeader + ", "
381 + "uncompressedSizeWithoutHeader=" + uncompressedSizeWithoutHeader);
382 }
383 }
384
385
386
387
388
389 public void expectType(BlockType expectedType) throws IOException {
390 if (blockType != expectedType) {
391 throw new IOException("Invalid block type: expected=" + expectedType
392 + ", actual=" + blockType);
393 }
394 }
395
396
397 public long getOffset() {
398 if (offset < 0) {
399 throw new IllegalStateException(
400 "HFile block offset not initialized properly");
401 }
402 return offset;
403 }
404
405
406
407
408 public DataInputStream getByteStream() {
409 return new DataInputStream(new ByteArrayInputStream(buf.array(),
410 buf.arrayOffset() + HEADER_SIZE, buf.limit() - HEADER_SIZE));
411 }
412
413 @Override
414 public long heapSize() {
415
416
417
418
419
420
421
422
423
424 if (buf != null) {
425 return ClassSize.align(ClassSize.OBJECT + 2 * ClassSize.REFERENCE + 3
426 * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_LONG + BYTE_BUFFER_HEAP_SIZE)
427 + ClassSize.align(buf.capacity());
428 } else {
429
430 return ClassSize.align(ClassSize.OBJECT + 2 * ClassSize.REFERENCE + 3
431 * Bytes.SIZEOF_INT + 2 * Bytes.SIZEOF_LONG + BYTE_BUFFER_HEAP_SIZE);
432 }
433 }
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450 public static boolean readWithExtra(InputStream in, byte buf[],
451 int bufOffset, int necessaryLen, int extraLen) throws IOException {
452 int bytesRemaining = necessaryLen + extraLen;
453 while (bytesRemaining > 0) {
454 int ret = in.read(buf, bufOffset, bytesRemaining);
455 if (ret == -1 && bytesRemaining <= extraLen) {
456
457 break;
458 }
459
460 if (ret < 0) {
461 throw new IOException("Premature EOF from inputStream (read "
462 + "returned " + ret + ", was trying to read " + necessaryLen
463 + " necessary bytes and " + extraLen + " extra bytes, "
464 + "successfully read "
465 + (necessaryLen + extraLen - bytesRemaining));
466 }
467 bufOffset += ret;
468 bytesRemaining -= ret;
469 }
470 return bytesRemaining <= 0;
471 }
472
473
474
475
476
477 public int getNextBlockOnDiskSizeWithHeader() {
478 return nextBlockOnDiskSizeWithHeader;
479 }
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498 public static class Writer {
499
500 private enum State {
501 INIT,
502 WRITING,
503 BLOCK_READY
504 };
505
506
507 private State state = State.INIT;
508
509
510 private final Compression.Algorithm compressAlgo;
511
512
513
514
515
516
517
518 private ByteArrayOutputStream baosOnDisk;
519
520
521
522
523
524 private ByteArrayOutputStream baosInMemory;
525
526
527 private Compressor compressor;
528
529
530 private BlockType blockType;
531
532
533
534
535
536 private DataOutputStream userDataStream;
537
538
539
540
541
542 private byte[] onDiskBytesWithHeader;
543
544
545
546
547
548 private int uncompressedSizeWithoutHeader;
549
550
551
552
553
554
555 private byte[] uncompressedBytesWithHeader;
556
557
558
559
560
561 private long startOffset;
562
563
564
565
566
567 private long[] prevOffsetByType;
568
569
570
571
572
573 private boolean cacheOnWrite;
574
575
576 private long prevOffset;
577
578
579
580
581
582 public Writer(Compression.Algorithm compressionAlgorithm) {
583 compressAlgo = compressionAlgorithm == null ? NONE
584 : compressionAlgorithm;
585
586 baosOnDisk = new ByteArrayOutputStream();
587 if (compressAlgo != NONE)
588 compressor = compressionAlgorithm.getCompressor();
589
590 prevOffsetByType = new long[BlockType.values().length];
591 for (int i = 0; i < prevOffsetByType.length; ++i)
592 prevOffsetByType[i] = -1;
593 }
594
595
596
597
598
599
600
601 public DataOutputStream startWriting(BlockType newBlockType,
602 boolean cacheOnWrite) throws IOException {
603 if (state == State.BLOCK_READY && startOffset != -1) {
604
605
606 prevOffsetByType[blockType.ordinal()] = startOffset;
607 }
608
609 this.cacheOnWrite = cacheOnWrite;
610
611 startOffset = -1;
612 blockType = newBlockType;
613
614 baosOnDisk.reset();
615 baosOnDisk.write(DUMMY_HEADER);
616
617 state = State.WRITING;
618 if (compressAlgo == NONE) {
619
620
621 userDataStream = new DataOutputStream(baosOnDisk);
622 } else {
623 OutputStream compressingOutputStream =
624 compressAlgo.createCompressionStream(baosOnDisk, compressor, 0);
625
626 if (cacheOnWrite) {
627
628 if (baosInMemory == null)
629 baosInMemory = new ByteArrayOutputStream();
630 baosInMemory.reset();
631 baosInMemory.write(DUMMY_HEADER);
632 userDataStream = new DataOutputStream(new DoubleOutputStream(
633 compressingOutputStream, baosInMemory));
634 } else {
635 userDataStream = new DataOutputStream(compressingOutputStream);
636 }
637 }
638
639 return userDataStream;
640 }
641
642
643
644
645
646
647
648
649 DataOutputStream getUserDataStream() {
650 expectState(State.WRITING);
651 return userDataStream;
652 }
653
654
655
656
657
658 private void ensureBlockReady() throws IOException {
659 Preconditions.checkState(state != State.INIT,
660 "Unexpected state: " + state);
661
662 if (state == State.BLOCK_READY)
663 return;
664
665 finishBlock();
666 state = State.BLOCK_READY;
667 }
668
669
670
671
672
673
674
675 private void finishBlock() throws IOException {
676 userDataStream.flush();
677 uncompressedSizeWithoutHeader = userDataStream.size();
678
679 onDiskBytesWithHeader = baosOnDisk.toByteArray();
680 prevOffset = prevOffsetByType[blockType.ordinal()];
681 putHeader(onDiskBytesWithHeader, 0);
682
683 if (cacheOnWrite && compressAlgo != NONE) {
684 uncompressedBytesWithHeader = baosInMemory.toByteArray();
685
686 if (uncompressedSizeWithoutHeader !=
687 uncompressedBytesWithHeader.length - HEADER_SIZE) {
688 throw new IOException("Uncompressed size mismatch: "
689 + uncompressedSizeWithoutHeader + " vs. "
690 + (uncompressedBytesWithHeader.length - HEADER_SIZE));
691 }
692
693
694 putHeader(uncompressedBytesWithHeader, 0);
695 }
696 }
697
698
699 private void putHeader(byte[] dest, int offset) {
700 offset = blockType.put(dest, offset);
701 offset = Bytes.putInt(dest, offset, onDiskBytesWithHeader.length
702 - HEADER_SIZE);
703 offset = Bytes.putInt(dest, offset, uncompressedSizeWithoutHeader);
704 Bytes.putLong(dest, offset, prevOffset);
705 }
706
707
708
709
710
711
712
713
714
715 public void writeHeaderAndData(FSDataOutputStream out) throws IOException {
716 long offset = out.getPos();
717 if (startOffset != -1 && offset != startOffset) {
718 throw new IOException("A " + blockType + " block written to a "
719 + "stream twice, first at offset " + startOffset + ", then at "
720 + offset);
721 }
722 startOffset = offset;
723
724 writeHeaderAndData((DataOutputStream) out);
725 }
726
727
728
729
730
731
732
733
734
735
736 private void writeHeaderAndData(DataOutputStream out) throws IOException {
737 ensureBlockReady();
738 out.write(onDiskBytesWithHeader);
739 }
740
741
742
743
744
745
746
747
748
749
750 public byte[] getHeaderAndData() throws IOException {
751 ensureBlockReady();
752 return onDiskBytesWithHeader;
753 }
754
755
756
757
758
759 public void releaseCompressor() {
760 if (compressor != null) {
761 compressAlgo.returnCompressor(compressor);
762 compressor = null;
763 }
764 }
765
766
767
768
769
770
771
772
773
774 public int getOnDiskSizeWithoutHeader() {
775 expectState(State.BLOCK_READY);
776 return onDiskBytesWithHeader.length - HEADER_SIZE;
777 }
778
779
780
781
782
783
784
785
786 public int getOnDiskSizeWithHeader() {
787 expectState(State.BLOCK_READY);
788 return onDiskBytesWithHeader.length;
789 }
790
791
792
793
794 public int getUncompressedSizeWithoutHeader() {
795 expectState(State.BLOCK_READY);
796 return uncompressedSizeWithoutHeader;
797 }
798
799
800
801
802 public int getUncompressedSizeWithHeader() {
803 expectState(State.BLOCK_READY);
804 return uncompressedSizeWithoutHeader + HEADER_SIZE;
805 }
806
807
808 public boolean isWriting() {
809 return state == State.WRITING;
810 }
811
812
813
814
815
816
817
818
819 public int blockSizeWritten() {
820 if (state != State.WRITING)
821 return 0;
822 return userDataStream.size();
823 }
824
825
826
827
828
829
830
831
832 private byte[] getUncompressedDataWithHeader() {
833 expectState(State.BLOCK_READY);
834
835 if (compressAlgo == NONE)
836 return onDiskBytesWithHeader;
837
838 if (!cacheOnWrite)
839 throw new IllegalStateException("Cache-on-write is turned off");
840
841 if (uncompressedBytesWithHeader == null)
842 throw new NullPointerException();
843
844 return uncompressedBytesWithHeader;
845 }
846
847 private void expectState(State expectedState) {
848 if (state != expectedState) {
849 throw new IllegalStateException("Expected state: " + expectedState +
850 ", actual state: " + state);
851 }
852 }
853
854
855
856
857
858
859
860 public ByteBuffer getUncompressedBufferWithHeader() {
861 byte[] b = getUncompressedDataWithHeader();
862 return ByteBuffer.wrap(b, 0, b.length);
863 }
864
865
866
867
868
869
870
871
872
873
874
875 public void writeBlock(BlockWritable bw, FSDataOutputStream out)
876 throws IOException {
877 bw.writeToBlock(startWriting(bw.getBlockType(), false));
878 writeHeaderAndData(out);
879 }
880
881 public HFileBlock getBlockForCaching() {
882 return new HFileBlock(blockType, onDiskBytesWithHeader.length
883 - HEADER_SIZE, uncompressedSizeWithoutHeader, prevOffset,
884 getUncompressedBufferWithHeader(), false, startOffset);
885 }
886
887 }
888
889
890 public interface BlockWritable {
891
892
893 BlockType getBlockType();
894
895
896
897
898
899
900
901 void writeToBlock(DataOutput out) throws IOException;
902 }
903
904
905
906
907 public interface BlockIterator {
908
909
910
911
912 HFileBlock nextBlock() throws IOException;
913
914
915
916
917
918
919 DataInputStream nextBlockAsStream(BlockType blockType) throws IOException;
920 }
921
922
923 public interface FSReader {
924
925
926
927
928
929
930
931
932
933
934
935
936 HFileBlock readBlockData(long offset, long onDiskSize,
937 int uncompressedSize, boolean pread) throws IOException;
938
939
940
941
942
943
944
945
946
947
948 BlockIterator blockRange(long startOffset, long endOffset);
949 }
950
951
952
953
954
955 public abstract static class AbstractFSReader implements FSReader {
956
957
958 protected FSDataInputStream istream;
959
960
961 protected Compression.Algorithm compressAlgo;
962
963
964 protected long fileSize;
965
966
967 public static final int DEFAULT_BUFFER_SIZE = 1 << 20;
968
969 public AbstractFSReader(FSDataInputStream istream, Algorithm compressAlgo,
970 long fileSize) {
971 this.istream = istream;
972 this.compressAlgo = compressAlgo;
973 this.fileSize = fileSize;
974 }
975
976 @Override
977 public BlockIterator blockRange(final long startOffset,
978 final long endOffset) {
979 return new BlockIterator() {
980 private long offset = startOffset;
981
982 @Override
983 public HFileBlock nextBlock() throws IOException {
984 if (offset >= endOffset)
985 return null;
986 HFileBlock b = readBlockData(offset, -1, -1, false);
987 offset += b.getOnDiskSizeWithHeader();
988 return b;
989 }
990
991 @Override
992 public DataInputStream nextBlockAsStream(BlockType blockType)
993 throws IOException {
994 HFileBlock blk = nextBlock();
995 if (blk.getBlockType() != blockType) {
996 throw new IOException("Expected block of type " + blockType
997 + " but found " + blk.getBlockType());
998 }
999 return blk.getByteStream();
1000 }
1001 };
1002 }
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018 protected int readAtOffset(byte[] dest, int destOffset, int size,
1019 boolean peekIntoNextBlock, long fileOffset, boolean pread)
1020 throws IOException {
1021 if (peekIntoNextBlock &&
1022 destOffset + size + HEADER_SIZE > dest.length) {
1023
1024
1025 throw new IOException("Attempted to read " + size + " bytes and " +
1026 HEADER_SIZE + " bytes of next header into a " + dest.length +
1027 "-byte array at offset " + destOffset);
1028 }
1029
1030 if (pread) {
1031
1032 int extraSize = peekIntoNextBlock ? HEADER_SIZE : 0;
1033
1034 int ret = istream.read(fileOffset, dest, destOffset, size + extraSize);
1035 if (ret < size) {
1036 throw new IOException("Positional read of " + size + " bytes " +
1037 "failed at offset " + fileOffset + " (returned " + ret + ")");
1038 }
1039
1040 if (ret == size || ret < size + extraSize) {
1041
1042 return -1;
1043 }
1044 } else {
1045
1046 synchronized (istream) {
1047 istream.seek(fileOffset);
1048
1049 long realOffset = istream.getPos();
1050 if (realOffset != fileOffset) {
1051 throw new IOException("Tried to seek to " + fileOffset + " to "
1052 + "read " + size + " bytes, but pos=" + realOffset
1053 + " after seek");
1054 }
1055
1056 if (!peekIntoNextBlock) {
1057 IOUtils.readFully(istream, dest, destOffset, size);
1058 return -1;
1059 }
1060
1061
1062 if (!readWithExtra(istream, dest, destOffset, size, HEADER_SIZE))
1063 return -1;
1064 }
1065 }
1066
1067 assert peekIntoNextBlock;
1068 return Bytes.toInt(dest, destOffset + size + BlockType.MAGIC_LENGTH) +
1069 HEADER_SIZE;
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086 protected void decompress(byte[] dest, int destOffset,
1087 InputStream bufferedBoundedStream, int compressedSize,
1088 int uncompressedSize) throws IOException {
1089 Decompressor decompressor = null;
1090 try {
1091 decompressor = compressAlgo.getDecompressor();
1092 InputStream is = compressAlgo.createDecompressionStream(
1093 bufferedBoundedStream, decompressor, 0);
1094
1095 IOUtils.readFully(is, dest, destOffset, uncompressedSize);
1096 is.close();
1097 } finally {
1098 if (decompressor != null) {
1099 compressAlgo.returnDecompressor(decompressor);
1100 }
1101 }
1102 }
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114 protected InputStream createBufferedBoundedStream(long offset,
1115 int size, boolean pread) {
1116 return new BufferedInputStream(new BoundedRangeFileInputStream(istream,
1117 offset, size, pread), Math.min(DEFAULT_BUFFER_SIZE, size));
1118 }
1119
1120 }
1121
1122
1123
1124
1125
1126
1127
1128
1129 public static class FSReaderV1 extends AbstractFSReader {
1130
1131
1132 private static final int HEADER_DELTA = HEADER_SIZE - MAGIC_LENGTH;
1133
1134 public FSReaderV1(FSDataInputStream istream, Algorithm compressAlgo,
1135 long fileSize) {
1136 super(istream, compressAlgo, fileSize);
1137 }
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157 @Override
1158 public HFileBlock readBlockData(long offset, long onDiskSizeWithMagic,
1159 int uncompressedSizeWithMagic, boolean pread) throws IOException {
1160 if (uncompressedSizeWithMagic <= 0) {
1161 throw new IOException("Invalid uncompressedSize="
1162 + uncompressedSizeWithMagic + " for a version 1 block");
1163 }
1164
1165 if (onDiskSizeWithMagic <= 0 || onDiskSizeWithMagic >= Integer.MAX_VALUE)
1166 {
1167 throw new IOException("Invalid onDiskSize=" + onDiskSizeWithMagic
1168 + " (maximum allowed: " + Integer.MAX_VALUE + ")");
1169 }
1170
1171 int onDiskSize = (int) onDiskSizeWithMagic;
1172
1173 if (uncompressedSizeWithMagic < MAGIC_LENGTH) {
1174 throw new IOException("Uncompressed size for a version 1 block is "
1175 + uncompressedSizeWithMagic + " but must be at least "
1176 + MAGIC_LENGTH);
1177 }
1178
1179
1180
1181 ByteBuffer buf = ByteBuffer.allocate(uncompressedSizeWithMagic
1182 + HEADER_DELTA);
1183
1184 int onDiskSizeWithoutHeader;
1185 if (compressAlgo == Compression.Algorithm.NONE) {
1186
1187 if (onDiskSize != uncompressedSizeWithMagic) {
1188 throw new IOException("onDiskSize=" + onDiskSize
1189 + " and uncompressedSize=" + uncompressedSizeWithMagic
1190 + " must be equal for version 1 with no compression");
1191 }
1192
1193
1194
1195 readAtOffset(buf.array(), buf.arrayOffset() + HEADER_DELTA,
1196 onDiskSize, false, offset, pread);
1197
1198 onDiskSizeWithoutHeader = uncompressedSizeWithMagic - MAGIC_LENGTH;
1199 } else {
1200 InputStream bufferedBoundedStream = createBufferedBoundedStream(
1201 offset, onDiskSize, pread);
1202 decompress(buf.array(), buf.arrayOffset() + HEADER_DELTA,
1203 bufferedBoundedStream, onDiskSize, uncompressedSizeWithMagic);
1204
1205
1206
1207 onDiskSizeWithoutHeader = onDiskSize;
1208 }
1209
1210 BlockType newBlockType = BlockType.parse(buf.array(), buf.arrayOffset()
1211 + HEADER_DELTA, MAGIC_LENGTH);
1212
1213
1214
1215
1216 HFileBlock b = new HFileBlock(newBlockType, onDiskSizeWithoutHeader,
1217 uncompressedSizeWithMagic - MAGIC_LENGTH, -1L, buf, true, offset);
1218 return b;
1219 }
1220 }
1221
1222
1223
1224
1225
1226 private static class PrefetchedHeader {
1227 long offset = -1;
1228 byte[] header = new byte[HEADER_SIZE];
1229 ByteBuffer buf = ByteBuffer.wrap(header, 0, HEADER_SIZE);
1230 }
1231
1232
1233 public static class FSReaderV2 extends AbstractFSReader {
1234
1235 private ThreadLocal<PrefetchedHeader> prefetchedHeaderForThread =
1236 new ThreadLocal<PrefetchedHeader>() {
1237 @Override
1238 public PrefetchedHeader initialValue() {
1239 return new PrefetchedHeader();
1240 }
1241 };
1242
1243 public FSReaderV2(FSDataInputStream istream, Algorithm compressAlgo,
1244 long fileSize) {
1245 super(istream, compressAlgo, fileSize);
1246 }
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259 @Override
1260 public HFileBlock readBlockData(long offset, long onDiskSizeWithHeaderL,
1261 int uncompressedSize, boolean pread) throws IOException {
1262 if (offset < 0) {
1263 throw new IOException("Invalid offset=" + offset + " trying to read "
1264 + "block (onDiskSize=" + onDiskSizeWithHeaderL
1265 + ", uncompressedSize=" + uncompressedSize + ")");
1266 }
1267 if (uncompressedSize != -1) {
1268 throw new IOException("Version 2 block reader API does not need " +
1269 "the uncompressed size parameter");
1270 }
1271
1272 if ((onDiskSizeWithHeaderL < HEADER_SIZE && onDiskSizeWithHeaderL != -1)
1273 || onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
1274 throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
1275 + ": expected to be at least " + HEADER_SIZE
1276 + " and at most " + Integer.MAX_VALUE + ", or -1 (offset="
1277 + offset + ", uncompressedSize=" + uncompressedSize + ")");
1278 }
1279
1280 int onDiskSizeWithHeader = (int) onDiskSizeWithHeaderL;
1281
1282 HFileBlock b;
1283 if (onDiskSizeWithHeader > 0) {
1284
1285
1286
1287
1288
1289
1290
1291
1292 int onDiskSizeWithoutHeader = onDiskSizeWithHeader - HEADER_SIZE;
1293 assert onDiskSizeWithoutHeader >= 0;
1294
1295
1296
1297
1298 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1299 byte[] header = prefetchedHeader.offset == offset
1300 ? prefetchedHeader.header : null;
1301
1302
1303 int preReadHeaderSize = header == null ? 0 : HEADER_SIZE;
1304
1305 if (compressAlgo == Compression.Algorithm.NONE) {
1306
1307
1308
1309 ByteBuffer headerAndData = ByteBuffer.allocate(onDiskSizeWithHeader
1310 + HEADER_SIZE);
1311 headerAndData.limit(onDiskSizeWithHeader);
1312
1313 if (header != null) {
1314 System.arraycopy(header, 0, headerAndData.array(), 0,
1315 HEADER_SIZE);
1316 }
1317
1318 int nextBlockOnDiskSizeWithHeader = readAtOffset(
1319 headerAndData.array(), headerAndData.arrayOffset()
1320 + preReadHeaderSize, onDiskSizeWithHeader
1321 - preReadHeaderSize, true, offset + preReadHeaderSize,
1322 pread);
1323
1324 b = new HFileBlock(headerAndData);
1325 b.assumeUncompressed();
1326 b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
1327 b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSizeWithHeader;
1328
1329 if (b.nextBlockOnDiskSizeWithHeader > 0)
1330 setNextBlockHeader(offset, b);
1331 } else {
1332
1333 byte[] onDiskBlock = new byte[onDiskSizeWithHeader + HEADER_SIZE];
1334
1335 int nextBlockOnDiskSize = readAtOffset(onDiskBlock,
1336 preReadHeaderSize, onDiskSizeWithHeader - preReadHeaderSize,
1337 true, offset + preReadHeaderSize, pread);
1338
1339 if (header == null)
1340 header = onDiskBlock;
1341
1342 try {
1343 b = new HFileBlock(ByteBuffer.wrap(header, 0, HEADER_SIZE));
1344 } catch (IOException ex) {
1345
1346 throw new IOException("Failed to read compressed block at "
1347 + offset + ", onDiskSizeWithoutHeader=" + onDiskSizeWithHeader
1348 + ", preReadHeaderSize=" + preReadHeaderSize
1349 + ", header.length=" + header.length + ", header bytes: "
1350 + Bytes.toStringBinary(header, 0, HEADER_SIZE), ex);
1351 }
1352 b.validateOnDiskSizeWithoutHeader(onDiskSizeWithoutHeader);
1353 b.nextBlockOnDiskSizeWithHeader = nextBlockOnDiskSize;
1354
1355 DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
1356 onDiskBlock, HEADER_SIZE, onDiskSizeWithoutHeader));
1357
1358
1359 b.allocateBuffer(b.nextBlockOnDiskSizeWithHeader > 0);
1360
1361 decompress(b.buf.array(), b.buf.arrayOffset() + HEADER_SIZE, dis,
1362 onDiskSizeWithoutHeader, b.uncompressedSizeWithoutHeader);
1363
1364
1365 if (nextBlockOnDiskSize > 0) {
1366 System.arraycopy(onDiskBlock, onDiskSizeWithHeader, b.buf.array(),
1367 b.buf.arrayOffset() + HEADER_SIZE
1368 + b.uncompressedSizeWithoutHeader, HEADER_SIZE);
1369
1370 setNextBlockHeader(offset, b);
1371 }
1372 }
1373
1374 } else {
1375
1376
1377
1378
1379
1380
1381
1382
1383 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1384 ByteBuffer headerBuf = prefetchedHeader.offset == offset ?
1385 prefetchedHeader.buf : null;
1386
1387 if (headerBuf == null) {
1388
1389
1390 headerBuf = ByteBuffer.allocate(HEADER_SIZE);;
1391 readAtOffset(headerBuf.array(), headerBuf.arrayOffset(), HEADER_SIZE,
1392 false, offset, pread);
1393 }
1394
1395 b = new HFileBlock(headerBuf);
1396
1397
1398 b.allocateBuffer(true);
1399
1400 if (compressAlgo == Compression.Algorithm.NONE) {
1401
1402
1403
1404 b.assumeUncompressed();
1405 b.nextBlockOnDiskSizeWithHeader = readAtOffset(b.buf.array(),
1406 b.buf.arrayOffset() + HEADER_SIZE,
1407 b.uncompressedSizeWithoutHeader, true, offset + HEADER_SIZE,
1408 pread);
1409
1410 if (b.nextBlockOnDiskSizeWithHeader > 0) {
1411 setNextBlockHeader(offset, b);
1412 }
1413 } else {
1414
1415 byte[] compressedBytes = new byte[b.getOnDiskSizeWithHeader()
1416 + HEADER_SIZE];
1417
1418 b.nextBlockOnDiskSizeWithHeader = readAtOffset(compressedBytes,
1419 HEADER_SIZE, b.onDiskSizeWithoutHeader, true, offset
1420 + HEADER_SIZE, pread);
1421 DataInputStream dis = new DataInputStream(new ByteArrayInputStream(
1422 compressedBytes, HEADER_SIZE, b.onDiskSizeWithoutHeader));
1423
1424 decompress(b.buf.array(), b.buf.arrayOffset() + HEADER_SIZE, dis,
1425 b.onDiskSizeWithoutHeader, b.uncompressedSizeWithoutHeader);
1426
1427 if (b.nextBlockOnDiskSizeWithHeader > 0) {
1428
1429 int nextHeaderOffset = b.buf.arrayOffset() + HEADER_SIZE
1430 + b.uncompressedSizeWithoutHeader;
1431 System.arraycopy(compressedBytes,
1432 compressedBytes.length - HEADER_SIZE,
1433 b.buf.array(),
1434 nextHeaderOffset,
1435 HEADER_SIZE);
1436
1437 setNextBlockHeader(offset, b);
1438 }
1439 }
1440 }
1441
1442 b.offset = offset;
1443 return b;
1444 }
1445
1446 private void setNextBlockHeader(long offset, HFileBlock b) {
1447 PrefetchedHeader prefetchedHeader = prefetchedHeaderForThread.get();
1448 prefetchedHeader.offset = offset + b.getOnDiskSizeWithHeader();
1449 int nextHeaderOffset = b.buf.arrayOffset() + HEADER_SIZE
1450 + b.uncompressedSizeWithoutHeader;
1451 System.arraycopy(b.buf.array(), nextHeaderOffset,
1452 prefetchedHeader.header, 0, HEADER_SIZE);
1453 }
1454
1455 }
1456
1457 @Override
1458 public int getSerializedLength() {
1459 if (buf != null) {
1460 return this.buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE;
1461 }
1462 return 0;
1463 }
1464
1465 @Override
1466 public void serialize(ByteBuffer destination) {
1467 destination.put(this.buf.duplicate());
1468 destination.putLong(this.offset);
1469 destination.putInt(this.nextBlockOnDiskSizeWithHeader);
1470 destination.rewind();
1471 }
1472
1473 @Override
1474 public CacheableDeserializer<Cacheable> getDeserializer() {
1475 return HFileBlock.blockDeserializer;
1476 }
1477
1478 @Override
1479 public boolean equals(Object comparison) {
1480 if (this == comparison) {
1481 return true;
1482 }
1483 if (comparison == null) {
1484 return false;
1485 }
1486 if (comparison.getClass() != this.getClass()) {
1487 return false;
1488 }
1489
1490 HFileBlock castedComparison = (HFileBlock) comparison;
1491
1492 if (castedComparison.blockType != this.blockType) {
1493 return false;
1494 }
1495 if (castedComparison.nextBlockOnDiskSizeWithHeader != this.nextBlockOnDiskSizeWithHeader) {
1496 return false;
1497 }
1498 if (castedComparison.offset != this.offset) {
1499 return false;
1500 }
1501 if (castedComparison.onDiskSizeWithoutHeader != this.onDiskSizeWithoutHeader) {
1502 return false;
1503 }
1504 if (castedComparison.prevBlockOffset != this.prevBlockOffset) {
1505 return false;
1506 }
1507 if (castedComparison.uncompressedSizeWithoutHeader != this.uncompressedSizeWithoutHeader) {
1508 return false;
1509 }
1510 if (this.buf.compareTo(castedComparison.buf) != 0) {
1511 return false;
1512 }
1513 if (this.buf.position() != castedComparison.buf.position()){
1514 return false;
1515 }
1516 if (this.buf.limit() != castedComparison.buf.limit()){
1517 return false;
1518 }
1519 return true;
1520 }
1521
1522
1523 }