View Javadoc

1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.concurrent.atomic.AtomicLong;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FSDataInputStream;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.KeyValue;
31  import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
32  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
33  import org.apache.hadoop.io.RawComparator;
34  
35  /**
36   * Common functionality needed by all versions of {@link HFile} readers.
37   */
38  public abstract class AbstractHFileReader implements HFile.Reader {
39  
40    private static final Log LOG = LogFactory.getLog(AbstractHFileReader.class);
41  
42    /** Filesystem-level block reader for this HFile format version. */
43    protected HFileBlock.FSReader fsBlockReader;
44  
45    /** Stream to read from. */
46    protected FSDataInputStream istream;
47  
48    /**
49     * True if we should close the input stream when done. We don't close it if we
50     * didn't open it.
51     */
52    protected final boolean closeIStream;
53  
54    /** Data block index reader keeping the root data index in memory */
55    protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
56  
57    /** Meta block index reader -- always single level */
58    protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
59  
60    protected final FixedFileTrailer trailer;
61  
62    /** Filled when we read in the trailer. */
63    protected final Compression.Algorithm compressAlgo;
64  
65    /** Last key in the file. Filled in when we read in the file info */
66    protected byte [] lastKey = null;
67  
68    /** Average key length read from file info */
69    protected int avgKeyLen = -1;
70  
71    /** Average value length read from file info */
72    protected int avgValueLen = -1;
73  
74    /** Key comparator */
75    protected RawComparator<byte []> comparator;
76  
77    /** Size of this file. */
78    protected final long fileSize;
79  
80    /** Block cache configuration. */
81    protected final CacheConfig cacheConf;
82  
83    protected AtomicLong cacheHits = new AtomicLong();
84    protected AtomicLong blockLoads = new AtomicLong();
85    protected AtomicLong metaLoads = new AtomicLong();
86  
87    /** Path of file */
88    protected final Path path;
89  
90    /** File name to be used for block names */
91    protected final String name;
92  
93    protected FileInfo fileInfo;
94  
95    /** Prefix of the form cf.<column_family_name> for statistics counters. */
96    private final String cfStatsPrefix;
97  
98    protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
99        final FSDataInputStream fsdis, final long fileSize,
100       final boolean closeIStream,
101       final CacheConfig cacheConf) {
102     this.trailer = trailer;
103     this.compressAlgo = trailer.getCompressionCodec();
104     this.cacheConf = cacheConf;
105     this.fileSize = fileSize;
106     this.istream = fsdis;
107     this.closeIStream = closeIStream;
108     this.path = path;
109     this.name = path.getName();
110     cfStatsPrefix = "cf." + parseCfNameFromPath(path.toString());
111   }
112 
113   @SuppressWarnings("serial")
114   public static class BlockIndexNotLoadedException
115       extends IllegalStateException {
116     public BlockIndexNotLoadedException() {
117       // Add a message in case anyone relies on it as opposed to class name.
118       super("Block index not loaded");
119     }
120   }
121 
122   protected String toStringFirstKey() {
123     return KeyValue.keyToString(getFirstKey());
124   }
125 
126   protected String toStringLastKey() {
127     return KeyValue.keyToString(getLastKey());
128   }
129 
130   /**
131    * Parse the HFile path to figure out which table and column family
132    * it belongs to. This is used to maintain read statistics on a
133    * per-column-family basis.
134    *
135    * @param path HFile path name
136    */
137   public static String parseCfNameFromPath(String path) {
138     String splits[] = path.split("/");
139     if (splits.length < 2) {
140       LOG.warn("Could not determine the table and column family of the " +
141           "HFile path " + path);
142       return "unknown";
143     }
144 
145     return splits[splits.length - 2];
146   }
147 
148   public abstract boolean isFileInfoLoaded();
149 
150   @Override
151   public String toString() {
152     return "reader=" + path.toString() +
153         (!isFileInfoLoaded()? "":
154           ", compression=" + compressAlgo.getName() +
155           ", cacheConf=" + cacheConf +
156           ", firstKey=" + toStringFirstKey() +
157           ", lastKey=" + toStringLastKey()) +
158           ", avgKeyLen=" + avgKeyLen +
159           ", avgValueLen=" + avgValueLen +
160           ", entries=" + trailer.getEntryCount() +
161           ", length=" + fileSize;
162   }
163 
164   @Override
165   public long length() {
166     return fileSize;
167   }
168 
169   /**
170    * Create a Scanner on this file. No seeks or reads are done on creation. Call
171    * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
172    * nothing to clean up in a Scanner. Letting go of your references to the
173    * scanner is sufficient. NOTE: Do not use this overload of getScanner for
174    * compactions.
175    *
176    * @param cacheBlocks True if we should cache blocks read in by this scanner.
177    * @param pread Use positional read rather than seek+read if true (pread is
178    *          better for random reads, seek+read is better scanning).
179    * @return Scanner on this file.
180    */
181   @Override
182   public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
183     return getScanner(cacheBlocks, pread, false);
184   }
185 
186   /**
187    * @return the first key in the file. May be null if file has no entries. Note
188    *         that this is not the first row key, but rather the byte form of the
189    *         first KeyValue.
190    */
191   @Override
192   public byte [] getFirstKey() {
193     if (dataBlockIndexReader == null) {
194       throw new BlockIndexNotLoadedException();
195     }
196     return dataBlockIndexReader.isEmpty() ? null
197         : dataBlockIndexReader.getRootBlockKey(0);
198   }
199 
200   /**
201    * TODO left from {@link HFile} version 1: move this to StoreFile after Ryan's
202    * patch goes in to eliminate {@link KeyValue} here.
203    *
204    * @return the first row key, or null if the file is empty.
205    */
206   @Override
207   public byte[] getFirstRowKey() {
208     byte[] firstKey = getFirstKey();
209     if (firstKey == null)
210       return null;
211     return KeyValue.createKeyValueFromKey(firstKey).getRow();
212   }
213 
214   /**
215    * TODO left from {@link HFile} version 1: move this to StoreFile after
216    * Ryan's patch goes in to eliminate {@link KeyValue} here.
217    *
218    * @return the last row key, or null if the file is empty.
219    */
220   @Override
221   public byte[] getLastRowKey() {
222     byte[] lastKey = getLastKey();
223     if (lastKey == null)
224       return null;
225     return KeyValue.createKeyValueFromKey(lastKey).getRow();
226   }
227 
228   /** @return number of KV entries in this HFile */
229   @Override
230   public long getEntries() {
231     return trailer.getEntryCount();
232   }
233 
234   /** @return comparator */
235   @Override
236   public RawComparator<byte []> getComparator() {
237     return comparator;
238   }
239 
240   /** @return compression algorithm */
241   @Override
242   public Compression.Algorithm getCompressionAlgorithm() {
243     return compressAlgo;
244   }
245 
246   /**
247    * @return the total heap size of data and meta block indexes in bytes. Does
248    *         not take into account non-root blocks of a multilevel data index.
249    */
250   public long indexSize() {
251     return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
252         + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
253             : 0);
254   }
255 
256   @Override
257   public String getName() {
258     return name;
259   }
260 
261   @Override
262   public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
263     return dataBlockIndexReader;
264   }
265 
266   @Override
267   public String getColumnFamilyName() {
268     return cfStatsPrefix;
269   }
270 
271   @Override
272   public FixedFileTrailer getTrailer() {
273     return trailer;
274   }
275 
276   @Override
277   public FileInfo loadFileInfo() throws IOException {
278     return fileInfo;
279   }
280 
281   /**
282    * An exception thrown when an operation requiring a scanner to be seeked
283    * is invoked on a scanner that is not seeked.
284    */
285   @SuppressWarnings("serial")
286   public static class NotSeekedException extends IllegalStateException {
287     public NotSeekedException() {
288       super("Not seeked to a key/value");
289     }
290   }
291 
292   protected static abstract class Scanner implements HFileScanner {
293     protected ByteBuffer blockBuffer;
294 
295     protected boolean cacheBlocks;
296     protected final boolean pread;
297     protected final boolean isCompaction;
298 
299     protected int currKeyLen;
300     protected int currValueLen;
301     protected int currMemstoreTSLen;
302     protected long currMemstoreTS;
303 
304     protected int blockFetches;
305 
306     public Scanner(final boolean cacheBlocks,
307         final boolean pread, final boolean isCompaction) {
308       this.cacheBlocks = cacheBlocks;
309       this.pread = pread;
310       this.isCompaction = isCompaction;
311     }
312 
313     @Override
314     public boolean isSeeked(){
315       return blockBuffer != null;
316     }
317 
318     @Override
319     public String toString() {
320       return "HFileScanner for reader " + String.valueOf(getReader());
321     }
322 
323     protected void assertSeeked() {
324       if (!isSeeked())
325         throw new NotSeekedException();
326     }
327   }
328 
329   /** For testing */
330   HFileBlock.FSReader getUncachedBlockReader() {
331     return fsBlockReader;
332   }
333 
334   public Path getPath() {
335     return path;
336   }
337 
338 }