View Javadoc

1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.client;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.TreeSet;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.HServerAddress;
37  import org.apache.hadoop.hbase.ServerName;
38  import org.apache.hadoop.hbase.TableNotFoundException;
39  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
40  import org.apache.hadoop.hbase.util.Addressing;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.Writables;
43  
44  /**
45   * Scanner class that contains the <code>.META.</code> table scanning logic
46   * and uses a Retryable scanner. Provided visitors will be called
47   * for each row.
48   * 
49   * Although public visibility, this is not a public-facing API and may evolve in
50   * minor releases.
51   */
52  public class MetaScanner {
53    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
54    /**
55     * Scans the meta table and calls a visitor on each RowResult and uses a empty
56     * start row value as table name.
57     *
58     * @param configuration conf
59     * @param visitor A custom visitor
60     * @throws IOException e
61     */
62    public static void metaScan(Configuration configuration,
63        MetaScannerVisitor visitor)
64    throws IOException {
65      metaScan(configuration, visitor, null);
66    }
67  
68    /**
69     * Scans the meta table and calls a visitor on each RowResult. Uses a table
70     * name to locate meta regions.
71     *
72     * @param configuration config
73     * @param visitor visitor object
74     * @param userTableName User table name in meta table to start scan at.  Pass
75     * null if not interested in a particular table.
76     * @throws IOException e
77     */
78    public static void metaScan(Configuration configuration,
79        MetaScannerVisitor visitor, byte [] userTableName)
80    throws IOException {
81      metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE);
82    }
83  
84    /**
85     * Scans the meta table and calls a visitor on each RowResult. Uses a table
86     * name and a row name to locate meta regions. And it only scans at most
87     * <code>rowLimit</code> of rows.
88     *
89     * @param configuration HBase configuration.
90     * @param visitor Visitor object.
91     * @param userTableName User table name in meta table to start scan at.  Pass
92     * null if not interested in a particular table.
93     * @param row Name of the row at the user table. The scan will start from
94     * the region row where the row resides.
95     * @param rowLimit Max of processed rows. If it is less than 0, it
96     * will be set to default value <code>Integer.MAX_VALUE</code>.
97     * @throws IOException e
98     */
99    public static void metaScan(Configuration configuration,
100       MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
101       int rowLimit)
102   throws IOException {
103     metaScan(configuration, visitor, userTableName, row, rowLimit,
104       HConstants.META_TABLE_NAME);
105   }
106 
107   /**
108    * Scans the meta table and calls a visitor on each RowResult. Uses a table
109    * name and a row name to locate meta regions. And it only scans at most
110    * <code>rowLimit</code> of rows.
111    *
112    * @param configuration HBase configuration.
113    * @param visitor Visitor object.
114    * @param tableName User table name in meta table to start scan at.  Pass
115    * null if not interested in a particular table.
116    * @param row Name of the row at the user table. The scan will start from
117    * the region row where the row resides.
118    * @param rowLimit Max of processed rows. If it is less than 0, it
119    * will be set to default value <code>Integer.MAX_VALUE</code>.
120    * @param metaTableName Meta table to scan, root or meta.
121    * @throws IOException e
122    */
123   public static void metaScan(Configuration configuration,
124       final MetaScannerVisitor visitor, final byte[] tableName,
125       final byte[] row, final int rowLimit, final byte[] metaTableName)
126       throws IOException {
127     HConnectionManager.execute(new HConnectable<Void>(configuration) {
128       @Override
129       public Void connect(HConnection connection) throws IOException {
130         metaScan(conf, connection, visitor, tableName, row, rowLimit,
131             metaTableName);
132         return null;
133       }
134     });
135   }
136 
137   private static void metaScan(Configuration configuration, HConnection connection,
138       MetaScannerVisitor visitor, byte [] tableName, byte[] row,
139       int rowLimit, final byte [] metaTableName)
140   throws IOException {
141     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
142 
143     // if row is not null, we want to use the startKey of the row's region as
144     // the startRow for the meta scan.
145     byte[] startRow;
146     if (row != null) {
147       // Scan starting at a particular row in a particular table
148       assert tableName != null;
149       byte[] searchRow =
150         HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
151           false);
152 
153       HTable metaTable = new HTable(configuration, HConstants.META_TABLE_NAME);
154       Result startRowResult = metaTable.getRowOrBefore(searchRow,
155           HConstants.CATALOG_FAMILY);
156       if (startRowResult == null) {
157         throw new TableNotFoundException("Cannot find row in .META. for table: "
158             + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
159       }
160       byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY,
161           HConstants.REGIONINFO_QUALIFIER);
162       if (value == null || value.length == 0) {
163         throw new IOException("HRegionInfo was null or empty in Meta for " +
164           Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
165       }
166       HRegionInfo regionInfo = Writables.getHRegionInfo(value);
167 
168       byte[] rowBefore = regionInfo.getStartKey();
169       startRow = HRegionInfo.createRegionName(tableName, rowBefore,
170           HConstants.ZEROES, false);
171     } else if (tableName == null || tableName.length == 0) {
172       // Full META scan
173       startRow = HConstants.EMPTY_START_ROW;
174     } else {
175       // Scan META for an entire table
176       startRow = HRegionInfo.createRegionName(
177           tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
178     }
179 
180     // Scan over each meta region
181     ScannerCallable callable;
182     int rows = Math.min(rowLimit, configuration.getInt(
183         HConstants.HBASE_META_SCANNER_CACHING,
184         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
185     do {
186       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
187       if (LOG.isDebugEnabled()) {
188         LOG.debug("Scanning " + Bytes.toString(metaTableName) +
189           " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
190           rowUpperLimit + " rows using " + connection.toString());
191       }
192       callable = new ScannerCallable(connection, metaTableName, scan);
193       // Open scanner
194       connection.getRegionServerWithRetries(callable);
195 
196       int processedRows = 0;
197       try {
198         callable.setCaching(rows);
199         done: do {
200           if (processedRows >= rowUpperLimit) {
201             break;
202           }
203           //we have all the rows here
204           Result [] rrs = connection.getRegionServerWithRetries(callable);
205           if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
206             break; //exit completely
207           }
208           for (Result rr : rrs) {
209             if (processedRows >= rowUpperLimit) {
210               break done;
211             }
212             if (!visitor.processRow(rr))
213               break done; //exit completely
214             processedRows++;
215           }
216           //here, we didn't break anywhere. Check if we have more rows
217         } while(true);
218         // Advance the startRow to the end key of the current region
219         startRow = callable.getHRegionInfo().getEndKey();
220       } finally {
221         // Close scanner
222         callable.setClose();
223         connection.getRegionServerWithRetries(callable);
224       }
225     } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
226   }
227 
228   /**
229    * Lists all of the regions currently in META.
230    * @param conf
231    * @return List of all user-space regions.
232    * @throws IOException
233    */
234   public static List<HRegionInfo> listAllRegions(Configuration conf)
235   throws IOException {
236     return listAllRegions(conf, true);
237   }
238 
239   /**
240    * Lists all of the regions currently in META.
241    * @param conf
242    * @param offlined True if we are to include offlined regions, false and we'll
243    * leave out offlined regions from returned list.
244    * @return List of all user-space regions.
245    * @throws IOException
246    */
247   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
248   throws IOException {
249     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
250     MetaScannerVisitor visitor = new MetaScannerVisitor() {
251         @Override
252         public boolean processRow(Result result) throws IOException {
253           if (result == null || result.isEmpty()) {
254             return true;
255           }
256           byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY,
257             HConstants.REGIONINFO_QUALIFIER);
258           if (bytes == null) {
259             LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
260             return true;
261           }
262           HRegionInfo regionInfo = Writables.getHRegionInfo(bytes);
263           // If region offline AND we are not to include offlined regions, return.
264           if (regionInfo.isOffline() && !offlined) return true;
265           regions.add(regionInfo);
266           return true;
267         }
268     };
269     metaScan(conf, visitor);
270     return regions;
271   }
272 
273   /**
274    * Lists all of the table regions currently in META.
275    * @param conf
276    * @param offlined True if we are to include offlined regions, false and we'll
277    * leave out offlined regions from returned list.
278    * @return Map of all user-space regions to servers
279    * @throws IOException
280    */
281   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf, final byte [] tablename, final boolean offlined)
282   throws IOException {
283     final NavigableMap<HRegionInfo, ServerName> regions =
284       new TreeMap<HRegionInfo, ServerName>();
285     MetaScannerVisitor visitor = new MetaScannerVisitor() {
286       @Override
287       public boolean processRow(Result rowResult) throws IOException {
288         HRegionInfo info = Writables.getHRegionInfo(
289             rowResult.getValue(HConstants.CATALOG_FAMILY,
290                 HConstants.REGIONINFO_QUALIFIER));
291         if (!(Bytes.equals(info.getTableName(), tablename))) {
292           return false;
293         }
294         byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
295           HConstants.SERVER_QUALIFIER);
296         String hostAndPort = null;
297         if (value != null && value.length > 0) {
298           hostAndPort = Bytes.toString(value);
299         }
300         value = rowResult.getValue(HConstants.CATALOG_FAMILY,
301           HConstants.STARTCODE_QUALIFIER);
302         long startcode = -1L;
303         if (value != null && value.length > 0) startcode = Bytes.toLong(value);
304         if (!(info.isOffline() || info.isSplit())) {
305           ServerName sn = null;
306           if (hostAndPort != null && hostAndPort.length() > 0) {
307             sn = new ServerName(hostAndPort, startcode);
308           }
309           regions.put(new UnmodifyableHRegionInfo(info), sn);
310         }
311         return true;
312       }
313     };
314     metaScan(conf, visitor, tablename);
315     return regions;
316   }
317 
318   /**
319    * Visitor class called to process each row of the .META. table
320    */
321   public interface MetaScannerVisitor {
322     /**
323      * Visitor method that accepts a RowResult and the meta region location.
324      * Implementations can return false to stop the region's loop if it becomes
325      * unnecessary for some reason.
326      *
327      * @param rowResult result
328      * @return A boolean to know if it should continue to loop in the region
329      * @throws IOException e
330      */
331     public boolean processRow(Result rowResult) throws IOException;
332   }
333 }