1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.Arrays;
23 import java.util.Comparator;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.ConcurrentHashMap;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30
31 import org.apache.commons.lang.NotImplementedException;
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.TableDescriptors;
44 import org.apache.hadoop.hbase.TableExistsException;
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64 public class FSTableDescriptors implements TableDescriptors {
65 private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
66 private final FileSystem fs;
67 private final Path rootdir;
68 private final boolean fsreadonly;
69 long cachehits = 0;
70 long invocations = 0;
71
72
73 public static final String TABLEINFO_NAME = ".tableinfo";
74
75
76
77
78 private final Map<String, TableDescriptorModtime> cache =
79 new ConcurrentHashMap<String, TableDescriptorModtime>();
80
81
82
83
84 static class TableDescriptorModtime {
85 private final HTableDescriptor descriptor;
86 private final long modtime;
87
88 TableDescriptorModtime(final long modtime, final HTableDescriptor htd) {
89 this.descriptor = htd;
90 this.modtime = modtime;
91 }
92
93 long getModtime() {
94 return this.modtime;
95 }
96
97 HTableDescriptor getTableDescriptor() {
98 return this.descriptor;
99 }
100 }
101
102 public FSTableDescriptors(final FileSystem fs, final Path rootdir) {
103 this(fs, rootdir, false);
104 }
105
106
107
108
109
110
111
112 public FSTableDescriptors(final FileSystem fs, final Path rootdir,
113 final boolean fsreadOnly) {
114 super();
115 this.fs = fs;
116 this.rootdir = rootdir;
117 this.fsreadonly = fsreadOnly;
118 }
119
120
121
122
123 @Override
124 public HTableDescriptor get(final byte [] tablename)
125 throws TableExistsException, FileNotFoundException, IOException {
126 return get(Bytes.toString(tablename));
127 }
128
129
130
131
132 @Override
133 public HTableDescriptor get(final String tablename)
134 throws TableExistsException, FileNotFoundException, IOException {
135 invocations++;
136 if (HTableDescriptor.ROOT_TABLEDESC.getNameAsString().equals(tablename)) {
137 cachehits++;
138 return HTableDescriptor.ROOT_TABLEDESC;
139 }
140 if (HTableDescriptor.META_TABLEDESC.getNameAsString().equals(tablename)) {
141 cachehits++;
142 return HTableDescriptor.META_TABLEDESC;
143 }
144
145
146 if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(tablename)) {
147 throw new IOException("No descriptor found for table = " + tablename);
148 }
149
150
151 TableDescriptorModtime tdm = this.cache.get(tablename);
152
153
154 long modtime = getTableInfoModtime(this.fs, this.rootdir, tablename);
155 if (tdm != null) {
156 if (modtime <= tdm.getModtime()) {
157 cachehits++;
158 return tdm.getTableDescriptor();
159 }
160 }
161 HTableDescriptor htd = getTableDescriptor(this.fs, this.rootdir, tablename);
162 if (htd == null) {
163
164 throw new TableExistsException("No descriptor for " + tablename);
165 }
166 this.cache.put(tablename, new TableDescriptorModtime(modtime, htd));
167 return htd;
168 }
169
170
171
172
173 @Override
174 public Map<String, HTableDescriptor> getAll()
175 throws IOException {
176 Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
177 List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
178 for (Path d: tableDirs) {
179 HTableDescriptor htd = null;
180 try {
181
182 htd = get(d.getName());
183 } catch (FileNotFoundException fnfe) {
184
185 LOG.warn("Trouble retrieving htd", fnfe);
186 }
187 if (htd == null) continue;
188 htds.put(d.getName(), htd);
189 }
190 return htds;
191 }
192
193 @Override
194 public void add(HTableDescriptor htd) throws IOException {
195 if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) {
196 throw new NotImplementedException();
197 }
198 if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) {
199 throw new NotImplementedException();
200 }
201 if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) {
202 throw new NotImplementedException();
203 }
204 if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd);
205 long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
206 this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
207 }
208
209 @Override
210 public HTableDescriptor remove(final String tablename)
211 throws IOException {
212 if (!this.fsreadonly) {
213 Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
214 if (this.fs.exists(tabledir)) {
215 if (!this.fs.delete(tabledir, true)) {
216 throw new IOException("Failed delete of " + tabledir.toString());
217 }
218 }
219 }
220 TableDescriptorModtime tdm = this.cache.remove(tablename);
221 return tdm == null? null: tdm.getTableDescriptor();
222 }
223
224
225
226
227
228
229
230
231
232
233 public static boolean isTableInfoExists(FileSystem fs, Path rootdir,
234 String tableName) throws IOException {
235 FileStatus status = getTableInfoPath(fs, rootdir, tableName);
236 return status == null? false: fs.exists(status.getPath());
237 }
238
239 private static FileStatus getTableInfoPath(final FileSystem fs,
240 final Path rootdir, final String tableName)
241 throws IOException {
242 Path tabledir = FSUtils.getTablePath(rootdir, tableName);
243 return getTableInfoPath(fs, tabledir);
244 }
245
246
247
248
249
250
251
252
253
254 private static FileStatus getTableInfoPath(final FileSystem fs,
255 final Path tabledir)
256 throws IOException {
257 FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() {
258 @Override
259 public boolean accept(Path p) {
260
261 return p.getName().startsWith(TABLEINFO_NAME);
262 }
263 });
264 if (status == null || status.length < 1) return null;
265 Arrays.sort(status, new FileStatusFileNameComparator());
266 if (status.length > 1) {
267
268 for (int i = 1; i < status.length; i++) {
269 Path p = status[i].getPath();
270
271 if (!fs.delete(p, false)) {
272 LOG.warn("Failed cleanup of " + status);
273 } else {
274 LOG.debug("Cleaned up old tableinfo file " + p);
275 }
276 }
277 }
278 return status[0];
279 }
280
281
282
283
284
285 static class FileStatusFileNameComparator
286 implements Comparator<FileStatus> {
287 @Override
288 public int compare(FileStatus left, FileStatus right) {
289 return -left.compareTo(right);
290 }
291 }
292
293
294
295
296 static final int WIDTH_OF_SEQUENCE_ID = 10;
297
298
299
300
301
302
303 static String formatTableInfoSequenceId(final int number) {
304 byte [] b = new byte[WIDTH_OF_SEQUENCE_ID];
305 int d = Math.abs(number);
306 for (int i = b.length - 1; i >= 0; i--) {
307 b[i] = (byte)((d % 10) + '0');
308 d /= 10;
309 }
310 return Bytes.toString(b);
311 }
312
313
314
315
316
317
318 private static final Pattern SUFFIX =
319 Pattern.compile(TABLEINFO_NAME + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$");
320
321
322
323
324
325
326 static int getTableInfoSequenceid(final Path p) {
327 if (p == null) return 0;
328 Matcher m = SUFFIX.matcher(p.getName());
329 if (!m.matches()) throw new IllegalArgumentException(p.toString());
330 String suffix = m.group(2);
331 if (suffix == null || suffix.length() <= 0) return 0;
332 return Integer.parseInt(m.group(2));
333 }
334
335
336
337
338
339
340 static Path getTableInfoFileName(final Path tabledir, final int sequenceid) {
341 return new Path(tabledir,
342 TABLEINFO_NAME + "." + formatTableInfoSequenceId(sequenceid));
343 }
344
345
346
347
348
349
350
351
352
353 static long getTableInfoModtime(final FileSystem fs, final Path rootdir,
354 final String tableName)
355 throws IOException {
356 FileStatus status = getTableInfoPath(fs, rootdir, tableName);
357 return status == null? 0: status.getModificationTime();
358 }
359
360
361
362
363
364
365
366
367
368 public static HTableDescriptor getTableDescriptor(FileSystem fs,
369 Path hbaseRootDir, byte[] tableName)
370 throws IOException {
371 return getTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
372 }
373
374 static HTableDescriptor getTableDescriptor(FileSystem fs,
375 Path hbaseRootDir, String tableName) {
376 HTableDescriptor htd = null;
377 try {
378 htd = getTableDescriptor(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
379 } catch (NullPointerException e) {
380 LOG.debug("Exception during readTableDecriptor. Current table name = " +
381 tableName , e);
382 } catch (IOException ioe) {
383 LOG.debug("Exception during readTableDecriptor. Current table name = " +
384 tableName , ioe);
385 }
386 return htd;
387 }
388
389 public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
390 throws IOException, NullPointerException {
391 if (tableDir == null) throw new NullPointerException();
392 FileStatus status = getTableInfoPath(fs, tableDir);
393 if (status == null) return null;
394 FSDataInputStream fsDataInputStream = fs.open(status.getPath());
395 HTableDescriptor hTableDescriptor = null;
396 try {
397 hTableDescriptor = new HTableDescriptor();
398 hTableDescriptor.readFields(fsDataInputStream);
399 } finally {
400 fsDataInputStream.close();
401 }
402 return hTableDescriptor;
403 }
404
405
406
407
408
409
410
411
412
413 static Path updateHTableDescriptor(FileSystem fs, Path rootdir,
414 HTableDescriptor hTableDescriptor)
415 throws IOException {
416 Path tableDir = FSUtils.getTablePath(rootdir, hTableDescriptor.getName());
417 Path p = writeTableDescriptor(fs, hTableDescriptor, tableDir,
418 getTableInfoPath(fs, tableDir));
419 if (p == null) throw new IOException("Failed update");
420 LOG.info("Updated tableinfo=" + p);
421 return p;
422 }
423
424
425
426
427
428 public static void deleteTableDescriptorIfExists(String tableName,
429 Configuration conf) throws IOException {
430 FileSystem fs = FSUtils.getCurrentFileSystem(conf);
431 FileStatus status = getTableInfoPath(fs, FSUtils.getRootDir(conf), tableName);
432
433 if (status != null && fs.exists(status.getPath())) {
434 FSUtils.deleteDirectory(fs, status.getPath());
435 }
436 }
437
438
439
440
441
442
443
444
445
446 private static Path writeTableDescriptor(final FileSystem fs,
447 final HTableDescriptor hTableDescriptor, final Path tableDir,
448 final FileStatus status)
449 throws IOException {
450
451
452 Path tmpTableDir = new Path(tableDir, ".tmp");
453
454
455
456
457
458 int currentSequenceid =
459 status == null? 0: getTableInfoSequenceid(status.getPath());
460 int sequenceid = currentSequenceid;
461
462 int retries = 10;
463 int retrymax = currentSequenceid + retries;
464 Path tableInfoPath = null;
465 do {
466 sequenceid += 1;
467 Path p = getTableInfoFileName(tmpTableDir, sequenceid);
468 if (fs.exists(p)) {
469 LOG.debug(p + " exists; retrying up to " + retries + " times");
470 continue;
471 }
472 try {
473 writeHTD(fs, p, hTableDescriptor);
474 tableInfoPath = getTableInfoFileName(tableDir, sequenceid);
475 if (!fs.rename(p, tableInfoPath)) {
476 throw new IOException("Failed rename of " + p + " to " + tableInfoPath);
477 }
478 } catch (IOException ioe) {
479
480 LOG.debug("Failed write and/or rename; retrying", ioe);
481 if (!FSUtils.deleteDirectory(fs, p)) {
482 LOG.warn("Failed cleanup of " + p);
483 }
484 tableInfoPath = null;
485 continue;
486 }
487
488 if (status != null) {
489 if (!FSUtils.deleteDirectory(fs, status.getPath())) {
490 LOG.warn("Failed delete of " + status.getPath() + "; continuing");
491 }
492 }
493 break;
494 } while (sequenceid < retrymax);
495 return tableInfoPath;
496 }
497
498 private static void writeHTD(final FileSystem fs, final Path p,
499 final HTableDescriptor htd)
500 throws IOException {
501 FSDataOutputStream out = fs.create(p, false);
502 try {
503 htd.write(out);
504 out.write('\n');
505 out.write('\n');
506 out.write(Bytes.toBytes(htd.toString()));
507 } finally {
508 out.close();
509 }
510 }
511
512
513
514
515
516
517
518 public static boolean createTableDescriptor(final HTableDescriptor htableDescriptor,
519 Configuration conf)
520 throws IOException {
521 return createTableDescriptor(htableDescriptor, conf, false);
522 }
523
524
525
526
527
528
529
530
531
532
533 static boolean createTableDescriptor(final HTableDescriptor htableDescriptor,
534 final Configuration conf, boolean forceCreation)
535 throws IOException {
536 FileSystem fs = FSUtils.getCurrentFileSystem(conf);
537 return createTableDescriptor(fs, FSUtils.getRootDir(conf), htableDescriptor,
538 forceCreation);
539 }
540
541
542
543
544
545
546
547
548 public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
549 HTableDescriptor htableDescriptor)
550 throws IOException {
551 return createTableDescriptor(fs, rootdir, htableDescriptor, false);
552 }
553
554
555
556
557
558
559
560
561
562
563
564
565 public static boolean createTableDescriptor(FileSystem fs, Path rootdir,
566 HTableDescriptor htableDescriptor, boolean forceCreation)
567 throws IOException {
568 FileStatus status =
569 getTableInfoPath(fs, rootdir, htableDescriptor.getNameAsString());
570 if (status != null) {
571 LOG.info("Current tableInfoPath = " + status.getPath());
572 if (!forceCreation) {
573 if (fs.exists(status.getPath()) && status.getLen() > 0) {
574 LOG.info("TableInfo already exists.. Skipping creation");
575 return false;
576 }
577 }
578 }
579 Path p = writeTableDescriptor(fs, htableDescriptor,
580 FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString()), status);
581 return p != null;
582 }
583 }