1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.io.UnsupportedEncodingException;
24 import java.lang.reflect.Constructor;
25 import java.util.AbstractList;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.NavigableSet;
33 import java.util.Random;
34 import java.util.Set;
35 import java.util.TreeMap;
36 import java.util.TreeSet;
37 import java.util.concurrent.ConcurrentSkipListMap;
38 import java.util.concurrent.atomic.AtomicBoolean;
39 import java.util.concurrent.atomic.AtomicLong;
40 import java.util.concurrent.locks.ReentrantReadWriteLock;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.hadoop.conf.Configuration;
45 import org.apache.hadoop.fs.FSDataOutputStream;
46 import org.apache.hadoop.fs.FileStatus;
47 import org.apache.hadoop.fs.FileSystem;
48 import org.apache.hadoop.fs.Path;
49 import org.apache.hadoop.hbase.DoNotRetryIOException;
50 import org.apache.hadoop.hbase.DroppedSnapshotException;
51 import org.apache.hadoop.hbase.HBaseConfiguration;
52 import org.apache.hadoop.hbase.HColumnDescriptor;
53 import org.apache.hadoop.hbase.HConstants;
54 import org.apache.hadoop.hbase.HRegionInfo;
55 import org.apache.hadoop.hbase.HTableDescriptor;
56 import org.apache.hadoop.hbase.KeyValue;
57 import org.apache.hadoop.hbase.NotServingRegionException;
58 import org.apache.hadoop.hbase.UnknownScannerException;
59 import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
60 import org.apache.hadoop.hbase.client.Delete;
61 import org.apache.hadoop.hbase.client.Get;
62 import org.apache.hadoop.hbase.client.Put;
63 import org.apache.hadoop.hbase.client.Result;
64 import org.apache.hadoop.hbase.client.RowLock;
65 import org.apache.hadoop.hbase.client.Scan;
66 import org.apache.hadoop.hbase.filter.Filter;
67 import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
68 import org.apache.hadoop.hbase.io.HeapSize;
69 import org.apache.hadoop.hbase.io.hfile.BlockCache;
70 import org.apache.hadoop.hbase.ipc.HRegionInterface;
71 import org.apache.hadoop.hbase.regionserver.wal.HLog;
72 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
73 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
74 import org.apache.hadoop.hbase.util.Bytes;
75 import org.apache.hadoop.hbase.util.ClassSize;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.FSUtils;
78 import org.apache.hadoop.hbase.util.Pair;
79 import org.apache.hadoop.hbase.util.Writables;
80 import org.apache.hadoop.io.Writable;
81 import org.apache.hadoop.util.Progressable;
82 import org.apache.hadoop.util.StringUtils;
83
84 import com.google.common.collect.Lists;
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122 public class HRegion implements HeapSize {
123 public static final Log LOG = LogFactory.getLog(HRegion.class);
124 static final String MERGEDIR = "merges";
125
126 final AtomicBoolean closed = new AtomicBoolean(false);
127
128
129
130
131
132 final AtomicBoolean closing = new AtomicBoolean(false);
133
134
135
136
137
138 private final Set<byte[]> lockedRows =
139 new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
140 private final Map<Integer, byte []> lockIds =
141 new HashMap<Integer, byte []>();
142 private int lockIdGenerator = 1;
143 static private Random rand = new Random();
144
145 protected final Map<byte [], Store> stores =
146 new ConcurrentSkipListMap<byte [], Store>(Bytes.BYTES_RAWCOMPARATOR);
147
148
149
150
151
152
153
154 final AtomicLong memstoreSize = new AtomicLong(0);
155
156
157
158
159
160 final Path tableDir;
161
162 final HLog log;
163 final FileSystem fs;
164 final Configuration conf;
165 final HRegionInfo regionInfo;
166 final Path regiondir;
167 KeyValue.KVComparator comparator;
168
169
170
171
172
173 private volatile boolean forceMajorCompaction = false;
174
175
176
177
178
179 static class WriteState {
180
181 volatile boolean flushing = false;
182
183 volatile boolean flushRequested = false;
184
185 volatile boolean compacting = false;
186
187 volatile boolean writesEnabled = true;
188
189 volatile boolean readOnly = false;
190
191
192
193
194
195
196 synchronized void setReadOnly(final boolean onOff) {
197 this.writesEnabled = !onOff;
198 this.readOnly = onOff;
199 }
200
201 boolean isReadOnly() {
202 return this.readOnly;
203 }
204
205 boolean isFlushRequested() {
206 return this.flushRequested;
207 }
208 }
209
210 private final WriteState writestate = new WriteState();
211
212 final long memstoreFlushSize;
213 private volatile long lastFlushTime;
214 final FlushRequester flushListener;
215 private final long blockingMemStoreSize;
216 final long threadWakeFrequency;
217
218 final ReentrantReadWriteLock lock =
219 new ReentrantReadWriteLock();
220
221
222 private final ReentrantReadWriteLock updatesLock =
223 new ReentrantReadWriteLock();
224 private boolean splitRequest;
225
226 private final ReadWriteConsistencyControl rwcc =
227 new ReadWriteConsistencyControl();
228
229
230
231
232 public final static String REGIONINFO_FILE = ".regioninfo";
233
234
235
236
237 public HRegion(){
238 this.tableDir = null;
239 this.blockingMemStoreSize = 0L;
240 this.conf = null;
241 this.flushListener = null;
242 this.fs = null;
243 this.memstoreFlushSize = 0L;
244 this.log = null;
245 this.regiondir = null;
246 this.regionInfo = null;
247 this.threadWakeFrequency = 0L;
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276 public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
277 HRegionInfo regionInfo, FlushRequester flushListener) {
278 this.tableDir = tableDir;
279 this.comparator = regionInfo.getComparator();
280 this.log = log;
281 this.fs = fs;
282 this.conf = conf;
283 this.regionInfo = regionInfo;
284 this.flushListener = flushListener;
285 this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
286 10 * 1000);
287 String encodedNameStr = this.regionInfo.getEncodedName();
288 this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
289 if (LOG.isDebugEnabled()) {
290
291 LOG.debug("Creating region " + this);
292 }
293 long flushSize = regionInfo.getTableDesc().getMemStoreFlushSize();
294 if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
295 flushSize = conf.getLong("hbase.hregion.memstore.flush.size",
296 HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE);
297 }
298 this.memstoreFlushSize = flushSize;
299 this.blockingMemStoreSize = this.memstoreFlushSize *
300 conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
301 }
302
303
304
305
306
307
308 public long initialize() throws IOException {
309 return initialize(null);
310 }
311
312
313
314
315
316
317
318
319 public long initialize(final Progressable reporter)
320 throws IOException {
321
322 this.closing.set(false);
323 this.closed.set(false);
324
325
326 checkRegioninfoOnFilesystem();
327
328
329 cleanupTmpDir();
330
331
332 long maxSeqId = -1;
333 for (HColumnDescriptor c : this.regionInfo.getTableDesc().getFamilies()) {
334 Store store = instantiateHStore(this.tableDir, c);
335 this.stores.put(c.getName(), store);
336 long storeSeqId = store.getMaxSequenceId();
337 if (storeSeqId > maxSeqId) {
338 maxSeqId = storeSeqId;
339 }
340 }
341
342 maxSeqId = replayRecoveredEditsIfAny(this.regiondir, maxSeqId, reporter);
343
344
345
346
347 SplitTransaction.cleanupAnySplitDetritus(this);
348 FSUtils.deleteDirectory(this.fs, new Path(regiondir, MERGEDIR));
349
350
351 if (this.regionInfo.getTableDesc().isReadOnly()) {
352 this.writestate.setReadOnly(true);
353 }
354
355 this.writestate.compacting = false;
356 this.lastFlushTime = EnvironmentEdgeManager.currentTimeMillis();
357
358
359 long nextSeqid = maxSeqId + 1;
360 LOG.info("Onlined " + this.toString() + "; next sequenceid=" + nextSeqid);
361 return nextSeqid;
362 }
363
364
365
366
367
368
369
370 static void moveInitialFilesIntoPlace(final FileSystem fs,
371 final Path initialFiles, final Path regiondir)
372 throws IOException {
373 if (initialFiles != null && fs.exists(initialFiles)) {
374 fs.rename(initialFiles, regiondir);
375 }
376 }
377
378
379
380
381 boolean hasReferences() {
382 for (Store store : this.stores.values()) {
383 for (StoreFile sf : store.getStorefiles()) {
384
385 if (sf.isReference()) return true;
386 }
387 }
388 return false;
389 }
390
391
392
393
394
395
396 private void checkRegioninfoOnFilesystem() throws IOException {
397
398
399
400
401 Path regioninfo = new Path(this.regiondir, REGIONINFO_FILE);
402 if (this.fs.exists(regioninfo) &&
403 this.fs.getFileStatus(regioninfo).getLen() > 0) {
404 return;
405 }
406 FSDataOutputStream out = this.fs.create(regioninfo, true);
407 try {
408 this.regionInfo.write(out);
409 out.write('\n');
410 out.write('\n');
411 out.write(Bytes.toBytes(this.regionInfo.toString()));
412 } finally {
413 out.close();
414 }
415 }
416
417
418 public HRegionInfo getRegionInfo() {
419 return this.regionInfo;
420 }
421
422
423 public boolean isClosed() {
424 return this.closed.get();
425 }
426
427
428
429
430 public boolean isClosing() {
431 return this.closing.get();
432 }
433
434 public ReadWriteConsistencyControl getRWCC() {
435 return rwcc;
436 }
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451 public List<StoreFile> close() throws IOException {
452 return close(false);
453 }
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469 public List<StoreFile> close(final boolean abort)
470 throws IOException {
471 if (isClosed()) {
472 LOG.warn("Region " + this + " already closed");
473 return null;
474 }
475 boolean wasFlushing = false;
476 synchronized (writestate) {
477
478
479 writestate.writesEnabled = false;
480 wasFlushing = writestate.flushing;
481 LOG.debug("Closing " + this + ": disabling compactions & flushes");
482 while (writestate.compacting || writestate.flushing) {
483 LOG.debug("waiting for" +
484 (writestate.compacting ? " compaction" : "") +
485 (writestate.flushing ?
486 (writestate.compacting ? "," : "") + " cache flush" :
487 "") + " to complete for region " + this);
488 try {
489 writestate.wait();
490 } catch (InterruptedException iex) {
491
492 }
493 }
494 }
495
496
497
498 if (!abort && !wasFlushing && worthPreFlushing()) {
499 LOG.info("Running close preflush of " + this.getRegionNameAsString());
500 internalFlushcache();
501 }
502 this.closing.set(true);
503 lock.writeLock().lock();
504 try {
505 if (this.isClosed()) {
506
507 return null;
508 }
509 LOG.debug("Updates disabled for region " + this);
510
511 if (!abort) {
512 internalFlushcache();
513 }
514
515 List<StoreFile> result = new ArrayList<StoreFile>();
516 for (Store store : stores.values()) {
517 result.addAll(store.close());
518 }
519 this.closed.set(true);
520 LOG.info("Closed " + this);
521 return result;
522 } finally {
523 lock.writeLock().unlock();
524 }
525 }
526
527
528
529
530 private boolean worthPreFlushing() {
531 return this.memstoreSize.get() >
532 this.conf.getLong("hbase.hregion.preclose.flush.size", 1024 * 1024 * 5);
533 }
534
535
536
537
538
539
540 public byte [] getStartKey() {
541 return this.regionInfo.getStartKey();
542 }
543
544
545 public byte [] getEndKey() {
546 return this.regionInfo.getEndKey();
547 }
548
549
550 public long getRegionId() {
551 return this.regionInfo.getRegionId();
552 }
553
554
555 public byte [] getRegionName() {
556 return this.regionInfo.getRegionName();
557 }
558
559
560 public String getRegionNameAsString() {
561 return this.regionInfo.getRegionNameAsString();
562 }
563
564
565 public HTableDescriptor getTableDesc() {
566 return this.regionInfo.getTableDesc();
567 }
568
569
570 public HLog getLog() {
571 return this.log;
572 }
573
574
575 public Configuration getConf() {
576 return this.conf;
577 }
578
579
580 public Path getRegionDir() {
581 return this.regiondir;
582 }
583
584
585
586
587
588
589
590
591 public static Path getRegionDir(final Path tabledir, final String name) {
592 return new Path(tabledir, name);
593 }
594
595
596 public FileSystem getFilesystem() {
597 return this.fs;
598 }
599
600
601 public long getLastFlushTime() {
602 return this.lastFlushTime;
603 }
604
605
606
607
608
609
610
611
612
613 public long getLargestHStoreSize() {
614 long size = 0;
615 for (Store h: stores.values()) {
616 long storeSize = h.getSize();
617 if (storeSize > size) {
618 size = storeSize;
619 }
620 }
621 return size;
622 }
623
624
625
626
627
628 private void doRegionCompactionPrep() throws IOException {
629 }
630
631
632
633
634 private void cleanupTmpDir() throws IOException {
635 FSUtils.deleteDirectory(this.fs, getTmpDir());
636 }
637
638
639
640
641
642 Path getTmpDir() {
643 return new Path(getRegionDir(), ".tmp");
644 }
645
646 void setForceMajorCompaction(final boolean b) {
647 this.forceMajorCompaction = b;
648 }
649
650 boolean getForceMajorCompaction() {
651 return this.forceMajorCompaction;
652 }
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668 public byte [] compactStores() throws IOException {
669 boolean majorCompaction = this.forceMajorCompaction;
670 this.forceMajorCompaction = false;
671 return compactStores(majorCompaction);
672 }
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689 byte [] compactStores(final boolean majorCompaction)
690 throws IOException {
691 if (this.closing.get()) {
692 LOG.debug("Skipping compaction on " + this + " because closing");
693 return null;
694 }
695 lock.readLock().lock();
696 try {
697 if (this.closed.get()) {
698 LOG.debug("Skipping compaction on " + this + " because closed");
699 return null;
700 }
701 byte [] splitRow = null;
702 if (this.closed.get()) {
703 return splitRow;
704 }
705 try {
706 synchronized (writestate) {
707 if (!writestate.compacting && writestate.writesEnabled) {
708 writestate.compacting = true;
709 } else {
710 LOG.info("NOT compacting region " + this +
711 ": compacting=" + writestate.compacting + ", writesEnabled=" +
712 writestate.writesEnabled);
713 return splitRow;
714 }
715 }
716 LOG.info("Starting" + (majorCompaction? " major " : " ") +
717 "compaction on region " + this);
718 long startTime = EnvironmentEdgeManager.currentTimeMillis();
719 doRegionCompactionPrep();
720 long maxSize = -1;
721 for (Store store: stores.values()) {
722 final Store.StoreSize ss = store.compact(majorCompaction);
723 if (ss != null && ss.getSize() > maxSize) {
724 maxSize = ss.getSize();
725 splitRow = ss.getSplitRow();
726 }
727 }
728 String timeTaken = StringUtils.formatTimeDiff(EnvironmentEdgeManager.currentTimeMillis(),
729 startTime);
730 LOG.info("compaction completed on region " + this + " in " + timeTaken);
731 } finally {
732 synchronized (writestate) {
733 writestate.compacting = false;
734 writestate.notifyAll();
735 }
736 }
737 return splitRow;
738 } finally {
739 lock.readLock().unlock();
740 }
741 }
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763 public boolean flushcache() throws IOException {
764
765 if (this.closing.get()) {
766 LOG.debug("Skipping flush on " + this + " because closing");
767 return false;
768 }
769 lock.readLock().lock();
770 try {
771 if (this.closed.get()) {
772 LOG.debug("Skipping flush on " + this + " because closed");
773 return false;
774 }
775 try {
776 synchronized (writestate) {
777 if (!writestate.flushing && writestate.writesEnabled) {
778 this.writestate.flushing = true;
779 } else {
780 if (LOG.isDebugEnabled()) {
781 LOG.debug("NOT flushing memstore for region " + this +
782 ", flushing=" +
783 writestate.flushing + ", writesEnabled=" +
784 writestate.writesEnabled);
785 }
786 return false;
787 }
788 }
789 return internalFlushcache();
790 } finally {
791 synchronized (writestate) {
792 writestate.flushing = false;
793 this.writestate.flushRequested = false;
794 writestate.notifyAll();
795 }
796 }
797 } finally {
798 lock.readLock().unlock();
799 }
800 }
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836 protected boolean internalFlushcache() throws IOException {
837 return internalFlushcache(this.log, -1);
838 }
839
840
841
842
843
844
845
846
847
848 protected boolean internalFlushcache(final HLog wal, final long myseqid)
849 throws IOException {
850 final long startTime = EnvironmentEdgeManager.currentTimeMillis();
851
852
853 this.lastFlushTime = startTime;
854
855 if (this.memstoreSize.get() <= 0) {
856 return false;
857 }
858 if (LOG.isDebugEnabled()) {
859 LOG.debug("Started memstore flush for region " + this +
860 ". Current region memstore size " +
861 StringUtils.humanReadableInt(this.memstoreSize.get()) +
862 ((wal != null)? "": "; wal is null, using passed sequenceid=" + myseqid));
863 }
864
865
866
867
868
869
870
871
872 long sequenceId = -1L;
873 long completeSequenceId = -1L;
874
875
876
877
878 this.updatesLock.writeLock().lock();
879 final long currentMemStoreSize = this.memstoreSize.get();
880 List<StoreFlusher> storeFlushers = new ArrayList<StoreFlusher>(stores.size());
881 try {
882 sequenceId = (wal == null)? myseqid: wal.startCacheFlush();
883 completeSequenceId = this.getCompleteCacheFlushSequenceId(sequenceId);
884
885 for (Store s : stores.values()) {
886 storeFlushers.add(s.getStoreFlusher(completeSequenceId));
887 }
888
889
890 for (StoreFlusher flusher : storeFlushers) {
891 flusher.prepare();
892 }
893 } finally {
894 this.updatesLock.writeLock().unlock();
895 }
896
897 LOG.debug("Finished snapshotting, commencing flushing stores");
898
899
900
901
902
903 boolean compactionRequested = false;
904 try {
905
906
907
908
909 for (StoreFlusher flusher : storeFlushers) {
910 flusher.flushCache();
911 }
912
913
914 for (StoreFlusher flusher : storeFlushers) {
915 boolean needsCompaction = flusher.commit();
916 if (needsCompaction) {
917 compactionRequested = true;
918 }
919 }
920 storeFlushers.clear();
921
922
923 this.memstoreSize.addAndGet(-currentMemStoreSize);
924 } catch (Throwable t) {
925
926
927
928
929
930
931 if (wal != null) wal.abortCacheFlush();
932 DroppedSnapshotException dse = new DroppedSnapshotException("region: " +
933 Bytes.toStringBinary(getRegionName()));
934 dse.initCause(t);
935 throw dse;
936 }
937
938
939
940
941
942
943
944
945 if (wal != null) {
946 wal.completeCacheFlush(getRegionName(),
947 regionInfo.getTableDesc().getName(), completeSequenceId,
948 this.getRegionInfo().isMetaRegion());
949 }
950
951
952
953 synchronized (this) {
954 notifyAll();
955 }
956
957 if (LOG.isDebugEnabled()) {
958 long now = EnvironmentEdgeManager.currentTimeMillis();
959 LOG.info("Finished memstore flush of ~" +
960 StringUtils.humanReadableInt(currentMemStoreSize) + " for region " +
961 this + " in " + (now - startTime) + "ms, sequenceid=" + sequenceId +
962 ", compaction requested=" + compactionRequested +
963 ((wal == null)? "; wal=null": ""));
964 }
965 return compactionRequested;
966 }
967
968
969
970
971
972
973
974
975
976 protected long getCompleteCacheFlushSequenceId(long currentSequenceId) {
977 return currentSequenceId;
978 }
979
980
981
982
983
984
985
986
987
988
989
990
991
992 Result getClosestRowBefore(final byte [] row)
993 throws IOException{
994 return getClosestRowBefore(row, HConstants.CATALOG_FAMILY);
995 }
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007 public Result getClosestRowBefore(final byte [] row, final byte [] family)
1008 throws IOException {
1009
1010
1011 KeyValue key = null;
1012 checkRow(row);
1013 startRegionOperation();
1014 try {
1015 Store store = getStore(family);
1016 KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP);
1017
1018 key = store.getRowKeyAtOrBefore(kv);
1019 if (key == null) {
1020 return null;
1021 }
1022 Get get = new Get(key.getRow());
1023 get.addFamily(family);
1024 return get(get, null);
1025 } finally {
1026 closeRegionOperation();
1027 }
1028 }
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040 public InternalScanner getScanner(Scan scan)
1041 throws IOException {
1042 return getScanner(scan, null);
1043 }
1044
1045 protected InternalScanner getScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
1046 startRegionOperation();
1047 try {
1048
1049 if(scan.hasFamilies()) {
1050 for(byte [] family : scan.getFamilyMap().keySet()) {
1051 checkFamily(family);
1052 }
1053 } else {
1054 for(byte[] family: regionInfo.getTableDesc().getFamiliesKeys()){
1055 scan.addFamily(family);
1056 }
1057 }
1058 return instantiateInternalScanner(scan, additionalScanners);
1059
1060 } finally {
1061 closeRegionOperation();
1062 }
1063 }
1064
1065 protected InternalScanner instantiateInternalScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
1066 return new RegionScanner(scan, additionalScanners);
1067 }
1068
1069
1070
1071
1072 private void prepareDelete(Delete delete) throws IOException {
1073
1074 if(delete.getFamilyMap().isEmpty()){
1075 for(byte [] family : regionInfo.getTableDesc().getFamiliesKeys()){
1076
1077 delete.deleteFamily(family, delete.getTimeStamp());
1078 }
1079 } else {
1080 for(byte [] family : delete.getFamilyMap().keySet()) {
1081 if(family == null) {
1082 throw new NoSuchColumnFamilyException("Empty family is invalid");
1083 }
1084 checkFamily(family);
1085 }
1086 }
1087 }
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098 public void delete(Delete delete, Integer lockid, boolean writeToWAL)
1099 throws IOException {
1100 checkReadOnly();
1101 checkResources();
1102 Integer lid = null;
1103 startRegionOperation();
1104 try {
1105 byte [] row = delete.getRow();
1106
1107 lid = getLock(lockid, row, true);
1108
1109
1110 prepareDelete(delete);
1111 delete(delete.getFamilyMap(), writeToWAL);
1112
1113 } finally {
1114 if(lockid == null) releaseRowLock(lid);
1115 closeRegionOperation();
1116 }
1117 }
1118
1119
1120
1121
1122
1123
1124
1125 public void delete(Map<byte[], List<KeyValue>> familyMap, boolean writeToWAL)
1126 throws IOException {
1127 long now = EnvironmentEdgeManager.currentTimeMillis();
1128 byte [] byteNow = Bytes.toBytes(now);
1129 boolean flush = false;
1130
1131 updatesLock.readLock().lock();
1132
1133 try {
1134
1135 for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
1136
1137 byte[] family = e.getKey();
1138 List<KeyValue> kvs = e.getValue();
1139 Map<byte[], Integer> kvCount = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
1140
1141 for (KeyValue kv: kvs) {
1142
1143
1144 if (kv.isLatestTimestamp() && kv.isDeleteType()) {
1145 byte[] qual = kv.getQualifier();
1146 if (qual == null) qual = HConstants.EMPTY_BYTE_ARRAY;
1147
1148 Integer count = kvCount.get(qual);
1149 if (count == null) {
1150 kvCount.put(qual, 1);
1151 } else {
1152 kvCount.put(qual, count + 1);
1153 }
1154 count = kvCount.get(qual);
1155
1156 Get get = new Get(kv.getRow());
1157 get.setMaxVersions(count);
1158 get.addColumn(family, qual);
1159
1160 List<KeyValue> result = get(get);
1161
1162 if (result.size() < count) {
1163
1164 kv.updateLatestStamp(byteNow);
1165 continue;
1166 }
1167 if (result.size() > count) {
1168 throw new RuntimeException("Unexpected size: " + result.size());
1169 }
1170 KeyValue getkv = result.get(count - 1);
1171 Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(),
1172 getkv.getBuffer(), getkv.getTimestampOffset(), Bytes.SIZEOF_LONG);
1173 } else {
1174 kv.updateLatestStamp(byteNow);
1175 }
1176 }
1177 }
1178
1179 if (writeToWAL) {
1180
1181
1182
1183
1184
1185
1186
1187
1188 WALEdit walEdit = new WALEdit();
1189 addFamilyMapToWALEdit(familyMap, walEdit);
1190 this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
1191 walEdit, now);
1192 }
1193
1194
1195 long addedSize = applyFamilyMapToMemstore(familyMap);
1196 flush = isFlushSize(memstoreSize.addAndGet(addedSize));
1197 } finally {
1198 this.updatesLock.readLock().unlock();
1199 }
1200
1201 if (flush) {
1202
1203 requestFlush();
1204 }
1205 }
1206
1207
1208
1209
1210
1211 public void put(Put put) throws IOException {
1212 this.put(put, null, put.getWriteToWAL());
1213 }
1214
1215
1216
1217
1218
1219
1220 public void put(Put put, boolean writeToWAL) throws IOException {
1221 this.put(put, null, writeToWAL);
1222 }
1223
1224
1225
1226
1227
1228
1229 public void put(Put put, Integer lockid) throws IOException {
1230 this.put(put, lockid, put.getWriteToWAL());
1231 }
1232
1233
1234
1235
1236
1237
1238
1239 public void put(Put put, Integer lockid, boolean writeToWAL)
1240 throws IOException {
1241 checkReadOnly();
1242
1243
1244
1245
1246
1247 checkResources();
1248 startRegionOperation();
1249 try {
1250
1251
1252
1253
1254
1255 byte [] row = put.getRow();
1256
1257 Integer lid = getLock(lockid, row, true);
1258
1259 try {
1260
1261 put(put.getFamilyMap(), writeToWAL);
1262 } finally {
1263 if(lockid == null) releaseRowLock(lid);
1264 }
1265 } finally {
1266 closeRegionOperation();
1267 }
1268 }
1269
1270
1271
1272
1273
1274
1275 private static class BatchOperationInProgress<T> {
1276 T[] operations;
1277 OperationStatusCode[] retCodes;
1278 int nextIndexToProcess = 0;
1279
1280 public BatchOperationInProgress(T[] operations) {
1281 this.operations = operations;
1282 retCodes = new OperationStatusCode[operations.length];
1283 Arrays.fill(retCodes, OperationStatusCode.NOT_RUN);
1284 }
1285
1286 public boolean isDone() {
1287 return nextIndexToProcess == operations.length;
1288 }
1289 }
1290
1291
1292
1293
1294
1295 public OperationStatusCode[] put(Put[] puts) throws IOException {
1296 @SuppressWarnings("unchecked")
1297 Pair<Put, Integer> putsAndLocks[] = new Pair[puts.length];
1298
1299 for (int i = 0; i < puts.length; i++) {
1300 putsAndLocks[i] = new Pair<Put, Integer>(puts[i], null);
1301 }
1302 return put(putsAndLocks);
1303 }
1304
1305
1306
1307
1308
1309
1310 public OperationStatusCode[] put(Pair<Put, Integer>[] putsAndLocks) throws IOException {
1311 BatchOperationInProgress<Pair<Put, Integer>> batchOp =
1312 new BatchOperationInProgress<Pair<Put,Integer>>(putsAndLocks);
1313
1314 while (!batchOp.isDone()) {
1315 checkReadOnly();
1316 checkResources();
1317
1318 long newSize;
1319 startRegionOperation();
1320 try {
1321 long addedSize = doMiniBatchPut(batchOp);
1322 newSize = memstoreSize.addAndGet(addedSize);
1323 } finally {
1324 closeRegionOperation();
1325 }
1326 if (isFlushSize(newSize)) {
1327 requestFlush();
1328 }
1329 }
1330 return batchOp.retCodes;
1331 }
1332
1333 private long doMiniBatchPut(BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException {
1334 long now = EnvironmentEdgeManager.currentTimeMillis();
1335 byte[] byteNow = Bytes.toBytes(now);
1336
1337
1338 List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
1339
1340 int firstIndex = batchOp.nextIndexToProcess;
1341 int lastIndexExclusive = firstIndex;
1342 boolean success = false;
1343 try {
1344
1345
1346
1347
1348 int numReadyToWrite = 0;
1349 while (lastIndexExclusive < batchOp.operations.length) {
1350 Pair<Put, Integer> nextPair = batchOp.operations[lastIndexExclusive];
1351 Put put = nextPair.getFirst();
1352 Integer providedLockId = nextPair.getSecond();
1353
1354
1355 try {
1356 checkFamilies(put.getFamilyMap().keySet());
1357 } catch (NoSuchColumnFamilyException nscf) {
1358 LOG.warn("No such column family in batch put", nscf);
1359 batchOp.retCodes[lastIndexExclusive] = OperationStatusCode.BAD_FAMILY;
1360 lastIndexExclusive++;
1361 continue;
1362 }
1363
1364
1365
1366 boolean shouldBlock = numReadyToWrite == 0;
1367 Integer acquiredLockId = getLock(providedLockId, put.getRow(), shouldBlock);
1368 if (acquiredLockId == null) {
1369
1370 assert !shouldBlock : "Should never fail to get lock when blocking";
1371 break;
1372 }
1373 if (providedLockId == null) {
1374 acquiredLocks.add(acquiredLockId);
1375 }
1376 lastIndexExclusive++;
1377 numReadyToWrite++;
1378 }
1379
1380 assert numReadyToWrite > 0;
1381
1382
1383
1384
1385 for (int i = firstIndex; i < lastIndexExclusive; i++) {
1386 updateKVTimestamps(
1387 batchOp.operations[i].getFirst().getFamilyMap().values(),
1388 byteNow);
1389 }
1390
1391
1392
1393
1394 WALEdit walEdit = new WALEdit();
1395 for (int i = firstIndex; i < lastIndexExclusive; i++) {
1396
1397 if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
1398
1399 Put p = batchOp.operations[i].getFirst();
1400 if (!p.getWriteToWAL()) continue;
1401 addFamilyMapToWALEdit(p.getFamilyMap(), walEdit);
1402 }
1403
1404
1405 this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
1406 walEdit, now);
1407
1408
1409
1410
1411 long addedSize = 0;
1412 for (int i = firstIndex; i < lastIndexExclusive; i++) {
1413 if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
1414
1415 Put p = batchOp.operations[i].getFirst();
1416 addedSize += applyFamilyMapToMemstore(p.getFamilyMap());
1417 batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
1418 }
1419 success = true;
1420 return addedSize;
1421 } finally {
1422 for (Integer toRelease : acquiredLocks) {
1423 releaseRowLock(toRelease);
1424 }
1425 if (!success) {
1426 for (int i = firstIndex; i < lastIndexExclusive; i++) {
1427 if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
1428 batchOp.retCodes[i] = OperationStatusCode.FAILURE;
1429 }
1430 }
1431 }
1432 batchOp.nextIndexToProcess = lastIndexExclusive;
1433 }
1434 }
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451 public boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
1452 byte [] expectedValue, Writable w, Integer lockId, boolean writeToWAL)
1453 throws IOException{
1454 checkReadOnly();
1455
1456
1457 checkResources();
1458 boolean isPut = w instanceof Put;
1459 if (!isPut && !(w instanceof Delete))
1460 throw new IOException("Action must be Put or Delete");
1461
1462 startRegionOperation();
1463 try {
1464 RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock();
1465 Get get = new Get(row, lock);
1466 checkFamily(family);
1467 get.addColumn(family, qualifier);
1468
1469
1470 Integer lid = getLock(lockId, get.getRow(), true);
1471 List<KeyValue> result = new ArrayList<KeyValue>();
1472 try {
1473 result = get(get);
1474
1475 boolean matches = false;
1476 if (result.size() == 0 &&
1477 (expectedValue == null || expectedValue.length == 0)) {
1478 matches = true;
1479 } else if (result.size() == 1) {
1480
1481 byte [] actualValue = result.get(0).getValue();
1482 matches = Bytes.equals(expectedValue, actualValue);
1483 }
1484
1485 if (matches) {
1486
1487 if (isPut) {
1488 put(((Put)w).getFamilyMap(), writeToWAL);
1489 } else {
1490 Delete d = (Delete)w;
1491 prepareDelete(d);
1492 delete(d.getFamilyMap(), writeToWAL);
1493 }
1494 return true;
1495 }
1496 return false;
1497 } finally {
1498 if(lockId == null) releaseRowLock(lid);
1499 }
1500 } finally {
1501 closeRegionOperation();
1502 }
1503 }
1504
1505
1506
1507
1508
1509
1510 private void updateKVTimestamps(
1511 final Iterable<List<KeyValue>> keyLists, final byte[] now) {
1512 for (List<KeyValue> keys: keyLists) {
1513 if (keys == null) continue;
1514 for (KeyValue key : keys) {
1515 key.updateLatestStamp(now);
1516 }
1517 }
1518 }
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551 private void checkResources() {
1552
1553
1554 if (this.getRegionInfo().isMetaRegion()) return;
1555
1556 boolean blocked = false;
1557 while (this.memstoreSize.get() > this.blockingMemStoreSize) {
1558 requestFlush();
1559 if (!blocked) {
1560 LOG.info("Blocking updates for '" + Thread.currentThread().getName() +
1561 "' on region " + Bytes.toStringBinary(getRegionName()) +
1562 ": memstore size " +
1563 StringUtils.humanReadableInt(this.memstoreSize.get()) +
1564 " is >= than blocking " +
1565 StringUtils.humanReadableInt(this.blockingMemStoreSize) + " size");
1566 }
1567 blocked = true;
1568 synchronized(this) {
1569 try {
1570 wait(threadWakeFrequency);
1571 } catch (InterruptedException e) {
1572
1573 }
1574 }
1575 }
1576 if (blocked) {
1577 LOG.info("Unblocking updates for region " + this + " '"
1578 + Thread.currentThread().getName() + "'");
1579 }
1580 }
1581
1582
1583
1584
1585 protected void checkReadOnly() throws IOException {
1586 if (this.writestate.isReadOnly()) {
1587 throw new IOException("region is read only");
1588 }
1589 }
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599 private void put(final byte [] family, final List<KeyValue> edits)
1600 throws IOException {
1601 Map<byte[], List<KeyValue>> familyMap = new HashMap<byte[], List<KeyValue>>();
1602 familyMap.put(family, edits);
1603 this.put(familyMap, true);
1604 }
1605
1606
1607
1608
1609
1610
1611
1612
1613 private void put(final Map<byte [], List<KeyValue>> familyMap,
1614 boolean writeToWAL) throws IOException {
1615 long now = EnvironmentEdgeManager.currentTimeMillis();
1616 byte[] byteNow = Bytes.toBytes(now);
1617 boolean flush = false;
1618 this.updatesLock.readLock().lock();
1619 try {
1620 checkFamilies(familyMap.keySet());
1621 updateKVTimestamps(familyMap.values(), byteNow);
1622
1623
1624
1625
1626
1627 if (writeToWAL) {
1628 WALEdit walEdit = new WALEdit();
1629 addFamilyMapToWALEdit(familyMap, walEdit);
1630 this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
1631 walEdit, now);
1632 }
1633
1634 long addedSize = applyFamilyMapToMemstore(familyMap);
1635 flush = isFlushSize(memstoreSize.addAndGet(addedSize));
1636 } finally {
1637 this.updatesLock.readLock().unlock();
1638 }
1639 if (flush) {
1640
1641 requestFlush();
1642 }
1643 }
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654 private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap) {
1655 ReadWriteConsistencyControl.WriteEntry w = null;
1656 long size = 0;
1657 try {
1658 w = rwcc.beginMemstoreInsert();
1659
1660 for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
1661 byte[] family = e.getKey();
1662 List<KeyValue> edits = e.getValue();
1663
1664 Store store = getStore(family);
1665 for (KeyValue kv: edits) {
1666 kv.setMemstoreTS(w.getWriteNumber());
1667 size += store.add(kv);
1668 }
1669 }
1670 } finally {
1671 rwcc.completeMemstoreInsert(w);
1672 }
1673 return size;
1674 }
1675
1676
1677
1678
1679
1680 private void checkFamilies(Collection<byte[]> families)
1681 throws NoSuchColumnFamilyException {
1682 for (byte[] family : families) {
1683 checkFamily(family);
1684 }
1685 }
1686
1687
1688
1689
1690
1691
1692
1693 private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
1694 WALEdit walEdit) {
1695 for (List<KeyValue> edits : familyMap.values()) {
1696 for (KeyValue kv : edits) {
1697 walEdit.add(kv);
1698 }
1699 }
1700 }
1701
1702 private void requestFlush() {
1703 if (this.flushListener == null) {
1704 return;
1705 }
1706 synchronized (writestate) {
1707 if (this.writestate.isFlushRequested()) {
1708 return;
1709 }
1710 writestate.flushRequested = true;
1711 }
1712
1713 this.flushListener.request(this);
1714 if (LOG.isDebugEnabled()) {
1715 LOG.debug("Flush requested on " + this);
1716 }
1717 }
1718
1719
1720
1721
1722
1723 private boolean isFlushSize(final long size) {
1724 return size > this.memstoreFlushSize;
1725 }
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763 protected long replayRecoveredEditsIfAny(final Path regiondir,
1764 final long minSeqId, final Progressable reporter)
1765 throws UnsupportedEncodingException, IOException {
1766 long seqid = minSeqId;
1767 NavigableSet<Path> files = HLog.getSplitEditFilesSorted(this.fs, regiondir);
1768 if (files == null || files.isEmpty()) return seqid;
1769 for (Path edits: files) {
1770 if (edits == null || !this.fs.exists(edits)) {
1771 LOG.warn("Null or non-existent edits file: " + edits);
1772 continue;
1773 }
1774 if (isZeroLengthThenDelete(this.fs, edits)) continue;
1775 try {
1776 seqid = replayRecoveredEdits(edits, seqid, reporter);
1777 } catch (IOException e) {
1778 boolean skipErrors = conf.getBoolean("hbase.skip.errors", false);
1779 if (skipErrors) {
1780 Path p = HLog.moveAsideBadEditsFile(fs, edits);
1781 LOG.error("hbase.skip.errors=true so continuing. Renamed " + edits +
1782 " as " + p, e);
1783 } else {
1784 throw e;
1785 }
1786 }
1787 }
1788 if (seqid > minSeqId) {
1789
1790 internalFlushcache(null, seqid);
1791 for (Path file: files) {
1792 if (!this.fs.delete(file, false)) {
1793 LOG.error("Failed delete of " + file);
1794 } else {
1795 LOG.debug("Deleted recovered.edits file=" + file);
1796 }
1797 }
1798 }
1799 return seqid;
1800 }
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811 private long replayRecoveredEdits(final Path edits,
1812 final long minSeqId, final Progressable reporter)
1813 throws IOException {
1814 LOG.info("Replaying edits from " + edits + "; minSequenceid=" + minSeqId);
1815 HLog.Reader reader = HLog.getReader(this.fs, edits, conf);
1816 try {
1817 return replayRecoveredEdits(reader, minSeqId, reporter);
1818 } finally {
1819 reader.close();
1820 }
1821 }
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831 private long replayRecoveredEdits(final HLog.Reader reader,
1832 final long minSeqId, final Progressable reporter)
1833 throws IOException {
1834 long currentEditSeqId = minSeqId;
1835 long firstSeqIdInLog = -1;
1836 long skippedEdits = 0;
1837 long editsCount = 0;
1838 HLog.Entry entry;
1839 Store store = null;
1840
1841 int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);
1842 while ((entry = reader.next()) != null) {
1843 HLogKey key = entry.getKey();
1844 WALEdit val = entry.getEdit();
1845 if (firstSeqIdInLog == -1) {
1846 firstSeqIdInLog = key.getLogSeqNum();
1847 }
1848
1849 if (key.getLogSeqNum() <= currentEditSeqId) {
1850 skippedEdits++;
1851 continue;
1852 }
1853 currentEditSeqId = key.getLogSeqNum();
1854 boolean flush = false;
1855 for (KeyValue kv: val.getKeyValues()) {
1856
1857
1858 if (kv.matchingFamily(HLog.METAFAMILY) ||
1859 !Bytes.equals(key.getRegionName(), this.regionInfo.getRegionName())) {
1860 skippedEdits++;
1861 continue;
1862 }
1863
1864 if (store == null || !kv.matchingFamily(store.getFamily().getName())) {
1865 store = this.stores.get(kv.getFamily());
1866 }
1867 if (store == null) {
1868
1869
1870 LOG.warn("No family for " + kv);
1871 skippedEdits++;
1872 continue;
1873 }
1874
1875
1876
1877 flush = restoreEdit(store, kv);
1878 editsCount++;
1879 }
1880 if (flush) internalFlushcache(null, currentEditSeqId);
1881
1882
1883
1884 if (reporter != null && (editsCount % interval) == 0) {
1885 reporter.progress();
1886 }
1887 }
1888 if (LOG.isDebugEnabled()) {
1889 LOG.debug("Applied " + editsCount + ", skipped " + skippedEdits +
1890 ", firstSequenceidInLog=" + firstSeqIdInLog +
1891 ", maxSequenceidInLog=" + currentEditSeqId);
1892 }
1893 return currentEditSeqId;
1894 }
1895
1896
1897
1898
1899
1900
1901
1902 protected boolean restoreEdit(final Store s, final KeyValue kv) {
1903 return isFlushSize(this.memstoreSize.addAndGet(s.add(kv)));
1904 }
1905
1906
1907
1908
1909
1910
1911
1912 private static boolean isZeroLengthThenDelete(final FileSystem fs, final Path p)
1913 throws IOException {
1914 FileStatus stat = fs.getFileStatus(p);
1915 if (stat.getLen() > 0) return false;
1916 LOG.warn("File " + p + " is zero-length, deleting.");
1917 fs.delete(p, false);
1918 return true;
1919 }
1920
1921 protected Store instantiateHStore(Path tableDir, HColumnDescriptor c)
1922 throws IOException {
1923 return new Store(tableDir, this, c, this.fs, this.conf);
1924 }
1925
1926
1927
1928
1929
1930
1931
1932
1933 public Store getStore(final byte [] column) {
1934 return this.stores.get(column);
1935 }
1936
1937
1938
1939
1940
1941
1942 private void checkRow(final byte [] row) throws IOException {
1943 if(!rowIsInRange(regionInfo, row)) {
1944 throw new WrongRegionException("Requested row out of range for " +
1945 "HRegion " + this + ", startKey='" +
1946 Bytes.toStringBinary(regionInfo.getStartKey()) + "', getEndKey()='" +
1947 Bytes.toStringBinary(regionInfo.getEndKey()) + "', row='" +
1948 Bytes.toStringBinary(row) + "'");
1949 }
1950 }
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975 public Integer obtainRowLock(final byte [] row) throws IOException {
1976 startRegionOperation();
1977 try {
1978 return internalObtainRowLock(row, true);
1979 } finally {
1980 closeRegionOperation();
1981 }
1982 }
1983
1984
1985
1986
1987
1988
1989
1990 public Integer tryObtainRowLock(final byte[] row) throws IOException {
1991 startRegionOperation();
1992 try {
1993 return internalObtainRowLock(row, false);
1994 } finally {
1995 closeRegionOperation();
1996 }
1997 }
1998
1999
2000
2001
2002
2003
2004
2005 private Integer internalObtainRowLock(final byte[] row, boolean waitForLock)
2006 throws IOException {
2007 checkRow(row);
2008 startRegionOperation();
2009 try {
2010 synchronized (lockedRows) {
2011 while (lockedRows.contains(row)) {
2012 if (!waitForLock) {
2013 return null;
2014 }
2015 try {
2016 lockedRows.wait();
2017 } catch (InterruptedException ie) {
2018
2019 }
2020 }
2021
2022
2023
2024
2025
2026
2027 byte [] prev = null;
2028 Integer lockId = null;
2029 do {
2030 lockId = new Integer(lockIdGenerator++);
2031 prev = lockIds.put(lockId, row);
2032 if (prev != null) {
2033 lockIds.put(lockId, prev);
2034 lockIdGenerator = rand.nextInt();
2035 }
2036 } while (prev != null);
2037
2038 lockedRows.add(row);
2039 lockedRows.notifyAll();
2040 return lockId;
2041 }
2042 } finally {
2043 closeRegionOperation();
2044 }
2045 }
2046
2047
2048
2049
2050
2051
2052 byte [] getRowFromLock(final Integer lockid) {
2053 synchronized (lockedRows) {
2054 return lockIds.get(lockid);
2055 }
2056 }
2057
2058
2059
2060
2061
2062 void releaseRowLock(final Integer lockid) {
2063 synchronized (lockedRows) {
2064 byte[] row = lockIds.remove(lockid);
2065 lockedRows.remove(row);
2066 lockedRows.notifyAll();
2067 }
2068 }
2069
2070
2071
2072
2073
2074
2075 boolean isRowLocked(final Integer lockid) {
2076 synchronized (lockedRows) {
2077 if (lockIds.get(lockid) != null) {
2078 return true;
2079 }
2080 return false;
2081 }
2082 }
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093 private Integer getLock(Integer lockid, byte [] row, boolean waitForLock)
2094 throws IOException {
2095 Integer lid = null;
2096 if (lockid == null) {
2097 lid = internalObtainRowLock(row, waitForLock);
2098 } else {
2099 if (!isRowLocked(lockid)) {
2100 throw new IOException("Invalid row lock");
2101 }
2102 lid = lockid;
2103 }
2104 return lid;
2105 }
2106
2107 public void bulkLoadHFile(String hfilePath, byte[] familyName)
2108 throws IOException {
2109 startRegionOperation();
2110 try {
2111 Store store = getStore(familyName);
2112 if (store == null) {
2113 throw new DoNotRetryIOException(
2114 "No such column family " + Bytes.toStringBinary(familyName));
2115 }
2116 store.bulkLoadHFile(hfilePath);
2117 } finally {
2118 closeRegionOperation();
2119 }
2120
2121 }
2122
2123
2124 @Override
2125 public boolean equals(Object o) {
2126 if (!(o instanceof HRegion)) {
2127 return false;
2128 }
2129 return this.hashCode() == ((HRegion)o).hashCode();
2130 }
2131
2132 @Override
2133 public int hashCode() {
2134 return Bytes.hashCode(this.regionInfo.getRegionName());
2135 }
2136
2137 @Override
2138 public String toString() {
2139 return this.regionInfo.getRegionNameAsString();
2140 }
2141
2142
2143 public Path getTableDir() {
2144 return this.tableDir;
2145 }
2146
2147
2148
2149
2150
2151
2152 class RegionScanner implements InternalScanner {
2153
2154 KeyValueHeap storeHeap = null;
2155 private final byte [] stopRow;
2156 private Filter filter;
2157 private List<KeyValue> results = new ArrayList<KeyValue>();
2158 private int batch;
2159 private int isScan;
2160 private boolean filterClosed = false;
2161 private long readPt;
2162
2163 public HRegionInfo getRegionName() {
2164 return regionInfo;
2165 }
2166 RegionScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
2167
2168 this.filter = scan.getFilter();
2169 this.batch = scan.getBatch();
2170 if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
2171 this.stopRow = null;
2172 } else {
2173 this.stopRow = scan.getStopRow();
2174 }
2175
2176
2177 this.isScan = scan.isGetScan() ? -1 : 0;
2178
2179 this.readPt = ReadWriteConsistencyControl.resetThreadReadPoint(rwcc);
2180
2181 List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
2182 if (additionalScanners != null) {
2183 scanners.addAll(additionalScanners);
2184 }
2185
2186 for (Map.Entry<byte[], NavigableSet<byte[]>> entry :
2187 scan.getFamilyMap().entrySet()) {
2188 Store store = stores.get(entry.getKey());
2189 scanners.add(store.getScanner(scan, entry.getValue()));
2190 }
2191 this.storeHeap = new KeyValueHeap(scanners, comparator);
2192 }
2193
2194 RegionScanner(Scan scan) throws IOException {
2195 this(scan, null);
2196 }
2197
2198
2199
2200
2201 protected void resetFilters() {
2202 if (filter != null) {
2203 filter.reset();
2204 }
2205 }
2206
2207 public synchronized boolean next(List<KeyValue> outResults, int limit)
2208 throws IOException {
2209 if (this.filterClosed) {
2210 throw new UnknownScannerException("Scanner was closed (timed out?) " +
2211 "after we renewed it. Could be caused by a very slow scanner " +
2212 "or a lengthy garbage collection");
2213 }
2214 startRegionOperation();
2215 try {
2216
2217
2218 ReadWriteConsistencyControl.setThreadReadPoint(this.readPt);
2219
2220 results.clear();
2221 boolean returnResult = nextInternal(limit);
2222
2223 outResults.addAll(results);
2224 resetFilters();
2225 if (isFilterDone()) {
2226 return false;
2227 }
2228 return returnResult;
2229 } finally {
2230 closeRegionOperation();
2231 }
2232 }
2233
2234 public synchronized boolean next(List<KeyValue> outResults)
2235 throws IOException {
2236
2237 return next(outResults, batch);
2238 }
2239
2240
2241
2242
2243 synchronized boolean isFilterDone() {
2244 return this.filter != null && this.filter.filterAllRemaining();
2245 }
2246
2247 private boolean nextInternal(int limit) throws IOException {
2248 while (true) {
2249 byte [] currentRow = peekRow();
2250 if (isStopRow(currentRow)) {
2251 if (filter != null && filter.hasFilterRow()) {
2252 filter.filterRow(results);
2253 }
2254 if (filter != null && filter.filterRow()) {
2255 results.clear();
2256 }
2257
2258 return false;
2259 } else if (filterRowKey(currentRow)) {
2260 nextRow(currentRow);
2261 } else {
2262 byte [] nextRow;
2263 do {
2264 this.storeHeap.next(results, limit - results.size());
2265 if (limit > 0 && results.size() == limit) {
2266 if (this.filter != null && filter.hasFilterRow()) throw new IncompatibleFilterException(
2267 "Filter with filterRow(List<KeyValue>) incompatible with scan with limit!");
2268 return true;
2269 }
2270 } while (Bytes.equals(currentRow, nextRow = peekRow()));
2271
2272 final boolean stopRow = isStopRow(nextRow);
2273
2274
2275
2276
2277 if (filter != null && filter.hasFilterRow()) {
2278 filter.filterRow(results);
2279 }
2280
2281 if (results.isEmpty() || filterRow()) {
2282
2283
2284
2285
2286
2287 nextRow(currentRow);
2288
2289
2290
2291
2292 if (!stopRow) continue;
2293 }
2294 return !stopRow;
2295 }
2296 }
2297 }
2298
2299 private boolean filterRow() {
2300 return filter != null
2301 && filter.filterRow();
2302 }
2303 private boolean filterRowKey(byte[] row) {
2304 return filter != null
2305 && filter.filterRowKey(row, 0, row.length);
2306 }
2307
2308 protected void nextRow(byte [] currentRow) throws IOException {
2309 while (Bytes.equals(currentRow, peekRow())) {
2310 this.storeHeap.next(MOCKED_LIST);
2311 }
2312 results.clear();
2313 resetFilters();
2314 }
2315
2316 private byte[] peekRow() {
2317 KeyValue kv = this.storeHeap.peek();
2318 return kv == null ? null : kv.getRow();
2319 }
2320
2321 private boolean isStopRow(byte [] currentRow) {
2322 return currentRow == null ||
2323 (stopRow != null &&
2324 comparator.compareRows(stopRow, 0, stopRow.length,
2325 currentRow, 0, currentRow.length) <= isScan);
2326 }
2327
2328 public synchronized void close() {
2329 if (storeHeap != null) {
2330 storeHeap.close();
2331 storeHeap = null;
2332 }
2333 this.filterClosed = true;
2334 }
2335 }
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359 public static HRegion newHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
2360 HRegionInfo regionInfo, FlushRequester flushListener) {
2361 try {
2362 @SuppressWarnings("unchecked")
2363 Class<? extends HRegion> regionClass =
2364 (Class<? extends HRegion>) conf.getClass(HConstants.REGION_IMPL, HRegion.class);
2365
2366 Constructor<? extends HRegion> c =
2367 regionClass.getConstructor(Path.class, HLog.class, FileSystem.class,
2368 Configuration.class, HRegionInfo.class, FlushRequester.class);
2369
2370 return c.newInstance(tableDir, log, fs, conf, regionInfo, flushListener);
2371 } catch (Throwable e) {
2372
2373 throw new IllegalStateException("Could not instantiate a region instance.", e);
2374 }
2375 }
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390 public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
2391 final Configuration conf)
2392 throws IOException {
2393 Path tableDir =
2394 HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName());
2395 Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
2396 FileSystem fs = FileSystem.get(conf);
2397 fs.mkdirs(regionDir);
2398 HRegion region = HRegion.newHRegion(tableDir,
2399 new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
2400 new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf, null),
2401 fs, conf, info, null);
2402 region.initialize();
2403 return region;
2404 }
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419 public static HRegion openHRegion(final HRegionInfo info, final Path rootDir,
2420 final HLog log, final Configuration conf)
2421 throws IOException {
2422 if (LOG.isDebugEnabled()) {
2423 LOG.debug("Opening region: " + info);
2424 }
2425 if (info == null) {
2426 throw new NullPointerException("Passed region info is null");
2427 }
2428 HRegion r = HRegion.newHRegion(
2429 HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName()),
2430 log, FileSystem.get(conf), conf, info, null);
2431 long seqid = r.initialize();
2432
2433 if (log != null) log.setSequenceNumber(seqid);
2434 return r;
2435 }
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447 public static void addRegionToMETA(HRegion meta, HRegion r)
2448 throws IOException {
2449 meta.checkResources();
2450
2451 byte[] row = r.getRegionName();
2452 Integer lid = meta.obtainRowLock(row);
2453 try {
2454 final List<KeyValue> edits = new ArrayList<KeyValue>(1);
2455 edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
2456 HConstants.REGIONINFO_QUALIFIER,
2457 EnvironmentEdgeManager.currentTimeMillis(),
2458 Writables.getBytes(r.getRegionInfo())));
2459 meta.put(HConstants.CATALOG_FAMILY, edits);
2460 } finally {
2461 meta.releaseRowLock(lid);
2462 }
2463 }
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474 public static void removeRegionFromMETA(final HRegionInterface srvr,
2475 final byte [] metaRegionName, final byte [] regionName)
2476 throws IOException {
2477 Delete delete = new Delete(regionName);
2478 srvr.delete(metaRegionName, delete);
2479 }
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489 public static void offlineRegionInMETA(final HRegionInterface srvr,
2490 final byte [] metaRegionName, final HRegionInfo info)
2491 throws IOException {
2492
2493
2494
2495 byte [] row = info.getRegionName();
2496 Put put = new Put(row);
2497 info.setOffline(true);
2498 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
2499 Writables.getBytes(info));
2500 srvr.put(metaRegionName, put);
2501 cleanRegionInMETA(srvr, metaRegionName, info);
2502 }
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512 public static void cleanRegionInMETA(final HRegionInterface srvr,
2513 final byte [] metaRegionName, final HRegionInfo info)
2514 throws IOException {
2515 Delete del = new Delete(info.getRegionName());
2516 del.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
2517 del.deleteColumns(HConstants.CATALOG_FAMILY,
2518 HConstants.STARTCODE_QUALIFIER);
2519 srvr.delete(metaRegionName, del);
2520 }
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530 public static void deleteRegion(FileSystem fs, Path rootdir, HRegionInfo info)
2531 throws IOException {
2532 deleteRegion(fs, HRegion.getRegionDir(rootdir, info));
2533 }
2534
2535 private static void deleteRegion(FileSystem fs, Path regiondir)
2536 throws IOException {
2537 if (LOG.isDebugEnabled()) {
2538 LOG.debug("DELETING region " + regiondir.toString());
2539 }
2540 if (!fs.delete(regiondir, true)) {
2541 LOG.warn("Failed delete of " + regiondir);
2542 }
2543 }
2544
2545
2546
2547
2548
2549
2550
2551
2552 public static Path getRegionDir(final Path rootdir, final HRegionInfo info) {
2553 return new Path(
2554 HTableDescriptor.getTableDir(rootdir, info.getTableDesc().getName()),
2555 info.getEncodedName());
2556 }
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566 public static boolean rowIsInRange(HRegionInfo info, final byte [] row) {
2567 return ((info.getStartKey().length == 0) ||
2568 (Bytes.compareTo(info.getStartKey(), row) <= 0)) &&
2569 ((info.getEndKey().length == 0) ||
2570 (Bytes.compareTo(info.getEndKey(), row) > 0));
2571 }
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582 public static void makeColumnFamilyDirs(FileSystem fs, Path tabledir,
2583 final HRegionInfo hri, byte [] colFamily)
2584 throws IOException {
2585 Path dir = Store.getStoreHomedir(tabledir, hri.getEncodedName(), colFamily);
2586 if (!fs.mkdirs(dir)) {
2587 LOG.warn("Failed to create " + dir);
2588 }
2589 }
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599 public static HRegion mergeAdjacent(final HRegion srcA, final HRegion srcB)
2600 throws IOException {
2601 HRegion a = srcA;
2602 HRegion b = srcB;
2603
2604
2605
2606 if (srcA.getStartKey() == null) {
2607 if (srcB.getStartKey() == null) {
2608 throw new IOException("Cannot merge two regions with null start key");
2609 }
2610
2611 } else if ((srcB.getStartKey() == null) ||
2612 (Bytes.compareTo(srcA.getStartKey(), srcB.getStartKey()) > 0)) {
2613 a = srcB;
2614 b = srcA;
2615 }
2616
2617 if (!(Bytes.compareTo(a.getEndKey(), b.getStartKey()) == 0)) {
2618 throw new IOException("Cannot merge non-adjacent regions");
2619 }
2620 return merge(a, b);
2621 }
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631 public static HRegion merge(HRegion a, HRegion b) throws IOException {
2632 if (!a.getRegionInfo().getTableDesc().getNameAsString().equals(
2633 b.getRegionInfo().getTableDesc().getNameAsString())) {
2634 throw new IOException("Regions do not belong to the same table");
2635 }
2636
2637 FileSystem fs = a.getFilesystem();
2638
2639
2640
2641 a.flushcache();
2642 b.flushcache();
2643
2644
2645
2646 a.compactStores(true);
2647 if (LOG.isDebugEnabled()) {
2648 LOG.debug("Files for region: " + a);
2649 listPaths(fs, a.getRegionDir());
2650 }
2651 b.compactStores(true);
2652 if (LOG.isDebugEnabled()) {
2653 LOG.debug("Files for region: " + b);
2654 listPaths(fs, b.getRegionDir());
2655 }
2656
2657 Configuration conf = a.getConf();
2658 HTableDescriptor tabledesc = a.getTableDesc();
2659 HLog log = a.getLog();
2660 Path tableDir = a.getTableDir();
2661
2662
2663 final byte[] startKey =
2664 (a.comparator.matchingRows(a.getStartKey(), 0, a.getStartKey().length,
2665 HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length)
2666 || b.comparator.matchingRows(b.getStartKey(), 0,
2667 b.getStartKey().length, HConstants.EMPTY_BYTE_ARRAY, 0,
2668 HConstants.EMPTY_BYTE_ARRAY.length))
2669 ? HConstants.EMPTY_BYTE_ARRAY
2670 : (a.comparator.compareRows(a.getStartKey(), 0, a.getStartKey().length,
2671 b.getStartKey(), 0, b.getStartKey().length) <= 0
2672 ? a.getStartKey()
2673 : b.getStartKey());
2674 final byte[] endKey =
2675 (a.comparator.matchingRows(a.getEndKey(), 0, a.getEndKey().length,
2676 HConstants.EMPTY_BYTE_ARRAY, 0, HConstants.EMPTY_BYTE_ARRAY.length)
2677 || a.comparator.matchingRows(b.getEndKey(), 0, b.getEndKey().length,
2678 HConstants.EMPTY_BYTE_ARRAY, 0,
2679 HConstants.EMPTY_BYTE_ARRAY.length))
2680 ? HConstants.EMPTY_BYTE_ARRAY
2681 : (a.comparator.compareRows(a.getEndKey(), 0, a.getEndKey().length,
2682 b.getEndKey(), 0, b.getEndKey().length) <= 0
2683 ? b.getEndKey()
2684 : a.getEndKey());
2685
2686 HRegionInfo newRegionInfo = new HRegionInfo(tabledesc, startKey, endKey);
2687 LOG.info("Creating new region " + newRegionInfo.toString());
2688 String encodedName = newRegionInfo.getEncodedName();
2689 Path newRegionDir = HRegion.getRegionDir(a.getTableDir(), encodedName);
2690 if(fs.exists(newRegionDir)) {
2691 throw new IOException("Cannot merge; target file collision at " +
2692 newRegionDir);
2693 }
2694 fs.mkdirs(newRegionDir);
2695
2696 LOG.info("starting merge of regions: " + a + " and " + b +
2697 " into new region " + newRegionInfo.toString() +
2698 " with start key <" + Bytes.toString(startKey) + "> and end key <" +
2699 Bytes.toString(endKey) + ">");
2700
2701
2702 Map<byte [], List<StoreFile>> byFamily =
2703 new TreeMap<byte [], List<StoreFile>>(Bytes.BYTES_COMPARATOR);
2704 byFamily = filesByFamily(byFamily, a.close());
2705 byFamily = filesByFamily(byFamily, b.close());
2706 for (Map.Entry<byte [], List<StoreFile>> es : byFamily.entrySet()) {
2707 byte [] colFamily = es.getKey();
2708 makeColumnFamilyDirs(fs, tableDir, newRegionInfo, colFamily);
2709
2710
2711 List<StoreFile> srcFiles = es.getValue();
2712 if (srcFiles.size() == 2) {
2713 long seqA = srcFiles.get(0).getMaxSequenceId();
2714 long seqB = srcFiles.get(1).getMaxSequenceId();
2715 if (seqA == seqB) {
2716
2717
2718
2719 throw new IOException("Files have same sequenceid: " + seqA);
2720 }
2721 }
2722 for (StoreFile hsf: srcFiles) {
2723 StoreFile.rename(fs, hsf.getPath(),
2724 StoreFile.getUniqueFile(fs, Store.getStoreHomedir(tableDir,
2725 newRegionInfo.getEncodedName(), colFamily)));
2726 }
2727 }
2728 if (LOG.isDebugEnabled()) {
2729 LOG.debug("Files for new region");
2730 listPaths(fs, newRegionDir);
2731 }
2732 HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null);
2733 dstRegion.initialize();
2734 dstRegion.compactStores();
2735 if (LOG.isDebugEnabled()) {
2736 LOG.debug("Files for new region");
2737 listPaths(fs, dstRegion.getRegionDir());
2738 }
2739 deleteRegion(fs, a.getRegionDir());
2740 deleteRegion(fs, b.getRegionDir());
2741
2742 LOG.info("merge completed. New region is " + dstRegion);
2743
2744 return dstRegion;
2745 }
2746
2747
2748
2749
2750
2751
2752
2753
2754 private static Map<byte [], List<StoreFile>> filesByFamily(
2755 Map<byte [], List<StoreFile>> byFamily, List<StoreFile> storeFiles) {
2756 for (StoreFile src: storeFiles) {
2757 byte [] family = src.getFamily();
2758 List<StoreFile> v = byFamily.get(family);
2759 if (v == null) {
2760 v = new ArrayList<StoreFile>();
2761 byFamily.put(family, v);
2762 }
2763 v.add(src);
2764 }
2765 return byFamily;
2766 }
2767
2768
2769
2770
2771
2772 boolean isMajorCompaction() throws IOException {
2773 for (Store store: this.stores.values()) {
2774 if (store.isMajorCompaction()) {
2775 return true;
2776 }
2777 }
2778 return false;
2779 }
2780
2781
2782
2783
2784
2785
2786
2787
2788 private static void listPaths(FileSystem fs, Path dir) throws IOException {
2789 if (LOG.isDebugEnabled()) {
2790 FileStatus[] stats = fs.listStatus(dir);
2791 if (stats == null || stats.length == 0) {
2792 return;
2793 }
2794 for (int i = 0; i < stats.length; i++) {
2795 String path = stats[i].getPath().toString();
2796 if (stats[i].isDir()) {
2797 LOG.debug("d " + path);
2798 listPaths(fs, stats[i].getPath());
2799 } else {
2800 LOG.debug("f " + path + " size=" + stats[i].getLen());
2801 }
2802 }
2803 }
2804 }
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816 public Result get(final Get get, final Integer lockid) throws IOException {
2817
2818 if (get.hasFamilies()) {
2819 for (byte [] family: get.familySet()) {
2820 checkFamily(family);
2821 }
2822 } else {
2823 for (byte[] family: regionInfo.getTableDesc().getFamiliesKeys()) {
2824 get.addFamily(family);
2825 }
2826 }
2827 List<KeyValue> result = get(get);
2828
2829 return new Result(result);
2830 }
2831
2832
2833
2834
2835 private List<KeyValue> get(final Get get) throws IOException {
2836 Scan scan = new Scan(get);
2837
2838 List<KeyValue> results = new ArrayList<KeyValue>();
2839
2840 InternalScanner scanner = null;
2841 try {
2842 scanner = getScanner(scan);
2843 scanner.next(results);
2844 } finally {
2845 if (scanner != null)
2846 scanner.close();
2847 }
2848 return results;
2849 }
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861 public long incrementColumnValue(byte [] row, byte [] family,
2862 byte [] qualifier, long amount, boolean writeToWAL)
2863 throws IOException {
2864 checkRow(row);
2865 boolean flush = false;
2866
2867 long result = amount;
2868 startRegionOperation();
2869 try {
2870 Integer lid = obtainRowLock(row);
2871 try {
2872 Store store = stores.get(family);
2873
2874
2875 Get get = new Get(row);
2876 get.addColumn(family, qualifier);
2877
2878 List<KeyValue> results = get(get);
2879
2880 if (!results.isEmpty()) {
2881 KeyValue kv = results.get(0);
2882 byte [] buffer = kv.getBuffer();
2883 int valueOffset = kv.getValueOffset();
2884 result += Bytes.toLong(buffer, valueOffset, Bytes.SIZEOF_LONG);
2885 }
2886
2887
2888 KeyValue newKv = new KeyValue(row, family,
2889 qualifier, EnvironmentEdgeManager.currentTimeMillis(),
2890 Bytes.toBytes(result));
2891
2892
2893 if (writeToWAL) {
2894 long now = EnvironmentEdgeManager.currentTimeMillis();
2895 WALEdit walEdit = new WALEdit();
2896 walEdit.add(newKv);
2897 this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
2898 walEdit, now);
2899 }
2900
2901
2902
2903
2904 long size = store.updateColumnValue(row, family, qualifier, result);
2905
2906 size = this.memstoreSize.addAndGet(size);
2907 flush = isFlushSize(size);
2908 } finally {
2909 releaseRowLock(lid);
2910 }
2911 } finally {
2912 closeRegionOperation();
2913 }
2914
2915 if (flush) {
2916
2917 requestFlush();
2918 }
2919
2920 return result;
2921 }
2922
2923
2924
2925
2926
2927
2928 private void checkFamily(final byte [] family)
2929 throws NoSuchColumnFamilyException {
2930 if(!regionInfo.getTableDesc().hasFamily(family)) {
2931 throw new NoSuchColumnFamilyException("Column family " +
2932 Bytes.toString(family) + " does not exist in region " + this
2933 + " in table " + regionInfo.getTableDesc());
2934 }
2935 }
2936
2937 public static final long FIXED_OVERHEAD = ClassSize.align(
2938 (4 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN +
2939 (18 * ClassSize.REFERENCE) + ClassSize.OBJECT + Bytes.SIZEOF_INT);
2940
2941 public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
2942 ClassSize.OBJECT + (2 * ClassSize.ATOMIC_BOOLEAN) +
2943 ClassSize.ATOMIC_LONG + ClassSize.ATOMIC_INTEGER +
2944
2945
2946 ClassSize.TREEMAP +
2947
2948
2949 ClassSize.TREEMAP +
2950
2951 ClassSize.CONCURRENT_SKIPLISTMAP + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY +
2952 ClassSize.align(ClassSize.OBJECT +
2953 (5 * Bytes.SIZEOF_BOOLEAN)) +
2954 (3 * ClassSize.REENTRANT_LOCK));
2955
2956 public long heapSize() {
2957 long heapSize = DEEP_OVERHEAD;
2958 for(Store store : this.stores.values()) {
2959 heapSize += store.heapSize();
2960 }
2961 return heapSize;
2962 }
2963
2964
2965
2966
2967
2968 private static void printUsageAndExit(final String message) {
2969 if (message != null && message.length() > 0) System.out.println(message);
2970 System.out.println("Usage: HRegion CATLALOG_TABLE_DIR [major_compact]");
2971 System.out.println("Options:");
2972 System.out.println(" major_compact Pass this option to major compact " +
2973 "passed region.");
2974 System.out.println("Default outputs scan of passed region.");
2975 System.exit(1);
2976 }
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988 private static void processTable(final FileSystem fs, final Path p,
2989 final HLog log, final Configuration c,
2990 final boolean majorCompact)
2991 throws IOException {
2992 HRegion region = null;
2993 String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME);
2994 String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
2995
2996 if (p.getName().startsWith(rootStr)) {
2997 region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, null);
2998 } else if (p.getName().startsWith(metaStr)) {
2999 region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO,
3000 null);
3001 } else {
3002 throw new IOException("Not a known catalog table: " + p.toString());
3003 }
3004 try {
3005 region.initialize();
3006 if (majorCompact) {
3007 region.compactStores(true);
3008 } else {
3009
3010 Scan scan = new Scan();
3011
3012 InternalScanner scanner = region.getScanner(scan);
3013 try {
3014 List<KeyValue> kvs = new ArrayList<KeyValue>();
3015 boolean done = false;
3016 do {
3017 kvs.clear();
3018 done = scanner.next(kvs);
3019 if (kvs.size() > 0) LOG.info(kvs);
3020 } while (done);
3021 } finally {
3022 scanner.close();
3023 }
3024
3025 }
3026 } finally {
3027 region.close();
3028 }
3029 }
3030
3031
3032
3033
3034
3035
3036 public boolean shouldSplit(boolean b) {
3037 boolean old = this.splitRequest;
3038 this.splitRequest = b;
3039 return old;
3040 }
3041
3042
3043
3044
3045
3046
3047 public boolean hasTooManyStoreFiles() {
3048 for(Store store : stores.values()) {
3049 if(store.hasTooManyStoreFiles()) {
3050 return true;
3051 }
3052 }
3053 return false;
3054 }
3055
3056
3057
3058
3059
3060
3061
3062
3063 private void startRegionOperation() throws NotServingRegionException {
3064 if (this.closing.get()) {
3065 throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
3066 " is closing");
3067 }
3068 lock.readLock().lock();
3069 if (this.closed.get()) {
3070 lock.readLock().unlock();
3071 throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
3072 " is closed");
3073 }
3074 }
3075
3076
3077
3078
3079
3080 private void closeRegionOperation(){
3081 lock.readLock().unlock();
3082 }
3083
3084
3085
3086
3087 private static final List<KeyValue> MOCKED_LIST = new AbstractList<KeyValue>() {
3088
3089 @Override
3090 public void add(int index, KeyValue element) {
3091
3092 }
3093
3094 @Override
3095 public boolean addAll(int index, Collection<? extends KeyValue> c) {
3096 return false;
3097 }
3098
3099 @Override
3100 public KeyValue get(int index) {
3101 throw new UnsupportedOperationException();
3102 }
3103
3104 @Override
3105 public int size() {
3106 return 0;
3107 }
3108 };
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121 public static void main(String[] args) throws IOException {
3122 if (args.length < 1) {
3123 printUsageAndExit(null);
3124 }
3125 boolean majorCompact = false;
3126 if (args.length > 1) {
3127 if (!args[1].toLowerCase().startsWith("major")) {
3128 printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
3129 }
3130 majorCompact = true;
3131 }
3132 final Path tableDir = new Path(args[0]);
3133 final Configuration c = HBaseConfiguration.create();
3134 final FileSystem fs = FileSystem.get(c);
3135 final Path logdir = new Path(c.get("hbase.tmp.dir"),
3136 "hlog" + tableDir.getName()
3137 + EnvironmentEdgeManager.currentTimeMillis());
3138 final Path oldLogDir = new Path(c.get("hbase.tmp.dir"),
3139 HConstants.HREGION_OLDLOGDIR_NAME);
3140 final HLog log = new HLog(fs, logdir, oldLogDir, c, null);
3141 try {
3142 processTable(fs, tableDir, log, c, majorCompact);
3143 } finally {
3144 log.close();
3145 BlockCache bc = StoreFile.getBlockCache(c);
3146 if (bc != null) bc.shutdown();
3147 }
3148 }
3149 }