1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import static org.junit.Assert.*;
23
24 import java.io.IOException;
25 import java.util.List;
26 import java.util.ArrayList;
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.regionserver.HRegionServer;
32 import org.apache.hadoop.hbase.regionserver.HRegion;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.hadoop.hbase.util.Threads;
35 import org.apache.hadoop.hbase.util.JVMClusterUtil;
36 import org.junit.Test;
37
38
39
40
41
42 public class TestGlobalMemStoreSize {
43 private final Log LOG = LogFactory.getLog(this.getClass().getName());
44 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
45
46 private static int regionServerNum =4;
47 private static int regionNum = 16;
48
49 private static int totalRegionNum = regionNum+2;
50
51 private HBaseTestingUtility TEST_UTIL;
52 private MiniHBaseCluster cluster;
53
54
55
56
57
58
59 @Test
60 public void testGlobalMemStore() throws Exception {
61
62 LOG.info("Starting cluster");
63 Configuration conf = HBaseConfiguration.create();
64 conf.setInt("hbase.master.assignment.timeoutmonitor.period", 2000);
65 conf.setInt("hbase.master.assignment.timeoutmonitor.timeout", 5000);
66 TEST_UTIL = new HBaseTestingUtility(conf);
67 TEST_UTIL.startMiniCluster(1, regionServerNum);
68 cluster = TEST_UTIL.getHBaseCluster();
69 LOG.info("Waiting for active/ready master");
70 cluster.waitForActiveAndReadyMaster();
71
72
73 byte [] table = Bytes.toBytes("TestGlobalMemStoreSize");
74 byte [] family = Bytes.toBytes("family");
75 LOG.info("Creating table with " + regionNum + " regions");
76 HTable ht = TEST_UTIL.createTable(table, family);
77 int numRegions = TEST_UTIL.createMultiRegions(conf, ht, family,
78 regionNum);
79 assertEquals(regionNum,numRegions);
80 waitForAllRegionsAssigned();
81
82 for (HRegionServer server : getOnlineRegionServers()) {
83 long globalMemStoreSize = 0;
84 for(HRegionInfo regionInfo : server.getOnlineRegions()) {
85 globalMemStoreSize +=
86 server.getFromOnlineRegions(regionInfo.getEncodedName()).
87 getMemstoreSize().get();
88 }
89 assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(),
90 globalMemStoreSize);
91 }
92
93
94 int i = 0;
95 for (HRegionServer server : getOnlineRegionServers()) {
96 LOG.info("Starting flushes on " + server.getServerName() +
97 ", size=" + server.getRegionServerAccounting().getGlobalMemstoreSize());
98
99 for (HRegionInfo regionInfo : server.getOnlineRegions()) {
100 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
101 flush(r, server);
102 }
103 LOG.info("Post flush on " + server.getServerName());
104 long now = System.currentTimeMillis();
105 long timeout = now + 1000;
106 while(server.getRegionServerAccounting().getGlobalMemstoreSize() != 0 &&
107 timeout < System.currentTimeMillis()) {
108 Threads.sleep(10);
109 }
110 long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
111 if (size > 0) {
112
113
114 for (HRegionInfo regionInfo : server.getOnlineRegions()) {
115 HRegion r = server.getFromOnlineRegions(regionInfo.getEncodedName());
116 long l = r.getMemstoreSize().longValue();
117 if (l > 0) {
118
119
120 assertTrue(regionInfo.isMetaRegion());
121 LOG.info(r.toString() + " " + l + ", reflushing");
122 r.flushcache();
123 }
124 }
125 }
126 size = server.getRegionServerAccounting().getGlobalMemstoreSize();
127 assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
128 }
129
130 TEST_UTIL.shutdownMiniCluster();
131 }
132
133
134
135
136
137
138
139 private void flush(final HRegion r, final HRegionServer server)
140 throws IOException {
141 LOG.info("Flush " + r.toString() + " on " + server.getServerName() +
142 ", " + r.flushcache() + ", size=" +
143 server.getRegionServerAccounting().getGlobalMemstoreSize());
144 }
145
146
147 private int getRegionCount() throws IOException {
148 int total = 0;
149 for (HRegionServer server : getOnlineRegionServers()) {
150 total += server.getOnlineRegions().size();
151 }
152 return total;
153 }
154
155 private List<HRegionServer> getOnlineRegionServers() {
156 List<HRegionServer> list = new ArrayList<HRegionServer>();
157 for (JVMClusterUtil.RegionServerThread rst :
158 cluster.getRegionServerThreads()) {
159 if (rst.getRegionServer().isOnline()) {
160 list.add(rst.getRegionServer());
161 }
162 }
163 return list;
164 }
165
166
167
168
169 private void waitForAllRegionsAssigned() throws IOException {
170 while (getRegionCount() < totalRegionNum) {
171 LOG.debug("Waiting for there to be "+totalRegionNum+" regions, but there are " + getRegionCount() + " right now.");
172 try {
173 Thread.sleep(100);
174 } catch (InterruptedException e) {}
175 }
176 }
177 }