1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.TableNotFoundException;
28 import org.apache.hadoop.hbase.client.Delete;
29 import org.apache.hadoop.hbase.client.HTableInterface;
30 import org.apache.hadoop.hbase.client.HTablePool;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.regionserver.wal.HLog;
33 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
34 import org.apache.hadoop.hbase.util.Bytes;
35 import org.apache.hadoop.hbase.Stoppable;
36
37 import java.io.IOException;
38 import java.util.ArrayList;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.TreeMap;
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57 public class ReplicationSink {
58
59 private static final Log LOG = LogFactory.getLog(ReplicationSink.class);
60
61 public static final String REPLICATION_LOG_DIR = ".replogs";
62 private final Configuration conf;
63
64 private final HTablePool pool;
65
66 private final Stoppable stopper;
67 private final ReplicationSinkMetrics metrics;
68
69
70
71
72
73
74
75
76 public ReplicationSink(Configuration conf, Stoppable stopper)
77 throws IOException {
78 this.conf = conf;
79 this.pool = new HTablePool(this.conf,
80 conf.getInt("replication.sink.htablepool.capacity", 10));
81 this.stopper = stopper;
82 this.metrics = new ReplicationSinkMetrics();
83 }
84
85
86
87
88
89
90
91
92 public void replicateEntries(HLog.Entry[] entries)
93 throws IOException {
94 if (entries.length == 0) {
95 return;
96 }
97
98
99 try {
100 long totalReplicated = 0;
101
102
103 Map<byte[], List<Put>> puts = new TreeMap<byte[], List<Put>>(Bytes.BYTES_COMPARATOR);
104 for (HLog.Entry entry : entries) {
105 WALEdit edit = entry.getEdit();
106 List<KeyValue> kvs = edit.getKeyValues();
107 if (kvs.get(0).isDelete()) {
108 Delete delete = new Delete(kvs.get(0).getRow(),
109 kvs.get(0).getTimestamp(), null);
110 delete.setClusterId(entry.getKey().getClusterId());
111 for (KeyValue kv : kvs) {
112 if (kv.isDeleteFamily()) {
113 delete.deleteFamily(kv.getFamily());
114 } else if (!kv.isEmptyColumn()) {
115 delete.deleteColumn(kv.getFamily(),
116 kv.getQualifier());
117 }
118 }
119 delete(entry.getKey().getTablename(), delete);
120 } else {
121 byte[] table = entry.getKey().getTablename();
122 List<Put> tableList = puts.get(table);
123 if (tableList == null) {
124 tableList = new ArrayList<Put>();
125 puts.put(table, tableList);
126 }
127
128 byte[] lastKey = kvs.get(0).getRow();
129 Put put = new Put(kvs.get(0).getRow(),
130 kvs.get(0).getTimestamp());
131 put.setClusterId(entry.getKey().getClusterId());
132 for (KeyValue kv : kvs) {
133 if (!Bytes.equals(lastKey, kv.getRow())) {
134 tableList.add(put);
135 put = new Put(kv.getRow(), kv.getTimestamp());
136 put.setClusterId(entry.getKey().getClusterId());
137 }
138 put.add(kv.getFamily(), kv.getQualifier(), kv.getValue());
139 lastKey = kv.getRow();
140 }
141 tableList.add(put);
142 }
143 totalReplicated++;
144 }
145 for(byte [] table : puts.keySet()) {
146 put(table, puts.get(table));
147 }
148 this.metrics.setAgeOfLastAppliedOp(
149 entries[entries.length-1].getKey().getWriteTime());
150 this.metrics.appliedBatchesRate.inc(1);
151 LOG.info("Total replicated: " + totalReplicated);
152 } catch (IOException ex) {
153 LOG.error("Unable to accept edit because:", ex);
154 throw ex;
155 }
156 }
157
158
159
160
161
162
163
164 private void put(byte[] tableName, List<Put> puts) throws IOException {
165 if (puts.isEmpty()) {
166 return;
167 }
168 HTableInterface table = null;
169 try {
170 table = this.pool.getTable(tableName);
171 table.put(puts);
172 this.metrics.appliedOpsRate.inc(puts.size());
173 } finally {
174 if (table != null) {
175 this.pool.putTable(table);
176 }
177 }
178 }
179
180
181
182
183
184
185
186 private void delete(byte[] tableName, Delete delete) throws IOException {
187 HTableInterface table = null;
188 try {
189 table = this.pool.getTable(tableName);
190 table.delete(delete);
191 this.metrics.appliedOpsRate.inc(1);
192 } finally {
193 if (table != null) {
194 this.pool.putTable(table);
195 }
196 }
197 }
198 }