1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.junit.Assert.*;
23  
24  import java.util.List;
25  import java.util.Map;
26  import java.util.Random;
27  import java.util.concurrent.atomic.AtomicInteger;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.MultithreadedTestUtil;
31  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
32  import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
33  import org.junit.Test;
34  
35  import com.google.common.collect.Iterables;
36  import com.google.common.collect.Lists;
37  import com.google.common.collect.Maps;
38  import com.google.common.primitives.Ints;
39  
40  public class TestMemStoreLAB {
41  
42    /**
43     * Test a bunch of random allocations
44     */
45    @Test
46    public void testLABRandomAllocation() {
47      Random rand = new Random();
48      MemStoreLAB mslab = new MemStoreLAB();
49      int expectedOff = 0;
50      byte[] lastBuffer = null;
51      // 100K iterations by 0-1K alloc -> 50MB expected
52      // should be reasonable for unit test and also cover wraparound
53      // behavior
54      for (int i = 0; i < 100000; i++) {
55        int size = rand.nextInt(1000);
56        Allocation alloc = mslab.allocateBytes(size);
57        
58        if (alloc.getData() != lastBuffer) {
59          expectedOff = 0;
60          lastBuffer = alloc.getData();
61        }
62        assertEquals(expectedOff, alloc.getOffset());
63        assertTrue("Allocation " + alloc + " overruns buffer",
64            alloc.getOffset() + size <= alloc.getData().length);
65        expectedOff += size;
66      }
67    }
68  
69    @Test
70    public void testLABLargeAllocation() {
71      MemStoreLAB mslab = new MemStoreLAB();
72      Allocation alloc = mslab.allocateBytes(2*1024*1024);
73      assertNull("2MB allocation shouldn't be satisfied by LAB.",
74        alloc);
75    } 
76  
77    /**
78     * Test allocation from lots of threads, making sure the results don't
79     * overlap in any way
80     */
81    @Test
82    public void testLABThreading() throws Exception {
83      Configuration conf = new Configuration();
84      MultithreadedTestUtil.TestContext ctx =
85        new MultithreadedTestUtil.TestContext(conf);
86      
87      final AtomicInteger totalAllocated = new AtomicInteger();
88      
89      final MemStoreLAB mslab = new MemStoreLAB();
90      List<List<AllocRecord>> allocations = Lists.newArrayList();
91      
92      for (int i = 0; i < 10; i++) {
93        final List<AllocRecord> allocsByThisThread = Lists.newLinkedList();
94        allocations.add(allocsByThisThread);
95        
96        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
97          private Random r = new Random();
98          @Override
99          public void doAnAction() throws Exception {
100           int size = r.nextInt(1000);
101           Allocation alloc = mslab.allocateBytes(size);
102           totalAllocated.addAndGet(size);
103           allocsByThisThread.add(new AllocRecord(alloc, size));
104         }
105       };
106       ctx.addThread(t);
107     }
108     
109     ctx.startThreads();
110     while (totalAllocated.get() < 50*1024*1024 && ctx.shouldRun()) {
111       Thread.sleep(10);
112     }
113     ctx.stop();
114     
115     // Partition the allocations by the actual byte[] they point into,
116     // make sure offsets are unique for each chunk
117     Map<byte[], Map<Integer, AllocRecord>> mapsByChunk =
118       Maps.newHashMap();
119     
120     int sizeCounted = 0;
121     for (AllocRecord rec : Iterables.concat(allocations)) {
122       sizeCounted += rec.size;
123       if (rec.size == 0) continue;
124       
125       Map<Integer, AllocRecord> mapForThisByteArray =
126         mapsByChunk.get(rec.alloc.getData());
127       if (mapForThisByteArray == null) {
128         mapForThisByteArray = Maps.newTreeMap();
129         mapsByChunk.put(rec.alloc.getData(), mapForThisByteArray);
130       }
131       AllocRecord oldVal = mapForThisByteArray.put(rec.alloc.getOffset(), rec);
132       assertNull("Already had an entry " + oldVal + " for allocation " + rec,
133           oldVal);
134     }
135     assertEquals("Sanity check test", sizeCounted, totalAllocated.get());
136     
137     // Now check each byte array to make sure allocations don't overlap
138     for (Map<Integer, AllocRecord> allocsInChunk : mapsByChunk.values()) {
139       int expectedOff = 0;
140       for (AllocRecord alloc : allocsInChunk.values()) {
141         assertEquals(expectedOff, alloc.alloc.getOffset());
142         assertTrue("Allocation " + alloc + " overruns buffer",
143             alloc.alloc.getOffset() + alloc.size <= alloc.alloc.getData().length);
144         expectedOff += alloc.size;
145       }
146     }
147 
148   }
149   
150   private static class AllocRecord implements Comparable<AllocRecord>{
151     private final Allocation alloc;
152     private final int size;
153     public AllocRecord(Allocation alloc, int size) {
154       super();
155       this.alloc = alloc;
156       this.size = size;
157     }
158 
159     @Override
160     public int compareTo(AllocRecord e) {
161       if (alloc.getData() != e.alloc.getData()) {
162         throw new RuntimeException("Can only compare within a particular array");
163       }
164       return Ints.compare(alloc.getOffset(), e.alloc.getOffset());
165     }
166     
167     @Override
168     public String toString() {
169       return "AllocRecord(alloc=" + alloc + ", size=" + size + ")";
170     }
171     
172   }
173 }