[120] | 1 | /** |
---|
| 2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
| 3 | * or more contributor license agreements. See the NOTICE file |
---|
| 4 | * distributed with this work for additional information |
---|
| 5 | * regarding copyright ownership. The ASF licenses this file |
---|
| 6 | * to you under the Apache License, Version 2.0 (the |
---|
| 7 | * "License"); you may not use this file except in compliance |
---|
| 8 | * with the License. You may obtain a copy of the License at |
---|
| 9 | * |
---|
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | * |
---|
| 12 | * Unless required by applicable law or agreed to in writing, software |
---|
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | * See the License for the specific language governing permissions and |
---|
| 16 | * limitations under the License. |
---|
| 17 | */ |
---|
| 18 | package org.apache.hadoop.mapred.lib; |
---|
| 19 | |
---|
| 20 | import org.apache.hadoop.io.Text; |
---|
| 21 | import org.apache.hadoop.mapred.JobConf; |
---|
| 22 | import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner; |
---|
| 23 | |
---|
| 24 | import junit.framework.TestCase; |
---|
| 25 | |
---|
| 26 | public class TestKeyFieldBasedPartitioner extends TestCase { |
---|
| 27 | |
---|
| 28 | /** |
---|
| 29 | * Test is key-field-based partitioned works with empty key. |
---|
| 30 | */ |
---|
| 31 | public void testEmptyKey() throws Exception { |
---|
| 32 | int numReducers = 10; |
---|
| 33 | KeyFieldBasedPartitioner<Text, Text> kfbp = |
---|
| 34 | new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 35 | JobConf conf = new JobConf(); |
---|
| 36 | conf.setInt("num.key.fields.for.partition", 10); |
---|
| 37 | kfbp.configure(conf); |
---|
| 38 | assertEquals("Empty key should map to 0th partition", |
---|
| 39 | 0, kfbp.getPartition(new Text(), new Text(), numReducers)); |
---|
| 40 | |
---|
| 41 | // check if the hashcode is correct when no keyspec is specified |
---|
| 42 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 43 | conf = new JobConf(); |
---|
| 44 | kfbp.configure(conf); |
---|
| 45 | String input = "abc\tdef\txyz"; |
---|
| 46 | int hashCode = input.hashCode(); |
---|
| 47 | int expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 48 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 49 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 50 | |
---|
| 51 | // check if the hashcode is correct with specified keyspec |
---|
| 52 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 53 | conf = new JobConf(); |
---|
| 54 | conf.set("mapred.text.key.partitioner.options", "-k2,2"); |
---|
| 55 | kfbp.configure(conf); |
---|
| 56 | String expectedOutput = "def"; |
---|
| 57 | byte[] eBytes = expectedOutput.getBytes(); |
---|
| 58 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); |
---|
| 59 | expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 60 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 61 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 62 | |
---|
| 63 | // test with invalid end index in keyspecs |
---|
| 64 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 65 | conf = new JobConf(); |
---|
| 66 | conf.set("mapred.text.key.partitioner.options", "-k2,5"); |
---|
| 67 | kfbp.configure(conf); |
---|
| 68 | expectedOutput = "def\txyz"; |
---|
| 69 | eBytes = expectedOutput.getBytes(); |
---|
| 70 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); |
---|
| 71 | expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 72 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 73 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 74 | |
---|
| 75 | // test with 0 end index in keyspecs |
---|
| 76 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 77 | conf = new JobConf(); |
---|
| 78 | conf.set("mapred.text.key.partitioner.options", "-k2"); |
---|
| 79 | kfbp.configure(conf); |
---|
| 80 | expectedOutput = "def\txyz"; |
---|
| 81 | eBytes = expectedOutput.getBytes(); |
---|
| 82 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); |
---|
| 83 | expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 84 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 85 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 86 | |
---|
| 87 | // test with invalid keyspecs |
---|
| 88 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 89 | conf = new JobConf(); |
---|
| 90 | conf.set("mapred.text.key.partitioner.options", "-k10"); |
---|
| 91 | kfbp.configure(conf); |
---|
| 92 | assertEquals("Partitioner doesnt work as expected", 0, |
---|
| 93 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 94 | |
---|
| 95 | // test with multiple keyspecs |
---|
| 96 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 97 | conf = new JobConf(); |
---|
| 98 | conf.set("mapred.text.key.partitioner.options", "-k2,2 -k4,4"); |
---|
| 99 | kfbp.configure(conf); |
---|
| 100 | input = "abc\tdef\tpqr\txyz"; |
---|
| 101 | expectedOutput = "def"; |
---|
| 102 | eBytes = expectedOutput.getBytes(); |
---|
| 103 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); |
---|
| 104 | expectedOutput = "xyz"; |
---|
| 105 | eBytes = expectedOutput.getBytes(); |
---|
| 106 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode); |
---|
| 107 | expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 108 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 109 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 110 | |
---|
| 111 | // test with invalid start index in keyspecs |
---|
| 112 | kfbp = new KeyFieldBasedPartitioner<Text, Text>(); |
---|
| 113 | conf = new JobConf(); |
---|
| 114 | conf.set("mapred.text.key.partitioner.options", "-k2,2 -k30,21 -k4,4 -k5"); |
---|
| 115 | kfbp.configure(conf); |
---|
| 116 | expectedOutput = "def"; |
---|
| 117 | eBytes = expectedOutput.getBytes(); |
---|
| 118 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, 0); |
---|
| 119 | expectedOutput = "xyz"; |
---|
| 120 | eBytes = expectedOutput.getBytes(); |
---|
| 121 | hashCode = kfbp.hashCode(eBytes, 0, eBytes.length - 1, hashCode); |
---|
| 122 | expectedPartition = kfbp.getPartition(hashCode, numReducers); |
---|
| 123 | assertEquals("Partitioner doesnt work as expected", expectedPartition, |
---|
| 124 | kfbp.getPartition(new Text(input), new Text(), numReducers)); |
---|
| 125 | } |
---|
| 126 | } |
---|