source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/mapred/lib/TestKeyFieldHelper.java @ 120

Last change on this file since 120 was 120, checked in by (none), 14 years ago

Added the mail files for the Hadoop JUNit Project
  • Property svn:executable set to *
File size: 16.0 KB
Line 
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18package org.apache.hadoop.mapred.lib;
19
20import org.apache.commons.logging.Log;
21import org.apache.commons.logging.LogFactory;
22
23import junit.framework.TestCase;
24
25public class TestKeyFieldHelper extends TestCase {
26  private static final Log LOG = LogFactory.getLog(TestKeyFieldHelper.class);
27  /**
28   * Test is key-field-helper's parse option.
29   */
30  public void testparseOption() throws Exception {
31    KeyFieldHelper helper = new KeyFieldHelper();
32    helper.setKeyFieldSeparator("\t");
33    String keySpecs = "-k1.2,3.4";
34    String eKeySpecs = keySpecs;
35    helper.parseOption(keySpecs);
36    String actKeySpecs = helper.keySpecs().get(0).toString();
37    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
38   
39    // test -k a.b
40    keySpecs = "-k 1.2";
41    eKeySpecs = "-k1.2,0.0";
42    helper = new KeyFieldHelper();
43    helper.parseOption(keySpecs);
44    actKeySpecs = helper.keySpecs().get(0).toString();
45    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
46   
47    keySpecs = "-nr -k1.2,3.4";
48    eKeySpecs = "-k1.2,3.4nr";
49    helper = new KeyFieldHelper();
50    helper.parseOption(keySpecs);
51    actKeySpecs = helper.keySpecs().get(0).toString();
52    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
53   
54    keySpecs = "-nr -k1.2,3.4n";
55    eKeySpecs = "-k1.2,3.4n";
56    helper = new KeyFieldHelper();
57    helper.parseOption(keySpecs);
58    actKeySpecs = helper.keySpecs().get(0).toString();
59    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
60   
61    keySpecs = "-nr -k1.2,3.4r";
62    eKeySpecs = "-k1.2,3.4r";
63    helper = new KeyFieldHelper();
64    helper.parseOption(keySpecs);
65    actKeySpecs = helper.keySpecs().get(0).toString();
66    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
67   
68    keySpecs = "-nr -k1.2,3.4 -k5.6,7.8n -k9.10,11.12r -k13.14,15.16nr";
69    //1st
70    eKeySpecs = "-k1.2,3.4nr";
71    helper = new KeyFieldHelper();
72    helper.parseOption(keySpecs);
73    actKeySpecs = helper.keySpecs().get(0).toString();
74    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
75    // 2nd
76    eKeySpecs = "-k5.6,7.8n";
77    actKeySpecs = helper.keySpecs().get(1).toString();
78    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
79    //3rd
80    eKeySpecs = "-k9.10,11.12r";
81    actKeySpecs = helper.keySpecs().get(2).toString();
82    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
83    //4th
84    eKeySpecs = "-k13.14,15.16nr";
85    actKeySpecs = helper.keySpecs().get(3).toString();
86    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
87   
88    keySpecs = "-k1.2n,3.4";
89    eKeySpecs = "-k1.2,3.4n";
90    helper = new KeyFieldHelper();
91    helper.parseOption(keySpecs);
92    actKeySpecs = helper.keySpecs().get(0).toString();
93    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
94   
95    keySpecs = "-k1.2r,3.4";
96    eKeySpecs = "-k1.2,3.4r";
97    helper = new KeyFieldHelper();
98    helper.parseOption(keySpecs);
99    actKeySpecs = helper.keySpecs().get(0).toString();
100    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
101   
102    keySpecs = "-k1.2nr,3.4";
103    eKeySpecs = "-k1.2,3.4nr";
104    helper = new KeyFieldHelper();
105    helper.parseOption(keySpecs);
106    actKeySpecs = helper.keySpecs().get(0).toString();
107    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
108   
109    keySpecs = "-k1.2,3.4n";
110    eKeySpecs = "-k1.2,3.4n";
111    helper = new KeyFieldHelper();
112    helper.parseOption(keySpecs);
113    actKeySpecs = helper.keySpecs().get(0).toString();
114    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
115   
116    keySpecs = "-k1.2,3.4r";
117    eKeySpecs = "-k1.2,3.4r";
118    helper = new KeyFieldHelper();
119    helper.parseOption(keySpecs);
120    actKeySpecs = helper.keySpecs().get(0).toString();
121    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
122   
123    keySpecs = "-k1.2,3.4nr";
124    eKeySpecs = "-k1.2,3.4nr";
125    helper = new KeyFieldHelper();
126    helper.parseOption(keySpecs);
127    actKeySpecs = helper.keySpecs().get(0).toString();
128    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
129   
130    keySpecs = "-nr -k1.2,3.4 -k5.6,7.8";
131    eKeySpecs = "-k1.2,3.4nr";
132    helper = new KeyFieldHelper();
133    helper.parseOption(keySpecs);
134    actKeySpecs = helper.keySpecs().get(0).toString();
135    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
136    eKeySpecs = "-k5.6,7.8nr";
137    actKeySpecs = helper.keySpecs().get(1).toString();
138    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
139   
140    keySpecs = "-n -k1.2,3.4 -k5.6,7.8";
141    eKeySpecs = "-k1.2,3.4n";
142    helper = new KeyFieldHelper();
143    helper.parseOption(keySpecs);
144    actKeySpecs = helper.keySpecs().get(0).toString();
145    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
146    eKeySpecs = "-k5.6,7.8n";
147    actKeySpecs = helper.keySpecs().get(1).toString();
148    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
149   
150    keySpecs = "-r -k1.2,3.4 -k5.6,7.8";
151    eKeySpecs = "-k1.2,3.4r";
152    helper = new KeyFieldHelper();
153    helper.parseOption(keySpecs);
154    actKeySpecs = helper.keySpecs().get(0).toString();
155    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
156    eKeySpecs = "-k5.6,7.8r";
157    actKeySpecs = helper.keySpecs().get(1).toString();
158    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
159   
160    keySpecs = "-k1.2,3.4n -k5.6,7.8";
161    eKeySpecs = "-k1.2,3.4n";
162    helper = new KeyFieldHelper();
163    helper.parseOption(keySpecs);
164    actKeySpecs = helper.keySpecs().get(0).toString();
165    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
166    eKeySpecs = "-k5.6,7.8";
167    actKeySpecs = helper.keySpecs().get(1).toString();
168    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
169   
170    keySpecs = "-k1.2,3.4r -k5.6,7.8";
171    eKeySpecs = "-k1.2,3.4r";
172    helper = new KeyFieldHelper();
173    helper.parseOption(keySpecs);
174    actKeySpecs = helper.keySpecs().get(0).toString();
175    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
176    eKeySpecs = "-k5.6,7.8";
177    actKeySpecs = helper.keySpecs().get(1).toString();
178    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
179   
180    keySpecs = "-k1.2,3.4nr -k5.6,7.8";
181    eKeySpecs = "-k1.2,3.4nr";
182    helper = new KeyFieldHelper();
183    helper.parseOption(keySpecs);
184    actKeySpecs = helper.keySpecs().get(0).toString();
185    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
186    eKeySpecs = "-k5.6,7.8";
187    actKeySpecs = helper.keySpecs().get(1).toString();
188    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
189   
190    keySpecs = "-n";
191    eKeySpecs = "-k1.1,0.0n";
192    helper = new KeyFieldHelper();
193    helper.parseOption(keySpecs);
194    actKeySpecs = helper.keySpecs().get(0).toString();
195    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
196   
197    keySpecs = "-r";
198    eKeySpecs = "-k1.1,0.0r";
199    helper = new KeyFieldHelper();
200    helper.parseOption(keySpecs);
201    actKeySpecs = helper.keySpecs().get(0).toString();
202    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
203   
204    keySpecs = "-nr";
205    eKeySpecs = "-k1.1,0.0nr";
206    helper = new KeyFieldHelper();
207    helper.parseOption(keySpecs);
208    actKeySpecs = helper.keySpecs().get(0).toString();
209    assertEquals("KeyFieldHelper's parsing is garbled", eKeySpecs, actKeySpecs);
210  }
211 
212  /**
213   * Test is key-field-helper's getWordLengths.
214   */
215  public void testGetWordLengths() throws Exception {
216    KeyFieldHelper helper = new KeyFieldHelper();
217    helper.setKeyFieldSeparator("\t");
218    // test getWordLengths with unspecified key-specifications
219    String input = "hi";
220    int[] result = helper.getWordLengths(input.getBytes(), 0, 2);
221    assertTrue(equals(result, new int[] {1}));
222   
223    // set the key specs
224    helper.setKeyFieldSpec(1, 2);
225   
226    // test getWordLengths with 3 words
227    input = "hi\thello there";
228    result = helper.getWordLengths(input.getBytes(), 0, input.length());
229    assertTrue(equals(result, new int[] {2, 2, 11}));
230   
231    // test getWordLengths with 4 words but with a different separator
232    helper.setKeyFieldSeparator(" ");
233    input = "hi hello\tthere you";
234    result = helper.getWordLengths(input.getBytes(), 0, input.length());
235    assertTrue(equals(result, new int[] {3, 2, 11, 3}));
236   
237    // test with non zero start index
238    input = "hi hello there you where me there";
239    //                 .....................
240    result = helper.getWordLengths(input.getBytes(), 10, 33);
241    assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 3}));
242   
243    input = "hi hello there you where me ";
244    //                 ..................
245    result = helper.getWordLengths(input.getBytes(), 10, input.length());
246    assertTrue(equals(result, new int[] {5, 4, 3, 5, 2, 0}));
247   
248    input = "";
249    result = helper.getWordLengths(input.getBytes(), 0, 0);
250    assertTrue(equals(result, new int[] {1, 0}));
251   
252    input = "  abc";
253    result = helper.getWordLengths(input.getBytes(), 0, 5);
254    assertTrue(equals(result, new int[] {3, 0, 0, 3}));
255   
256    input = "  abc";
257    result = helper.getWordLengths(input.getBytes(), 0, 2);
258    assertTrue(equals(result, new int[] {3, 0, 0, 0}));
259   
260    input = " abc ";
261    result = helper.getWordLengths(input.getBytes(), 0, 2);
262    assertTrue(equals(result, new int[] {2, 0, 1}));
263   
264    helper.setKeyFieldSeparator("abcd");
265    input = "abc";
266    result = helper.getWordLengths(input.getBytes(), 0, 3);
267    assertTrue(equals(result, new int[] {1, 3}));
268  }
269 
270  /**
271   * Test is key-field-helper's getStartOffset/getEndOffset.
272   */
273  public void testgetStartEndOffset() throws Exception {
274    KeyFieldHelper helper = new KeyFieldHelper();
275    helper.setKeyFieldSeparator("\t");
276    // test getStartOffset with -k1,2
277    helper.setKeyFieldSpec(1, 2);
278    String input = "hi\thello";
279    String expectedOutput = input;
280    testKeySpecs(input, expectedOutput, helper);
281   
282    // test getStartOffset with -k1.0,0 .. should result into start = -1
283    helper = new KeyFieldHelper();
284    helper.setKeyFieldSeparator("\t");
285    helper.parseOption("-k1.0,0");
286    testKeySpecs(input, null, helper);
287   
288    // test getStartOffset with -k1,0
289    helper = new KeyFieldHelper();
290    helper.setKeyFieldSeparator("\t");
291    helper.parseOption("-k1,0");
292    expectedOutput = input;
293    testKeySpecs(input, expectedOutput, helper);
294   
295    // test getStartOffset with -k1.2,0
296    helper = new KeyFieldHelper();
297    helper.setKeyFieldSeparator("\t");
298    helper.parseOption("-k1.2,0");
299    expectedOutput = "i\thello";
300    testKeySpecs(input, expectedOutput, helper);
301   
302    // test getWordLengths with -k1.0,2.3
303    helper = new KeyFieldHelper();
304    helper.setKeyFieldSeparator("\t");
305    helper.parseOption("-k1.1,2.3");
306    expectedOutput = "hi\thel";
307    testKeySpecs(input, expectedOutput, helper);
308   
309    // test getWordLengths with -k1.2,2.3
310    helper = new KeyFieldHelper();
311    helper.setKeyFieldSeparator("\t");
312    helper.parseOption("-k1.2,2.3");
313    expectedOutput = "i\thel";
314    testKeySpecs(input, expectedOutput, helper);
315   
316    // test getStartOffset with -k1.2,3.0
317    helper = new KeyFieldHelper();
318    helper.setKeyFieldSeparator("\t");
319    helper.parseOption("-k1.2,3.0");
320    expectedOutput = "i\thello";
321    testKeySpecs(input, expectedOutput, helper);
322   
323    // test getStartOffset with -k2,2
324    helper = new KeyFieldHelper();
325    helper.setKeyFieldSeparator("\t");
326    helper.parseOption("-k2,2");
327    expectedOutput = "hello";
328    testKeySpecs(input, expectedOutput, helper);
329   
330    // test getStartOffset with -k3.0,4.0
331    helper = new KeyFieldHelper();
332    helper.setKeyFieldSeparator("\t");
333    helper.parseOption("-k3.1,4.0");
334    testKeySpecs(input, null, helper);
335   
336    // test getStartOffset with -k2.1
337    helper = new KeyFieldHelper();
338    input = "123123123123123hi\thello\thow";
339    helper.setKeyFieldSeparator("\t");
340    helper.parseOption("-k2.1");
341    expectedOutput = "hello\thow";
342    testKeySpecs(input, expectedOutput, helper, 15, input.length());
343   
344    // test getStartOffset with -k2.1,4 with end ending on \t
345    helper = new KeyFieldHelper();
346    input = "123123123123123hi\thello\t\thow\tare";
347    helper.setKeyFieldSeparator("\t");
348    helper.parseOption("-k2.1,3");
349    expectedOutput = "hello\t";
350    testKeySpecs(input, expectedOutput, helper, 17, input.length());
351   
352    // test getStartOffset with -k2.1 with end ending on \t
353    helper = new KeyFieldHelper();
354    input = "123123123123123hi\thello\thow\tare";
355    helper.setKeyFieldSeparator("\t");
356    helper.parseOption("-k2.1");
357    expectedOutput = "hello\thow\t";
358    testKeySpecs(input, expectedOutput, helper, 17, 28);
359   
360    // test getStartOffset with -k2.1,3 with smaller length
361    helper = new KeyFieldHelper();
362    input = "123123123123123hi\thello\thow";
363    helper.setKeyFieldSeparator("\t");
364    helper.parseOption("-k2.1,3");
365    expectedOutput = "hello";
366    testKeySpecs(input, expectedOutput, helper, 15, 23);
367  }
368 
369  private void testKeySpecs(String input, String expectedOutput, 
370                            KeyFieldHelper helper) {
371    testKeySpecs(input, expectedOutput, helper, 0, -1);
372  }
373 
374  private void testKeySpecs(String input, String expectedOutput, 
375                            KeyFieldHelper helper, int s1, int e1) {
376    LOG.info("input : " + input);
377    String keySpecs = helper.keySpecs().get(0).toString();
378    LOG.info("keyspecs : " + keySpecs);
379    byte[] inputBytes = input.getBytes(); // get the input bytes
380    if (e1 == -1) {
381      e1 = inputBytes.length;
382    }
383    LOG.info("length : " + e1);
384    // get the word lengths
385    int[] indices = helper.getWordLengths(inputBytes, s1, e1);
386    // get the start index
387    int start = helper.getStartOffset(inputBytes, s1, e1, indices, 
388                                      helper.keySpecs().get(0));
389    LOG.info("start : " + start);
390    if (expectedOutput == null) {
391      assertEquals("Expected -1 when the start index is invalid", -1, start);
392      return;
393    }
394    // get the end index
395    int end = helper.getEndOffset(inputBytes, s1, e1, indices, 
396                                  helper.keySpecs().get(0));
397    LOG.info("end : " + end);
398    //my fix
399    end = (end >= inputBytes.length) ? inputBytes.length -1 : end;
400    int length = end + 1 - start;
401    LOG.info("length : " + length);
402    byte[] outputBytes = new byte[length];
403    System.arraycopy(inputBytes, start, outputBytes, 0, length);
404    String output = new String(outputBytes);
405    LOG.info("output : " + output);
406    LOG.info("expected-output : " + expectedOutput);
407    assertEquals(keySpecs + " failed on input '" + input + "'", 
408                 expectedOutput, output);
409  }
410
411  // check for equality of 2 int arrays
412  private boolean equals(int[] test, int[] expected) {
413    // check array length
414    if (test[0] != expected[0]) {
415      return false;
416    }
417    // if length is same then check the contents
418    for (int i = 0; i < test[0] && i < expected[0]; ++i) {
419      if (test[i] != expected[i]) {
420        return false;
421      }
422    }
423    return true;
424  }
425}
Note: See TracBrowser for help on using the repository browser.