1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | package org.apache.hadoop.mapred; |
---|
19 | |
---|
20 | import org.apache.hadoop.fs.*; |
---|
21 | import org.apache.hadoop.io.*; |
---|
22 | import org.apache.hadoop.mapred.lib.*; |
---|
23 | import junit.framework.TestCase; |
---|
24 | import java.io.*; |
---|
25 | import java.util.*; |
---|
26 | |
---|
27 | /** |
---|
28 | * TestMapOutputType checks whether the Map task handles type mismatch |
---|
29 | * between mapper output and the type specified in |
---|
30 | * JobConf.MapOutputKeyType and JobConf.MapOutputValueType. |
---|
31 | */ |
---|
32 | public class TestMapOutputType extends TestCase |
---|
33 | { |
---|
34 | JobConf conf = new JobConf(TestMapOutputType.class); |
---|
35 | JobClient jc; |
---|
36 | /** |
---|
37 | * TextGen is a Mapper that generates a Text key-value pair. The |
---|
38 | * type specified in conf will be anything but. |
---|
39 | */ |
---|
40 | |
---|
41 | static class TextGen |
---|
42 | implements Mapper<WritableComparable, Writable, Text, Text> { |
---|
43 | |
---|
44 | public void configure(JobConf job) { |
---|
45 | } |
---|
46 | |
---|
47 | public void map(WritableComparable key, Writable val, |
---|
48 | OutputCollector<Text, Text> out, |
---|
49 | Reporter reporter) throws IOException { |
---|
50 | out.collect(new Text("Hello"), new Text("World")); |
---|
51 | } |
---|
52 | |
---|
53 | public void close() { |
---|
54 | } |
---|
55 | } |
---|
56 | |
---|
57 | /** A do-nothing reducer class. We won't get this far, really. |
---|
58 | * |
---|
59 | */ |
---|
60 | static class TextReduce |
---|
61 | implements Reducer<Text, Text, Text, Text> { |
---|
62 | |
---|
63 | public void configure(JobConf job) { |
---|
64 | } |
---|
65 | |
---|
66 | public void reduce(Text key, |
---|
67 | Iterator<Text> values, |
---|
68 | OutputCollector<Text, Text> out, |
---|
69 | Reporter reporter) throws IOException { |
---|
70 | out.collect(new Text("Test"), new Text("Me")); |
---|
71 | } |
---|
72 | |
---|
73 | public void close() { |
---|
74 | } |
---|
75 | } |
---|
76 | |
---|
77 | |
---|
78 | public void configure() throws Exception { |
---|
79 | Path testdir = new Path("build/test/test.mapred.spill"); |
---|
80 | Path inDir = new Path(testdir, "in"); |
---|
81 | Path outDir = new Path(testdir, "out"); |
---|
82 | FileSystem fs = FileSystem.get(conf); |
---|
83 | fs.delete(testdir, true); |
---|
84 | conf.setInt("io.sort.mb", 1); |
---|
85 | conf.setInputFormat(SequenceFileInputFormat.class); |
---|
86 | FileInputFormat.setInputPaths(conf, inDir); |
---|
87 | FileOutputFormat.setOutputPath(conf, outDir); |
---|
88 | conf.setMapperClass(TextGen.class); |
---|
89 | conf.setReducerClass(TextReduce.class); |
---|
90 | conf.setOutputKeyClass(Text.class); |
---|
91 | conf.setOutputValueClass(Text.class); |
---|
92 | |
---|
93 | conf.setOutputFormat(SequenceFileOutputFormat.class); |
---|
94 | if (!fs.mkdirs(testdir)) { |
---|
95 | throw new IOException("Mkdirs failed to create " + testdir.toString()); |
---|
96 | } |
---|
97 | if (!fs.mkdirs(inDir)) { |
---|
98 | throw new IOException("Mkdirs failed to create " + inDir.toString()); |
---|
99 | } |
---|
100 | Path inFile = new Path(inDir, "part0"); |
---|
101 | SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile, |
---|
102 | Text.class, Text.class); |
---|
103 | writer.append(new Text("rec: 1"), new Text("Hello")); |
---|
104 | writer.close(); |
---|
105 | |
---|
106 | jc = new JobClient(conf); |
---|
107 | } |
---|
108 | |
---|
109 | public void testKeyMismatch() throws Exception { |
---|
110 | configure(); |
---|
111 | |
---|
112 | // Set bad MapOutputKeyClass and MapOutputValueClass |
---|
113 | conf.setMapOutputKeyClass(IntWritable.class); |
---|
114 | conf.setMapOutputValueClass(IntWritable.class); |
---|
115 | |
---|
116 | RunningJob r_job = jc.submitJob(conf); |
---|
117 | while (!r_job.isComplete()) { |
---|
118 | Thread.sleep(1000); |
---|
119 | } |
---|
120 | |
---|
121 | if (r_job.isSuccessful()) { |
---|
122 | fail("Oops! The job was supposed to break due to an exception"); |
---|
123 | } |
---|
124 | } |
---|
125 | |
---|
126 | public void testValueMismatch() throws Exception { |
---|
127 | configure(); |
---|
128 | |
---|
129 | // Set good MapOutputKeyClass, bad MapOutputValueClass |
---|
130 | conf.setMapOutputKeyClass(Text.class); |
---|
131 | conf.setMapOutputValueClass(IntWritable.class); |
---|
132 | |
---|
133 | RunningJob r_job = jc.submitJob(conf); |
---|
134 | while (!r_job.isComplete()) { |
---|
135 | Thread.sleep(1000); |
---|
136 | } |
---|
137 | |
---|
138 | if (r_job.isSuccessful()) { |
---|
139 | fail("Oops! The job was supposed to break due to an exception"); |
---|
140 | } |
---|
141 | } |
---|
142 | |
---|
143 | public void testNoMismatch() throws Exception{ |
---|
144 | configure(); |
---|
145 | |
---|
146 | // Set good MapOutputKeyClass and MapOutputValueClass |
---|
147 | conf.setMapOutputKeyClass(Text.class); |
---|
148 | conf.setMapOutputValueClass(Text.class); |
---|
149 | |
---|
150 | RunningJob r_job = jc.submitJob(conf); |
---|
151 | while (!r_job.isComplete()) { |
---|
152 | Thread.sleep(1000); |
---|
153 | } |
---|
154 | |
---|
155 | if (!r_job.isSuccessful()) { |
---|
156 | fail("Oops! The job broke due to an unexpected error"); |
---|
157 | } |
---|
158 | } |
---|
159 | } |
---|