1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.mapred; |
---|
20 | |
---|
21 | import java.io.*; |
---|
22 | import junit.framework.TestCase; |
---|
23 | |
---|
24 | import org.apache.hadoop.fs.*; |
---|
25 | import org.apache.hadoop.io.*; |
---|
26 | |
---|
27 | public class TestFileOutputCommitter extends TestCase { |
---|
28 | private static Path outDir = new Path( |
---|
29 | System.getProperty("test.build.data", "."), "output"); |
---|
30 | |
---|
31 | // A random task attempt id for testing. |
---|
32 | private static String attempt = "attempt_200707121733_0001_m_000000_0"; |
---|
33 | private static TaskAttemptID taskID = TaskAttemptID.forName(attempt); |
---|
34 | |
---|
35 | @SuppressWarnings("unchecked") |
---|
36 | public void testCommitter() throws Exception { |
---|
37 | JobConf job = new JobConf(); |
---|
38 | job.set("mapred.task.id", attempt); |
---|
39 | job.setOutputCommitter(FileOutputCommitter.class); |
---|
40 | FileOutputFormat.setOutputPath(job, outDir); |
---|
41 | JobContext jContext = new JobContext(job, taskID.getJobID()); |
---|
42 | TaskAttemptContext tContext = new TaskAttemptContext(job, taskID); |
---|
43 | FileOutputCommitter committer = new FileOutputCommitter(); |
---|
44 | FileOutputFormat.setWorkOutputPath(job, |
---|
45 | committer.getTempTaskOutputPath(tContext)); |
---|
46 | |
---|
47 | committer.setupJob(jContext); |
---|
48 | committer.setupTask(tContext); |
---|
49 | String file = "test.txt"; |
---|
50 | |
---|
51 | // A reporter that does nothing |
---|
52 | Reporter reporter = Reporter.NULL; |
---|
53 | FileSystem localFs = FileSystem.getLocal(job); |
---|
54 | TextOutputFormat theOutputFormat = new TextOutputFormat(); |
---|
55 | RecordWriter theRecordWriter = |
---|
56 | theOutputFormat.getRecordWriter(localFs, job, file, reporter); |
---|
57 | Text key1 = new Text("key1"); |
---|
58 | Text key2 = new Text("key2"); |
---|
59 | Text val1 = new Text("val1"); |
---|
60 | Text val2 = new Text("val2"); |
---|
61 | NullWritable nullWritable = NullWritable.get(); |
---|
62 | |
---|
63 | try { |
---|
64 | theRecordWriter.write(key1, val1); |
---|
65 | theRecordWriter.write(null, nullWritable); |
---|
66 | theRecordWriter.write(null, val1); |
---|
67 | theRecordWriter.write(nullWritable, val2); |
---|
68 | theRecordWriter.write(key2, nullWritable); |
---|
69 | theRecordWriter.write(key1, null); |
---|
70 | theRecordWriter.write(null, null); |
---|
71 | theRecordWriter.write(key2, val2); |
---|
72 | } finally { |
---|
73 | theRecordWriter.close(reporter); |
---|
74 | } |
---|
75 | committer.commitTask(tContext); |
---|
76 | committer.cleanupJob(jContext); |
---|
77 | |
---|
78 | File expectedFile = new File(new Path(outDir, file).toString()); |
---|
79 | StringBuffer expectedOutput = new StringBuffer(); |
---|
80 | expectedOutput.append(key1).append('\t').append(val1).append("\n"); |
---|
81 | expectedOutput.append(val1).append("\n"); |
---|
82 | expectedOutput.append(val2).append("\n"); |
---|
83 | expectedOutput.append(key2).append("\n"); |
---|
84 | expectedOutput.append(key1).append("\n"); |
---|
85 | expectedOutput.append(key2).append('\t').append(val2).append("\n"); |
---|
86 | String output = UtilsForTests.slurp(expectedFile); |
---|
87 | assertEquals(output, expectedOutput.toString()); |
---|
88 | } |
---|
89 | |
---|
90 | public static void main(String[] args) throws Exception { |
---|
91 | new TestFileOutputCommitter().testCommitter(); |
---|
92 | } |
---|
93 | } |
---|