1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.mapred.pipes; |
---|
20 | |
---|
21 | import java.io.*; |
---|
22 | import java.util.*; |
---|
23 | |
---|
24 | import org.apache.hadoop.fs.*; |
---|
25 | import org.apache.hadoop.io.*; |
---|
26 | import org.apache.hadoop.mapred.*; |
---|
27 | |
---|
28 | /** |
---|
29 | * This is a support class to test Hadoop Pipes when using C++ RecordReaders. |
---|
30 | * It defines an InputFormat with InputSplits that are just strings. The |
---|
31 | * RecordReaders are not implemented in Java, naturally... |
---|
32 | */ |
---|
33 | public class WordCountInputFormat |
---|
34 | extends FileInputFormat<IntWritable, Text> { |
---|
35 | |
---|
36 | static class WordCountInputSplit implements InputSplit { |
---|
37 | private String filename; |
---|
38 | WordCountInputSplit() { } |
---|
39 | WordCountInputSplit(Path filename) { |
---|
40 | this.filename = filename.toUri().getPath(); |
---|
41 | } |
---|
42 | public void write(DataOutput out) throws IOException { |
---|
43 | Text.writeString(out, filename); |
---|
44 | } |
---|
45 | public void readFields(DataInput in) throws IOException { |
---|
46 | filename = Text.readString(in); |
---|
47 | } |
---|
48 | public long getLength() { return 0L; } |
---|
49 | public String[] getLocations() { return new String[0]; } |
---|
50 | } |
---|
51 | |
---|
52 | public InputSplit[] getSplits(JobConf conf, |
---|
53 | int numSplits) throws IOException { |
---|
54 | ArrayList<InputSplit> result = new ArrayList<InputSplit>(); |
---|
55 | FileSystem local = FileSystem.getLocal(conf); |
---|
56 | for(Path dir: getInputPaths(conf)) { |
---|
57 | for(FileStatus file: local.listStatus(dir)) { |
---|
58 | result.add(new WordCountInputSplit(file.getPath())); |
---|
59 | } |
---|
60 | } |
---|
61 | return result.toArray(new InputSplit[result.size()]); |
---|
62 | } |
---|
63 | public RecordReader<IntWritable, Text> getRecordReader(InputSplit split, |
---|
64 | JobConf conf, |
---|
65 | Reporter reporter) { |
---|
66 | return new RecordReader<IntWritable, Text>(){ |
---|
67 | public boolean next(IntWritable key, Text value) throws IOException { |
---|
68 | return false; |
---|
69 | } |
---|
70 | public IntWritable createKey() { |
---|
71 | return new IntWritable(); |
---|
72 | } |
---|
73 | public Text createValue() { |
---|
74 | return new Text(); |
---|
75 | } |
---|
76 | public long getPos() { |
---|
77 | return 0; |
---|
78 | } |
---|
79 | public void close() { } |
---|
80 | public float getProgress() { |
---|
81 | return 0.0f; |
---|
82 | } |
---|
83 | }; |
---|
84 | } |
---|
85 | } |
---|