1 | <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> |
---|
2 | <!--NewPage--> |
---|
3 | <HTML> |
---|
4 | <HEAD> |
---|
5 | <!-- Generated by javadoc (build 1.6.0_07) on Tue Sep 01 20:56:59 UTC 2009 --> |
---|
6 | <TITLE> |
---|
7 | org.apache.hadoop.mapred.lib.aggregate (Hadoop 0.20.1 API) |
---|
8 | </TITLE> |
---|
9 | |
---|
10 | <META NAME="date" CONTENT="2009-09-01"> |
---|
11 | |
---|
12 | <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../../../stylesheet.css" TITLE="Style"> |
---|
13 | |
---|
14 | <SCRIPT type="text/javascript"> |
---|
15 | function windowTitle() |
---|
16 | { |
---|
17 | if (location.href.indexOf('is-external=true') == -1) { |
---|
18 | parent.document.title="org.apache.hadoop.mapred.lib.aggregate (Hadoop 0.20.1 API)"; |
---|
19 | } |
---|
20 | } |
---|
21 | </SCRIPT> |
---|
22 | <NOSCRIPT> |
---|
23 | </NOSCRIPT> |
---|
24 | |
---|
25 | </HEAD> |
---|
26 | |
---|
27 | <BODY BGCOLOR="white" onload="windowTitle();"> |
---|
28 | <HR> |
---|
29 | |
---|
30 | |
---|
31 | <!-- ========= START OF TOP NAVBAR ======= --> |
---|
32 | <A NAME="navbar_top"><!-- --></A> |
---|
33 | <A HREF="#skip-navbar_top" title="Skip navigation links"></A> |
---|
34 | <TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0" SUMMARY=""> |
---|
35 | <TR> |
---|
36 | <TD COLSPAN=2 BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> |
---|
37 | <A NAME="navbar_top_firstrow"><!-- --></A> |
---|
38 | <TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3" SUMMARY=""> |
---|
39 | <TR ALIGN="center" VALIGN="top"> |
---|
40 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../overview-summary.html"><FONT CLASS="NavBarFont1"><B>Overview</B></FONT></A> </TD> |
---|
41 | <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> <FONT CLASS="NavBarFont1Rev"><B>Package</B></FONT> </TD> |
---|
42 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <FONT CLASS="NavBarFont1">Class</FONT> </TD> |
---|
43 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-use.html"><FONT CLASS="NavBarFont1"><B>Use</B></FONT></A> </TD> |
---|
44 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A> </TD> |
---|
45 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A> </TD> |
---|
46 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A> </TD> |
---|
47 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A> </TD> |
---|
48 | </TR> |
---|
49 | </TABLE> |
---|
50 | </TD> |
---|
51 | <TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM> |
---|
52 | </EM> |
---|
53 | </TD> |
---|
54 | </TR> |
---|
55 | |
---|
56 | <TR> |
---|
57 | <TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2"> |
---|
58 | <A HREF="../../../../../../org/apache/hadoop/mapred/lib/package-summary.html"><B>PREV PACKAGE</B></A> |
---|
59 | <A HREF="../../../../../../org/apache/hadoop/mapred/lib/db/package-summary.html"><B>NEXT PACKAGE</B></A></FONT></TD> |
---|
60 | <TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2"> |
---|
61 | <A HREF="../../../../../../index.html?org/apache/hadoop/mapred/lib/aggregate/package-summary.html" target="_top"><B>FRAMES</B></A> |
---|
62 | <A HREF="package-summary.html" target="_top"><B>NO FRAMES</B></A> |
---|
63 | <SCRIPT type="text/javascript"> |
---|
64 | <!-- |
---|
65 | if(window==top) { |
---|
66 | document.writeln('<A HREF="../../../../../../allclasses-noframe.html"><B>All Classes</B></A>'); |
---|
67 | } |
---|
68 | //--> |
---|
69 | </SCRIPT> |
---|
70 | <NOSCRIPT> |
---|
71 | <A HREF="../../../../../../allclasses-noframe.html"><B>All Classes</B></A> |
---|
72 | </NOSCRIPT> |
---|
73 | |
---|
74 | |
---|
75 | </FONT></TD> |
---|
76 | </TR> |
---|
77 | </TABLE> |
---|
78 | <A NAME="skip-navbar_top"></A> |
---|
79 | <!-- ========= END OF TOP NAVBAR ========= --> |
---|
80 | |
---|
81 | <HR> |
---|
82 | <H2> |
---|
83 | Package org.apache.hadoop.mapred.lib.aggregate |
---|
84 | </H2> |
---|
85 | Classes for performing various counting and aggregations. |
---|
86 | <P> |
---|
87 | <B>See:</B> |
---|
88 | <BR> |
---|
89 | <A HREF="#package_description"><B>Description</B></A> |
---|
90 | <P> |
---|
91 | |
---|
92 | <TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY=""> |
---|
93 | <TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor"> |
---|
94 | <TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2"> |
---|
95 | <B>Interface Summary</B></FONT></TH> |
---|
96 | </TR> |
---|
97 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
98 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregator.html" title="interface in org.apache.hadoop.mapred.lib.aggregate">ValueAggregator</A></B></TD> |
---|
99 | <TD>This interface defines the minimal protocol for value aggregators.</TD> |
---|
100 | </TR> |
---|
101 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
102 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorDescriptor.html" title="interface in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorDescriptor</A></B></TD> |
---|
103 | <TD>This interface defines the contract a value aggregator descriptor must |
---|
104 | support.</TD> |
---|
105 | </TR> |
---|
106 | </TABLE> |
---|
107 | |
---|
108 | |
---|
109 | <P> |
---|
110 | |
---|
111 | <TABLE BORDER="1" WIDTH="100%" CELLPADDING="3" CELLSPACING="0" SUMMARY=""> |
---|
112 | <TR BGCOLOR="#CCCCFF" CLASS="TableHeadingColor"> |
---|
113 | <TH ALIGN="left" COLSPAN="2"><FONT SIZE="+2"> |
---|
114 | <B>Class Summary</B></FONT></TH> |
---|
115 | </TR> |
---|
116 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
117 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/DoubleValueSum.html" title="class in org.apache.hadoop.mapred.lib.aggregate">DoubleValueSum</A></B></TD> |
---|
118 | <TD>This class implements a value aggregator that sums up a sequence of double |
---|
119 | values.</TD> |
---|
120 | </TR> |
---|
121 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
122 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/LongValueMax.html" title="class in org.apache.hadoop.mapred.lib.aggregate">LongValueMax</A></B></TD> |
---|
123 | <TD>This class implements a value aggregator that maintain the maximum of |
---|
124 | a sequence of long values.</TD> |
---|
125 | </TR> |
---|
126 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
127 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/LongValueMin.html" title="class in org.apache.hadoop.mapred.lib.aggregate">LongValueMin</A></B></TD> |
---|
128 | <TD>This class implements a value aggregator that maintain the minimum of |
---|
129 | a sequence of long values.</TD> |
---|
130 | </TR> |
---|
131 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
132 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/LongValueSum.html" title="class in org.apache.hadoop.mapred.lib.aggregate">LongValueSum</A></B></TD> |
---|
133 | <TD>This class implements a value aggregator that sums up |
---|
134 | a sequence of long values.</TD> |
---|
135 | </TR> |
---|
136 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
137 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/StringValueMax.html" title="class in org.apache.hadoop.mapred.lib.aggregate">StringValueMax</A></B></TD> |
---|
138 | <TD>This class implements a value aggregator that maintain the biggest of |
---|
139 | a sequence of strings.</TD> |
---|
140 | </TR> |
---|
141 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
142 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/StringValueMin.html" title="class in org.apache.hadoop.mapred.lib.aggregate">StringValueMin</A></B></TD> |
---|
143 | <TD>This class implements a value aggregator that maintain the smallest of |
---|
144 | a sequence of strings.</TD> |
---|
145 | </TR> |
---|
146 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
147 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/UniqValueCount.html" title="class in org.apache.hadoop.mapred.lib.aggregate">UniqValueCount</A></B></TD> |
---|
148 | <TD>This class implements a value aggregator that dedupes a sequence of objects.</TD> |
---|
149 | </TR> |
---|
150 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
151 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/UserDefinedValueAggregatorDescriptor.html" title="class in org.apache.hadoop.mapred.lib.aggregate">UserDefinedValueAggregatorDescriptor</A></B></TD> |
---|
152 | <TD>This class implements a wrapper for a user defined value aggregator descriptor.</TD> |
---|
153 | </TR> |
---|
154 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
155 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorBaseDescriptor.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorBaseDescriptor</A></B></TD> |
---|
156 | <TD>This class implements the common functionalities of |
---|
157 | the subclasses of ValueAggregatorDescriptor class.</TD> |
---|
158 | </TR> |
---|
159 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
160 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorCombiner.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorCombiner<K1 extends WritableComparable,V1 extends Writable></A></B></TD> |
---|
161 | <TD>This class implements the generic combiner of Aggregate.</TD> |
---|
162 | </TR> |
---|
163 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
164 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorJob.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorJob</A></B></TD> |
---|
165 | <TD>This is the main class for creating a map/reduce job using Aggregate |
---|
166 | framework.</TD> |
---|
167 | </TR> |
---|
168 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
169 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorJobBase.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorJobBase<K1 extends WritableComparable,V1 extends Writable></A></B></TD> |
---|
170 | <TD>This abstract class implements some common functionalities of the |
---|
171 | the generic mapper, reducer and combiner classes of Aggregate.</TD> |
---|
172 | </TR> |
---|
173 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
174 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorMapper.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorMapper<K1 extends WritableComparable,V1 extends Writable></A></B></TD> |
---|
175 | <TD>This class implements the generic mapper of Aggregate.</TD> |
---|
176 | </TR> |
---|
177 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
178 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueAggregatorReducer.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueAggregatorReducer<K1 extends WritableComparable,V1 extends Writable></A></B></TD> |
---|
179 | <TD>This class implements the generic reducer of Aggregate.</TD> |
---|
180 | </TR> |
---|
181 | <TR BGCOLOR="white" CLASS="TableRowColor"> |
---|
182 | <TD WIDTH="15%"><B><A HREF="../../../../../../org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.html" title="class in org.apache.hadoop.mapred.lib.aggregate">ValueHistogram</A></B></TD> |
---|
183 | <TD>This class implements a value aggregator that computes the |
---|
184 | histogram of a sequence of strings.</TD> |
---|
185 | </TR> |
---|
186 | </TABLE> |
---|
187 | |
---|
188 | |
---|
189 | <P> |
---|
190 | <A NAME="package_description"><!-- --></A><H2> |
---|
191 | Package org.apache.hadoop.mapred.lib.aggregate Description |
---|
192 | </H2> |
---|
193 | |
---|
194 | <P> |
---|
195 | Classes for performing various counting and aggregations. |
---|
196 | <p /> |
---|
197 | <h2><a name="Aggregate"></a>Aggregate framework </h2> |
---|
198 | <p /> |
---|
199 | Generally speaking, in order to implement an application using Map/Reduce |
---|
200 | model, the developer needs to implement Map and Reduce functions (and possibly |
---|
201 | Combine function). However, for a lot of applications related to counting and |
---|
202 | statistics computing, these functions have very similar |
---|
203 | characteristics. This provides a package implementing |
---|
204 | those patterns. In particular, the package provides a generic mapper class, |
---|
205 | a reducer class and a combiner class, and a set of built-in value aggregators. |
---|
206 | It also provides a generic utility class, ValueAggregatorJob, that offers a static function that |
---|
207 | creates map/reduce jobs: |
---|
208 | <blockquote> |
---|
209 | <pre> |
---|
210 | public static JobConf createValueAggregatorJob(String args[]) throws IOException; |
---|
211 | </pre> |
---|
212 | </blockquote> |
---|
213 | To call this function, the user needs to pass in arguments specifying the input directories, the output directory, |
---|
214 | the number of reducers, the input data format (textinputformat or sequencefileinputformat), and a file specifying user plugin class(es) to load by the mapper. |
---|
215 | A user plugin class is responsible for specifying what |
---|
216 | aggregators to use and what values are for which aggregators. |
---|
217 | A plugin class must implement the following interface: |
---|
218 | <blockquote> |
---|
219 | <pre> |
---|
220 | public interface ValueAggregatorDescriptor { |
---|
221 | public ArrayList<Entry> generateKeyValPairs(Object key, Object value); |
---|
222 | public void configure(JobConfjob); |
---|
223 | } |
---|
224 | </pre> |
---|
225 | </blockquote> |
---|
226 | Function generateKeyValPairs will generate aggregation key/value pairs for the |
---|
227 | input key/value pair. Each aggregation key encodes two pieces of information: the aggregation type and aggregation ID. |
---|
228 | The value is the value to be aggregated onto the aggregation ID according to the aggregation type. Here |
---|
229 | is a simple example user plugin class for counting the words in the input texts: |
---|
230 | <blockquote> |
---|
231 | <pre> |
---|
232 | public class WordCountAggregatorDescriptor extends ValueAggregatorBaseDescriptor { |
---|
233 | public ArrayList<Entry> generateKeyValPairs(Object key, Object val) { |
---|
234 | String words [] = val.toString().split(" |\t"); |
---|
235 | ArrayList<Entry> retv = new ArrayList<Entry>(); |
---|
236 | for (int i = 0; i < words.length; i++) { |
---|
237 | retv.add(generateEntry(LONG_VALUE_SUM, words[i], ONE)) |
---|
238 | } |
---|
239 | return retv; |
---|
240 | } |
---|
241 | public void configure(JobConf job) {} |
---|
242 | } |
---|
243 | </pre> |
---|
244 | </blockquote> |
---|
245 | In the above code, LONG_VALUE_SUM is a string denoting the aggregation type LongValueSum, which sums over long values. |
---|
246 | ONE denotes a string "1". Function generateEntry(LONG_VALUE_SUM, words[i], ONE) will inperpret the first argument as an aggregation type, the second as an aggregation ID, and the the third argumnent as the value to be aggregated. The output will look like: "LongValueSum:xxxx", where XXXX is the string value of words[i]. The value will be "1". The mapper will call generateKeyValPairs(Object key, Object val) for each input key/value pair to generate the desired aggregation id/value pairs. |
---|
247 | The down stream combiner/reducer will interpret these pairs as adding one to the aggregator XXXX. |
---|
248 | <p /> |
---|
249 | Class ValueAggregatorBaseDescriptor is a base class that user plugin classes can extend. Here is the XML fragment specifying the user plugin class: |
---|
250 | <blockquote> |
---|
251 | <pre> |
---|
252 | <property> |
---|
253 | <name>aggregator.descriptor.num</name> |
---|
254 | <value>1</value> |
---|
255 | </property> |
---|
256 | <property> |
---|
257 | <name>aggregator.descriptor.0</name> |
---|
258 | <value>UserDefined,org.apache.hadoop.mapred.lib.aggregate.examples.WordCountAggregatorDescriptor</value> |
---|
259 | </property> |
---|
260 | </pre> |
---|
261 | </blockquote> |
---|
262 | Class ValueAggregatorBaseDescriptor itself provides a default implementation for generateKeyValPairs: |
---|
263 | <blockquote> |
---|
264 | <pre> |
---|
265 | public ArrayList<Entry> generateKeyValPairs(Object key, Object val) { |
---|
266 | ArrayList<Entry> retv = new ArrayList<Entry>(); |
---|
267 | String countType = LONG_VALUE_SUM; |
---|
268 | String id = "record_count"; |
---|
269 | retv.add(generateEntry(countType, id, ONE)); |
---|
270 | return retv; |
---|
271 | } |
---|
272 | </pre> |
---|
273 | </blockquote> |
---|
274 | Thus, if no user plugin class is specified, the default behavior of the map/reduce job is to count the number of records (lines) in the imput files. |
---|
275 | <p /> |
---|
276 | During runtime, the mapper will invoke the generateKeyValPairs function for each input key/value pair, and emit the generated |
---|
277 | key/value pairs: |
---|
278 | <blockquote> |
---|
279 | <pre> |
---|
280 | public void map(WritableComparable key, Writable value, |
---|
281 | OutputCollector output, Reporter reporter) throws IOException { |
---|
282 | Iterator iter = this.aggregatorDescriptorList.iterator(); |
---|
283 | while (iter.hasNext()) { |
---|
284 | ValueAggregatorDescriptor ad = (ValueAggregatorDescriptor) iter.next(); |
---|
285 | Iterator<Entry> ens = ad.generateKeyValPairs(key, value).iterator(); |
---|
286 | while (ens.hasNext()) { |
---|
287 | Entry en = ens.next(); |
---|
288 | output.collect((WritableComparable)en.getKey(), (Writable)en.getValue()); |
---|
289 | } |
---|
290 | } |
---|
291 | } |
---|
292 | </pre> |
---|
293 | </blockquote> |
---|
294 | The reducer will create an aggregator object for each key/value list pair, and perform the appropriate aggregation. |
---|
295 | At the end, it will emit the aggregator's results: |
---|
296 | <blockquote> |
---|
297 | <pre> |
---|
298 | public void reduce(WritableComparable key, Iterator values, |
---|
299 | OutputCollector output, Reporter reporter) throws IOException { |
---|
300 | String keyStr = key.toString(); |
---|
301 | int pos = keyStr.indexOf(ValueAggregatorDescriptor.TYPE_SEPARATOR); |
---|
302 | String type = keyStr.substring(0,pos); |
---|
303 | keyStr = keyStr.substring(pos+ValueAggregatorDescriptor.TYPE_SEPARATOR.length()); |
---|
304 | ValueAggregator aggregator = |
---|
305 | ValueAggregatorBaseDescriptor.generateValueAggregator(type); |
---|
306 | while (values.hasNext()) { |
---|
307 | aggregator.addNextValue(values.next()); |
---|
308 | } |
---|
309 | String val = aggregator.getReport(); |
---|
310 | key = new Text(keyStr); |
---|
311 | output.collect(key, new Text(val)); |
---|
312 | } |
---|
313 | </pre> |
---|
314 | </blockquote> |
---|
315 | In order to be able to use combiner, all the aggregation type be aggregators must be associative and communitive. |
---|
316 | The following are the types supported: <ul> |
---|
317 | <li> LongValueSum: sum over long values |
---|
318 | </li> <li> DoubleValueSum: sum over float/double values |
---|
319 | </li> <li> uniqValueCount: count the number of distinct values |
---|
320 | </li> <li> ValueHistogram: compute the histogram of values compute the minimum, maximum, media,average, standard deviation of numeric values |
---|
321 | </li></ul> |
---|
322 | <p /> |
---|
323 | <h2><a name="Create_and_run"></a> Create and run an application </h2> |
---|
324 | <p /> |
---|
325 | To create an application, the user needs to do the following things: |
---|
326 | <p /> |
---|
327 | 1. Implement a user plugin: |
---|
328 | <blockquote> |
---|
329 | <pre> |
---|
330 | import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor; |
---|
331 | import org.apache.hadoop.mapred.JobConf; |
---|
332 | |
---|
333 | public class WordCountAggregatorDescriptor extends ValueAggregatorBaseDescriptor { |
---|
334 | public void map(WritableComparable key, Writable value, |
---|
335 | OutputCollector output, Reporter reporter) throws IOException { |
---|
336 | } |
---|
337 | public void configure(JobConf job) { |
---|
338 | |
---|
339 | } |
---|
340 | } |
---|
341 | </pre> |
---|
342 | </blockquote> |
---|
343 | |
---|
344 | 2. Create an xml file specifying the user plugin. |
---|
345 | <p /> |
---|
346 | 3. Compile your java class and create a jar file, say wc.jar. |
---|
347 | |
---|
348 | <p /> |
---|
349 | Finally, run the job: |
---|
350 | <blockquote> |
---|
351 | <pre> |
---|
352 | hadoop jar wc.jar org.apache.hadoop.mapred.lib.aggregate..ValueAggregatorJob indirs outdir numofreducers textinputformat|sequencefileinputformat spec_file |
---|
353 | </pre> |
---|
354 | </blockquote> |
---|
355 | <p /> |
---|
356 | <P> |
---|
357 | |
---|
358 | <P> |
---|
359 | <DL> |
---|
360 | </DL> |
---|
361 | <HR> |
---|
362 | |
---|
363 | |
---|
364 | <!-- ======= START OF BOTTOM NAVBAR ====== --> |
---|
365 | <A NAME="navbar_bottom"><!-- --></A> |
---|
366 | <A HREF="#skip-navbar_bottom" title="Skip navigation links"></A> |
---|
367 | <TABLE BORDER="0" WIDTH="100%" CELLPADDING="1" CELLSPACING="0" SUMMARY=""> |
---|
368 | <TR> |
---|
369 | <TD COLSPAN=2 BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> |
---|
370 | <A NAME="navbar_bottom_firstrow"><!-- --></A> |
---|
371 | <TABLE BORDER="0" CELLPADDING="0" CELLSPACING="3" SUMMARY=""> |
---|
372 | <TR ALIGN="center" VALIGN="top"> |
---|
373 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../overview-summary.html"><FONT CLASS="NavBarFont1"><B>Overview</B></FONT></A> </TD> |
---|
374 | <TD BGCOLOR="#FFFFFF" CLASS="NavBarCell1Rev"> <FONT CLASS="NavBarFont1Rev"><B>Package</B></FONT> </TD> |
---|
375 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <FONT CLASS="NavBarFont1">Class</FONT> </TD> |
---|
376 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-use.html"><FONT CLASS="NavBarFont1"><B>Use</B></FONT></A> </TD> |
---|
377 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="package-tree.html"><FONT CLASS="NavBarFont1"><B>Tree</B></FONT></A> </TD> |
---|
378 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../deprecated-list.html"><FONT CLASS="NavBarFont1"><B>Deprecated</B></FONT></A> </TD> |
---|
379 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../index-all.html"><FONT CLASS="NavBarFont1"><B>Index</B></FONT></A> </TD> |
---|
380 | <TD BGCOLOR="#EEEEFF" CLASS="NavBarCell1"> <A HREF="../../../../../../help-doc.html"><FONT CLASS="NavBarFont1"><B>Help</B></FONT></A> </TD> |
---|
381 | </TR> |
---|
382 | </TABLE> |
---|
383 | </TD> |
---|
384 | <TD ALIGN="right" VALIGN="top" ROWSPAN=3><EM> |
---|
385 | </EM> |
---|
386 | </TD> |
---|
387 | </TR> |
---|
388 | |
---|
389 | <TR> |
---|
390 | <TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2"> |
---|
391 | <A HREF="../../../../../../org/apache/hadoop/mapred/lib/package-summary.html"><B>PREV PACKAGE</B></A> |
---|
392 | <A HREF="../../../../../../org/apache/hadoop/mapred/lib/db/package-summary.html"><B>NEXT PACKAGE</B></A></FONT></TD> |
---|
393 | <TD BGCOLOR="white" CLASS="NavBarCell2"><FONT SIZE="-2"> |
---|
394 | <A HREF="../../../../../../index.html?org/apache/hadoop/mapred/lib/aggregate/package-summary.html" target="_top"><B>FRAMES</B></A> |
---|
395 | <A HREF="package-summary.html" target="_top"><B>NO FRAMES</B></A> |
---|
396 | <SCRIPT type="text/javascript"> |
---|
397 | <!-- |
---|
398 | if(window==top) { |
---|
399 | document.writeln('<A HREF="../../../../../../allclasses-noframe.html"><B>All Classes</B></A>'); |
---|
400 | } |
---|
401 | //--> |
---|
402 | </SCRIPT> |
---|
403 | <NOSCRIPT> |
---|
404 | <A HREF="../../../../../../allclasses-noframe.html"><B>All Classes</B></A> |
---|
405 | </NOSCRIPT> |
---|
406 | |
---|
407 | |
---|
408 | </FONT></TD> |
---|
409 | </TR> |
---|
410 | </TABLE> |
---|
411 | <A NAME="skip-navbar_bottom"></A> |
---|
412 | <!-- ======== END OF BOTTOM NAVBAR ======= --> |
---|
413 | |
---|
414 | <HR> |
---|
415 | Copyright © 2009 The Apache Software Foundation |
---|
416 | </BODY> |
---|
417 | </HTML> |
---|