source: proiecte/HadoopJUnit/hadoop-0.20.1/docs/native_libraries.html @ 120

Last change on this file since 120 was 120, checked in by (none), 14 years ago

Added the mail files for the Hadoop JUNit Project
  • Property svn:executable set to *
File size: 15.0 KB
Line 
1<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2<html>
3<head>
4<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
5<meta content="Apache Forrest" name="Generator">
6<meta name="Forrest-version" content="0.8">
7<meta name="Forrest-skin-name" content="pelt">
8<title>Native Libraries Guide</title>
9<link type="text/css" href="skin/basic.css" rel="stylesheet">
10<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
11<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
12<link type="text/css" href="skin/profile.css" rel="stylesheet">
13<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
14<link rel="shortcut icon" href="images/favicon.ico">
15</head>
16<body onload="init()">
17<script type="text/javascript">ndeSetTextSize();</script>
18<div id="top">
19<!--+
20    |breadtrail
21    +-->
22<div class="breadtrail">
23<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/core/">Core</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
24</div>
25<!--+
26    |header
27    +-->
28<div class="header">
29<!--+
30    |start group logo
31    +-->
32<div class="grouplogo">
33<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
34</div>
35<!--+
36    |end group logo
37    +-->
38<!--+
39    |start Project Logo
40    +-->
41<div class="projectlogo">
42<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/core-logo.gif" title="Scalable Computing Platform"></a>
43</div>
44<!--+
45    |end Project Logo
46    +-->
47<!--+
48    |start Search
49    +-->
50<div class="searchbox">
51<form action="http://www.google.com/search" method="get" class="roundtopsmall">
52<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
53                    <input name="Search" value="Search" type="submit">
54</form>
55</div>
56<!--+
57    |end search
58    +-->
59<!--+
60    |start Tabs
61    +-->
62<ul id="tabs">
63<li>
64<a class="unselected" href="http://hadoop.apache.org/core/">Project</a>
65</li>
66<li>
67<a class="unselected" href="http://wiki.apache.org/hadoop">Wiki</a>
68</li>
69<li class="current">
70<a class="selected" href="index.html">Hadoop 0.20 Documentation</a>
71</li>
72</ul>
73<!--+
74    |end Tabs
75    +-->
76</div>
77</div>
78<div id="main">
79<div id="publishedStrip">
80<!--+
81    |start Subtabs
82    +-->
83<div id="level2tabs"></div>
84<!--+
85    |end Endtabs
86    +-->
87<script type="text/javascript"><!--
88document.write("Last Published: " + document.lastModified);
89//  --></script>
90</div>
91<!--+
92    |breadtrail
93    +-->
94<div class="breadtrail">
95
96             &nbsp;
97           </div>
98<!--+
99    |start Menu, mainarea
100    +-->
101<!--+
102    |start Menu
103    +-->
104<div id="menu">
105<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Getting Started</div>
106<div id="menu_1.1" class="menuitemgroup">
107<div class="menuitem">
108<a href="index.html">Overview</a>
109</div>
110<div class="menuitem">
111<a href="quickstart.html">Quick Start</a>
112</div>
113<div class="menuitem">
114<a href="cluster_setup.html">Cluster Setup</a>
115</div>
116<div class="menuitem">
117<a href="mapred_tutorial.html">Map/Reduce Tutorial</a>
118</div>
119</div>
120<div onclick="SwitchMenu('menu_selected_1.2', 'skin/')" id="menu_selected_1.2Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Programming Guides</div>
121<div id="menu_selected_1.2" class="selectedmenuitemgroup" style="display: block;">
122<div class="menuitem">
123<a href="commands_manual.html">Commands</a>
124</div>
125<div class="menuitem">
126<a href="distcp.html">DistCp</a>
127</div>
128<div class="menupage">
129<div class="menupagetitle">Native Libraries</div>
130</div>
131<div class="menuitem">
132<a href="streaming.html">Streaming</a>
133</div>
134<div class="menuitem">
135<a href="fair_scheduler.html">Fair Scheduler</a>
136</div>
137<div class="menuitem">
138<a href="capacity_scheduler.html">Capacity Scheduler</a>
139</div>
140<div class="menuitem">
141<a href="service_level_auth.html">Service Level Authorization</a>
142</div>
143<div class="menuitem">
144<a href="vaidya.html">Vaidya</a>
145</div>
146<div class="menuitem">
147<a href="hadoop_archives.html">Archives</a>
148</div>
149</div>
150<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">HDFS</div>
151<div id="menu_1.3" class="menuitemgroup">
152<div class="menuitem">
153<a href="hdfs_user_guide.html">User Guide</a>
154</div>
155<div class="menuitem">
156<a href="hdfs_design.html">Architecture</a>
157</div>
158<div class="menuitem">
159<a href="hdfs_shell.html">File System Shell Guide</a>
160</div>
161<div class="menuitem">
162<a href="hdfs_permissions_guide.html">Permissions Guide</a>
163</div>
164<div class="menuitem">
165<a href="hdfs_quota_admin_guide.html">Quotas Guide</a>
166</div>
167<div class="menuitem">
168<a href="SLG_user_guide.html">Synthetic Load Generator Guide</a>
169</div>
170<div class="menuitem">
171<a href="libhdfs.html">C API libhdfs</a>
172</div>
173</div>
174<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">HOD</div>
175<div id="menu_1.4" class="menuitemgroup">
176<div class="menuitem">
177<a href="hod_user_guide.html">User Guide</a>
178</div>
179<div class="menuitem">
180<a href="hod_admin_guide.html">Admin Guide</a>
181</div>
182<div class="menuitem">
183<a href="hod_config_guide.html">Config Guide</a>
184</div>
185</div>
186<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div>
187<div id="menu_1.5" class="menuitemgroup">
188<div class="menuitem">
189<a href="api/index.html">API Docs</a>
190</div>
191<div class="menuitem">
192<a href="jdiff/changes.html">API Changes</a>
193</div>
194<div class="menuitem">
195<a href="http://wiki.apache.org/hadoop/">Wiki</a>
196</div>
197<div class="menuitem">
198<a href="http://wiki.apache.org/hadoop/FAQ">FAQ</a>
199</div>
200<div class="menuitem">
201<a href="releasenotes.html">Release Notes</a>
202</div>
203<div class="menuitem">
204<a href="changes.html">Change Log</a>
205</div>
206</div>
207<div id="credit"></div>
208<div id="roundbottom">
209<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
210<!--+
211  |alternative credits
212  +-->
213<div id="credit2"></div>
214</div>
215<!--+
216    |end Menu
217    +-->
218<!--+
219    |start content
220    +-->
221<div id="content">
222<div title="Portable Document Format" class="pdflink">
223<a class="dida" href="native_libraries.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
224        PDF</a>
225</div>
226<h1>Native Libraries Guide</h1>
227<div id="minitoc-area">
228<ul class="minitoc">
229<li>
230<a href="#Purpose">Purpose</a>
231</li>
232<li>
233<a href="#Components">Components</a>
234</li>
235<li>
236<a href="#Usage">Usage</a>
237</li>
238<li>
239<a href="#Supported+Platforms">Supported Platforms</a>
240</li>
241<li>
242<a href="#Building+Native+Hadoop+Libraries">Building Native Hadoop Libraries</a>
243<ul class="minitoc">
244<li>
245<a href="#Notes">Notes</a>
246</li>
247</ul>
248</li>
249<li>
250<a href="#Loading+native+libraries+through+DistributedCache"> Loading native libraries through DistributedCache </a>
251</li>
252</ul>
253</div>
254 
255   
256<a name="N1000D"></a><a name="Purpose"></a>
257<h2 class="h3">Purpose</h2>
258<div class="section">
259<p>Hadoop has native implementations of certain components for reasons of
260      both performance and non-availability of Java implementations. These
261      components are available in a single, dynamically-linked, native library.
262      On the *nix platform it is <em>libhadoop.so</em>. This document describes
263      the usage and details on how to build the native libraries.</p>
264</div>
265   
266   
267<a name="N1001A"></a><a name="Components"></a>
268<h2 class="h3">Components</h2>
269<div class="section">
270<p>Hadoop currently has the following
271      <a href="api/org/apache/hadoop/io/compress/CompressionCodec.html">
272      compression codecs</a> as the native components:</p>
273<ul>
274       
275<li>
276<a href="http://www.zlib.net/">zlib</a>
277</li>
278       
279<li>
280<a href="http://www.gzip.org/">gzip</a>
281</li>
282       
283<li>
284<a href="http://www.bzip.org/">bzip2</a>
285</li>
286     
287</ul>
288<p>Of the above, the availability of native hadoop libraries is imperative
289      for the gzip and bzip2 compression codecs to work.</p>
290</div>
291
292   
293<a name="N1003D"></a><a name="Usage"></a>
294<h2 class="h3">Usage</h2>
295<div class="section">
296<p>It is fairly simple to use the native hadoop libraries:</p>
297<ul>
298       
299<li>
300          Take a look at the
301          <a href="#Supported+Platforms">supported platforms</a>.
302        </li>
303       
304<li>
305          Either <a href="http://hadoop.apache.org/core/releases.html#Download">download</a> the pre-built
306          32-bit i386-Linux native hadoop libraries (available as part of hadoop
307          distribution in <span class="codefrag">lib/native</span> directory) or
308          <a href="#Building+Native+Hadoop+Libraries">build</a> them yourself.
309        </li>
310       
311<li>
312          Make sure you have any of or all of <strong>&gt;zlib-1.2</strong>,
313          <strong>&gt;gzip-1.2</strong>, and <strong>&gt;bzip2-1.0</strong>
314          packages for your platform installed;
315          depending on your needs.
316        </li>
317     
318</ul>
319<p>The <span class="codefrag">bin/hadoop</span> script ensures that the native hadoop
320      library is on the library path via the system property
321      <em>-Djava.library.path=&lt;path&gt;</em>.</p>
322<p>To check everything went alright check the hadoop log files for:</p>
323<p>
324       
325<span class="codefrag">
326          DEBUG util.NativeCodeLoader - Trying to load the custom-built
327          native-hadoop library...
328        </span>
329<br>
330       
331<span class="codefrag">
332          INFO  util.NativeCodeLoader - Loaded the native-hadoop library
333        </span>
334     
335</p>
336<p>If something goes wrong, then:</p>
337<p>
338       
339<span class="codefrag">
340          INFO util.NativeCodeLoader - Unable to load native-hadoop library for
341          your platform... using builtin-java classes where applicable
342        </span>
343     
344</p>
345</div>
346   
347   
348<a name="N1008A"></a><a name="Supported+Platforms"></a>
349<h2 class="h3">Supported Platforms</h2>
350<div class="section">
351<p>Hadoop native library is supported only on *nix platforms only.
352      Unfortunately it is known not to work on <a href="http://www.cygwin.com/">Cygwin</a> 
353      and <a href="http://www.apple.com/macosx">Mac OS X</a> and has mainly been used on the
354      GNU/Linux platform.</p>
355<p>It has been tested on the following GNU/Linux distributions:</p>
356<ul>
357       
358<li>
359         
360<a href="http://www.redhat.com/rhel/">RHEL4</a>/<a href="http://fedora.redhat.com/">Fedora</a>
361       
362</li>
363       
364<li>
365<a href="http://www.ubuntu.com/">Ubuntu</a>
366</li>
367       
368<li>
369<a href="http://www.gentoo.org/">Gentoo</a>
370</li>
371     
372</ul>
373<p>On all the above platforms a 32/64 bit Hadoop native library will work
374      with a respective 32/64 bit jvm.</p>
375</div>
376   
377   
378<a name="N100BA"></a><a name="Building+Native+Hadoop+Libraries"></a>
379<h2 class="h3">Building Native Hadoop Libraries</h2>
380<div class="section">
381<p>Hadoop native library is written in
382      <a href="http://en.wikipedia.org/wiki/ANSI_C">ANSI C</a> and built using
383      the GNU autotools-chain (autoconf, autoheader, automake, autoscan, libtool).
384      This means it should be straight-forward to build them on any platform with
385      a standards compliant C compiler and the GNU autotools-chain.
386      See <a href="#Supported+Platforms">supported platforms</a>.</p>
387<p>In particular the various packages you would need on the target
388      platform are:</p>
389<ul>
390       
391<li>
392          C compiler (e.g. <a href="http://gcc.gnu.org/">GNU C Compiler</a>)
393        </li>
394       
395<li>
396          GNU Autools Chain:
397          <a href="http://www.gnu.org/software/autoconf/">autoconf</a>,
398          <a href="http://www.gnu.org/software/automake/">automake</a>,
399          <a href="http://www.gnu.org/software/libtool/">libtool</a>
400       
401</li>
402       
403<li> 
404          zlib-development package (stable version &gt;= 1.2.0)
405        </li>
406     
407</ul>
408<p>Once you have the pre-requisites use the standard <span class="codefrag">build.xml</span> 
409      and pass along the <span class="codefrag">compile.native</span> flag (set to
410      <span class="codefrag">true</span>) to build the native hadoop library:</p>
411<p>
412<span class="codefrag">$ ant -Dcompile.native=true &lt;target&gt;</span>
413</p>
414<p>The native hadoop library is not built by default since not everyone is
415      interested in building them.</p>
416<p>You should see the newly-built native hadoop library in:</p>
417<p>
418<span class="codefrag">$ build/native/&lt;platform&gt;/lib</span>
419</p>
420<p>where &lt;platform&gt; is combination of the system-properties:
421      <span class="codefrag">${os.name}-${os.arch}-${sun.arch.data.model}</span>; for e.g.
422      Linux-i386-32.</p>
423<a name="N1010A"></a><a name="Notes"></a>
424<h3 class="h4">Notes</h3>
425<ul>
426         
427<li>
428            It is <strong>mandatory</strong> to have the
429            zlib, gzip, and bzip2
430            development packages on the target platform for building the
431            native hadoop library; however for deployment it is sufficient to
432            install one of them if you wish to use only one of them.
433          </li>
434         
435<li>
436            It is necessary to have the correct 32/64 libraries of both zlib
437            depending on the 32/64 bit jvm for the target platform for
438            building/deployment of the native hadoop library.
439          </li>
440       
441</ul>
442</div>
443   
444<a name="N1011E"></a><a name="Loading+native+libraries+through+DistributedCache"></a>
445<h2 class="h3"> Loading native libraries through DistributedCache </h2>
446<div class="section">
447<p>User can load native shared libraries through 
448      <a href="mapred_tutorial.html#DistributedCache">DistributedCache</a>
449      for <em>distributing</em> and <em>symlinking</em> the library files</p>
450<p>Here is an example, describing how to distribute the library and
451      load it from map/reduce task. </p>
452<ol>
453     
454<li> First copy the library to the HDFS. <br>
455     
456<span class="codefrag">bin/hadoop fs -copyFromLocal mylib.so.1 /libraries/mylib.so.1</span>
457     
458</li>
459     
460<li> The job launching program should contain the following: <br>
461     
462<span class="codefrag"> DistributedCache.createSymlink(conf); </span> 
463<br>
464     
465<span class="codefrag"> DistributedCache.addCacheFile("hdfs://host:port/libraries/mylib.so.1#mylib.so", conf);
466      </span>
467     
468</li>
469     
470<li> The map/reduce task can contain: <br>
471     
472<span class="codefrag"> System.loadLibrary("mylib.so"); </span>
473     
474</li>
475     
476</ol>
477</div>
478 
479</div>
480<!--+
481    |end content
482    +-->
483<div class="clearboth">&nbsp;</div>
484</div>
485<div id="footer">
486<!--+
487    |start bottomstrip
488    +-->
489<div class="lastmodified">
490<script type="text/javascript"><!--
491document.write("Last Published: " + document.lastModified);
492//  --></script>
493</div>
494<div class="copyright">
495        Copyright &copy;
496         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
497</div>
498<!--+
499    |end bottomstrip
500    +-->
501</div>
502</body>
503</html>
Note: See TracBrowser for help on using the repository browser.