source: proiecte/HadoopJUnit/hadoop-0.20.1/docs/hod_config_guide.html @ 120

Last change on this file since 120 was 120, checked in by (none), 14 years ago

Added the mail files for the Hadoop JUNit Project
  • Property svn:executable set to *
File size: 24.7 KB
Line 
1<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2<html>
3<head>
4<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
5<meta content="Apache Forrest" name="Generator">
6<meta name="Forrest-version" content="0.8">
7<meta name="Forrest-skin-name" content="pelt">
8<title> 
9      HOD Configuration Guide
10    </title>
11<link type="text/css" href="skin/basic.css" rel="stylesheet">
12<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
13<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
14<link type="text/css" href="skin/profile.css" rel="stylesheet">
15<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
16<link rel="shortcut icon" href="images/favicon.ico">
17</head>
18<body onload="init()">
19<script type="text/javascript">ndeSetTextSize();</script>
20<div id="top">
21<!--+
22    |breadtrail
23    +-->
24<div class="breadtrail">
25<a href="http://www.apache.org/">Apache</a> &gt; <a href="http://hadoop.apache.org/">Hadoop</a> &gt; <a href="http://hadoop.apache.org/core/">Core</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script>
26</div>
27<!--+
28    |header
29    +-->
30<div class="header">
31<!--+
32    |start group logo
33    +-->
34<div class="grouplogo">
35<a href="http://hadoop.apache.org/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a>
36</div>
37<!--+
38    |end group logo
39    +-->
40<!--+
41    |start Project Logo
42    +-->
43<div class="projectlogo">
44<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/core-logo.gif" title="Scalable Computing Platform"></a>
45</div>
46<!--+
47    |end Project Logo
48    +-->
49<!--+
50    |start Search
51    +-->
52<div class="searchbox">
53<form action="http://www.google.com/search" method="get" class="roundtopsmall">
54<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google">&nbsp; 
55                    <input name="Search" value="Search" type="submit">
56</form>
57</div>
58<!--+
59    |end search
60    +-->
61<!--+
62    |start Tabs
63    +-->
64<ul id="tabs">
65<li>
66<a class="unselected" href="http://hadoop.apache.org/core/">Project</a>
67</li>
68<li>
69<a class="unselected" href="http://wiki.apache.org/hadoop">Wiki</a>
70</li>
71<li class="current">
72<a class="selected" href="index.html">Hadoop 0.20 Documentation</a>
73</li>
74</ul>
75<!--+
76    |end Tabs
77    +-->
78</div>
79</div>
80<div id="main">
81<div id="publishedStrip">
82<!--+
83    |start Subtabs
84    +-->
85<div id="level2tabs"></div>
86<!--+
87    |end Endtabs
88    +-->
89<script type="text/javascript"><!--
90document.write("Last Published: " + document.lastModified);
91//  --></script>
92</div>
93<!--+
94    |breadtrail
95    +-->
96<div class="breadtrail">
97
98             &nbsp;
99           </div>
100<!--+
101    |start Menu, mainarea
102    +-->
103<!--+
104    |start Menu
105    +-->
106<div id="menu">
107<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Getting Started</div>
108<div id="menu_1.1" class="menuitemgroup">
109<div class="menuitem">
110<a href="index.html">Overview</a>
111</div>
112<div class="menuitem">
113<a href="quickstart.html">Quick Start</a>
114</div>
115<div class="menuitem">
116<a href="cluster_setup.html">Cluster Setup</a>
117</div>
118<div class="menuitem">
119<a href="mapred_tutorial.html">Map/Reduce Tutorial</a>
120</div>
121</div>
122<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Programming Guides</div>
123<div id="menu_1.2" class="menuitemgroup">
124<div class="menuitem">
125<a href="commands_manual.html">Commands</a>
126</div>
127<div class="menuitem">
128<a href="distcp.html">DistCp</a>
129</div>
130<div class="menuitem">
131<a href="native_libraries.html">Native Libraries</a>
132</div>
133<div class="menuitem">
134<a href="streaming.html">Streaming</a>
135</div>
136<div class="menuitem">
137<a href="fair_scheduler.html">Fair Scheduler</a>
138</div>
139<div class="menuitem">
140<a href="capacity_scheduler.html">Capacity Scheduler</a>
141</div>
142<div class="menuitem">
143<a href="service_level_auth.html">Service Level Authorization</a>
144</div>
145<div class="menuitem">
146<a href="vaidya.html">Vaidya</a>
147</div>
148<div class="menuitem">
149<a href="hadoop_archives.html">Archives</a>
150</div>
151</div>
152<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">HDFS</div>
153<div id="menu_1.3" class="menuitemgroup">
154<div class="menuitem">
155<a href="hdfs_user_guide.html">User Guide</a>
156</div>
157<div class="menuitem">
158<a href="hdfs_design.html">Architecture</a>
159</div>
160<div class="menuitem">
161<a href="hdfs_shell.html">File System Shell Guide</a>
162</div>
163<div class="menuitem">
164<a href="hdfs_permissions_guide.html">Permissions Guide</a>
165</div>
166<div class="menuitem">
167<a href="hdfs_quota_admin_guide.html">Quotas Guide</a>
168</div>
169<div class="menuitem">
170<a href="SLG_user_guide.html">Synthetic Load Generator Guide</a>
171</div>
172<div class="menuitem">
173<a href="libhdfs.html">C API libhdfs</a>
174</div>
175</div>
176<div onclick="SwitchMenu('menu_selected_1.4', 'skin/')" id="menu_selected_1.4Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">HOD</div>
177<div id="menu_selected_1.4" class="selectedmenuitemgroup" style="display: block;">
178<div class="menuitem">
179<a href="hod_user_guide.html">User Guide</a>
180</div>
181<div class="menuitem">
182<a href="hod_admin_guide.html">Admin Guide</a>
183</div>
184<div class="menupage">
185<div class="menupagetitle">Config Guide</div>
186</div>
187</div>
188<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div>
189<div id="menu_1.5" class="menuitemgroup">
190<div class="menuitem">
191<a href="api/index.html">API Docs</a>
192</div>
193<div class="menuitem">
194<a href="jdiff/changes.html">API Changes</a>
195</div>
196<div class="menuitem">
197<a href="http://wiki.apache.org/hadoop/">Wiki</a>
198</div>
199<div class="menuitem">
200<a href="http://wiki.apache.org/hadoop/FAQ">FAQ</a>
201</div>
202<div class="menuitem">
203<a href="releasenotes.html">Release Notes</a>
204</div>
205<div class="menuitem">
206<a href="changes.html">Change Log</a>
207</div>
208</div>
209<div id="credit"></div>
210<div id="roundbottom">
211<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
212<!--+
213  |alternative credits
214  +-->
215<div id="credit2"></div>
216</div>
217<!--+
218    |end Menu
219    +-->
220<!--+
221    |start content
222    +-->
223<div id="content">
224<div title="Portable Document Format" class="pdflink">
225<a class="dida" href="hod_config_guide.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br>
226        PDF</a>
227</div>
228<h1> 
229      HOD Configuration Guide
230    </h1>
231<div id="minitoc-area">
232<ul class="minitoc">
233<li>
234<a href="#1.+Introduction">1. Introduction</a>
235</li>
236<li>
237<a href="#2.+Sections">2. Sections</a>
238</li>
239<li>
240<a href="#3.+HOD+Configuration+Options">3. HOD Configuration Options</a>
241<ul class="minitoc">
242<li>
243<a href="#3.1+Common+configuration+options">3.1 Common configuration options</a>
244</li>
245<li>
246<a href="#3.2+hod+options">3.2 hod options</a>
247</li>
248<li>
249<a href="#3.3+resource_manager+options">3.3 resource_manager options</a>
250</li>
251<li>
252<a href="#3.4+ringmaster+options">3.4 ringmaster options</a>
253</li>
254<li>
255<a href="#3.5+gridservice-hdfs+options">3.5 gridservice-hdfs options</a>
256</li>
257<li>
258<a href="#3.6+gridservice-mapred+options">3.6 gridservice-mapred options</a>
259</li>
260<li>
261<a href="#3.7+hodring+options">3.7 hodring options</a>
262</li>
263</ul>
264</li>
265</ul>
266</div>
267   
268<a name="N1000C"></a><a name="1.+Introduction"></a>
269<h2 class="h3">1. Introduction</h2>
270<div class="section">
271<p>This guide discusses Hadoop on Demand (HOD) configuration sections and shows you how to work with the most important
272      and commonly used HOD configuration options.</p>
273<p>Configuration options
274      can be specified in two ways: a configuration file
275      in the INI format, and as command line options to the HOD shell,
276      specified in the format --section.option[=value]. If the same option is
277      specified in both places, the value specified on the command line
278      overrides the value in the configuration file.</p>
279<p>
280        To get a simple description of all configuration options, type:
281      </p>
282<table class="ForrestTable" cellspacing="1" cellpadding="4">
283<tr>
284<td colspan="1" rowspan="1"><span class="codefrag">$ hod --verbose-help</span></td>
285</tr>
286</table>
287</div>
288   
289   
290<a name="N10024"></a><a name="2.+Sections"></a>
291<h2 class="h3">2. Sections</h2>
292<div class="section">
293<p>HOD organizes configuration options into these sections:</p>
294<ul>
295       
296<li>  hod:                  Options for the HOD client</li>
297       
298<li>  resource_manager:     Options for specifying which resource manager
299         to use, and other parameters for using that resource manager</li>
300       
301<li>  ringmaster:           Options for the RingMaster process, </li>
302       
303<li>  hodring:              Options for the HodRing processes</li>
304       
305<li>  gridservice-mapred:   Options for the Map/Reduce daemons</li>
306       
307<li>  gridservice-hdfs:     Options for the HDFS daemons.</li>
308     
309</ul>
310</div>
311   
312   
313<a name="N10043"></a><a name="3.+HOD+Configuration+Options"></a>
314<h2 class="h3">3. HOD Configuration Options</h2>
315<div class="section">
316<p>The following section describes configuration options common to most
317      HOD sections followed by sections that describe configuration options
318      specific to each HOD section.</p>
319<a name="N1004C"></a><a name="3.1+Common+configuration+options"></a>
320<h3 class="h4">3.1 Common configuration options</h3>
321<p>Certain configuration options are defined in most of the sections of
322        the HOD configuration. Options defined in a section, are used by the
323        process for which that section applies. These options have the same
324        meaning, but can have different values in each section.
325        </p>
326<ul>
327         
328<li>temp-dir: Temporary directory for usage by the HOD processes. Make
329                      sure that the users who will run hod have rights to create
330                      directories under the directory specified here. If you
331                      wish to make this directory vary across allocations,
332                      you can make use of the environmental variables which will
333                      be made available by the resource manager to the HOD
334                      processes. For example, in a Torque setup, having
335                      --ringmaster.temp-dir=/tmp/hod-temp-dir.$PBS_JOBID would
336                      let ringmaster use different temp-dir for each
337                      allocation; Torque expands this variable before starting
338                      the ringmaster.</li>
339         
340         
341<li>debug: Numeric value from 1-4. 4 produces the most log information,
342                   and 1 the least.</li>
343         
344         
345<li>log-dir: Directory where log files are stored. By default, this is
346                     &lt;install-location&gt;/logs/. The restrictions and notes for the
347                     temp-dir variable apply here too.
348          </li>
349         
350         
351<li>xrs-port-range: Range of ports, among which an available port shall
352                            be picked for use to run an XML-RPC server.</li>
353         
354         
355<li>http-port-range: Range of ports, among which an available port shall
356                             be picked for use to run an HTTP server.</li>
357         
358         
359<li>java-home: Location of Java to be used by Hadoop.</li>
360         
361<li>syslog-address: Address to which a syslog daemon is bound to. The format
362                              of the value is host:port. If configured, HOD log messages
363                              will be logged to syslog using this value.</li>
364                             
365       
366</ul>
367<a name="N1006E"></a><a name="3.2+hod+options"></a>
368<h3 class="h4">3.2 hod options</h3>
369<ul>
370         
371<li>cluster: Descriptive name given to the cluster. For Torque, this is
372                     specified as a 'Node property' for every node in the cluster.
373                     HOD uses this value to compute the number of available nodes.</li>
374         
375         
376<li>client-params: Comma-separated list of hadoop config parameters
377                           specified as key-value pairs. These will be used to
378                           generate a hadoop-site.xml on the submit node that
379                           should be used for running Map/Reduce jobs.</li>
380         
381<li>job-feasibility-attr: Regular expression string that specifies
382                           whether and how to check job feasibility - resource
383                           manager or scheduler limits. The current
384                           implementation corresponds to the torque job
385                           attribute 'comment' and by default is disabled.
386                           When set, HOD uses it to decide what type
387                           of limit violation is triggered and either
388                           deallocates the cluster or stays in queued state
389                           according as the request is beyond maximum limits or
390                           the cumulative usage has crossed maximum limits.
391                           The torque comment attribute may be updated
392                           periodically by an external mechanism. For example,
393                           comment attribute can be updated by running <a href="hod_admin_guide.html#checklimits.sh+-+Tool+to+update+torque+comment+field+reflecting+resource+limits">
394                           checklimits.sh</a> script in hod/support directory,
395                           and then setting job-feasibility-attr equal to the
396                           value TORQUE_USER_LIMITS_COMMENT_FIELD,
397                           "User-limits exceeded. Requested:([0-9]*)
398                           Used:([0-9]*) MaxLimit:([0-9]*)", will make HOD
399                           behave accordingly.
400                           </li>
401         
402</ul>
403<a name="N10085"></a><a name="3.3+resource_manager+options"></a>
404<h3 class="h4">3.3 resource_manager options</h3>
405<ul>
406         
407<li>queue: Name of the queue configured in the resource manager to which
408                   jobs are to be submitted.</li>
409         
410         
411<li>batch-home: Install directory to which 'bin' is appended and under
412                        which the executables of the resource manager can be
413                        found.</li> 
414         
415         
416<li>env-vars: Comma-separated list of key-value pairs,
417                      expressed as key=value, which would be passed to the jobs
418                      launched on the compute nodes.
419                      For example, if the python installation is
420                      in a non-standard location, one can set the environment
421                      variable 'HOD_PYTHON_HOME' to the path to the python
422                      executable. The HOD processes launched on the compute nodes
423                      can then use this variable.</li>
424         
425<li>options: Comma-separated list of key-value pairs,
426                      expressed as
427                      &lt;option&gt;:&lt;sub-option&gt;=&lt;value&gt;. When
428                      passing to the job submission program, these are expanded
429                      as -&lt;option&gt; &lt;sub-option&gt;=&lt;value&gt;. These
430                      are generally used for specifying additional resource
431                      contraints for scheduling. For instance, with a Torque
432                      setup, one can specify
433                      --resource_manager.options='l:arch=x86_64' for
434                      constraining the nodes being allocated to a particular
435                      architecture; this option will be passed to Torque's qsub
436                      command as "-l arch=x86_64".</li>
437       
438</ul>
439<a name="N1009B"></a><a name="3.4+ringmaster+options"></a>
440<h3 class="h4">3.4 ringmaster options</h3>
441<ul>
442         
443<li>work-dirs: Comma-separated list of paths that will serve
444                       as the root for directories that HOD generates and passes
445                       to Hadoop for use to store DFS and Map/Reduce data. For
446                       example,
447                       this is where DFS data blocks will be stored. Typically,
448                       as many paths are specified as there are disks available
449                       to ensure all disks are being utilized. The restrictions
450                       and notes for the temp-dir variable apply here too.</li>
451         
452<li>max-master-failures: Number of times a hadoop master
453                       daemon can fail to launch, beyond which HOD will fail
454                       the cluster allocation altogether. In HOD clusters,
455                       sometimes there might be a single or few "bad" nodes due
456                       to issues like missing java, missing or incorrect version
457                       of Hadoop etc. When this configuration variable is set
458                       to a positive integer, the RingMaster returns an error
459                       to the client only when the number of times a hadoop
460                       master (JobTracker or NameNode) fails to start on these
461                       bad nodes because of above issues, exceeds the specified
462                       value. If the number is not exceeded, the next HodRing
463                       which requests for a command to launch is given the same
464                       hadoop master again. This way, HOD tries its best for a
465                       successful allocation even in the presence of a few bad
466                       nodes in the cluster.
467                       </li>
468         
469<li>workers_per_ring: Number of workers per service per HodRing.
470                       By default this is set to 1. If this configuration
471                       variable is set to a value 'n', the HodRing will run
472                       'n' instances of the workers (TaskTrackers or DataNodes)
473                       on each node acting as a slave. This can be used to run
474                       multiple workers per HodRing, so that the total number of
475                       workers  in a HOD cluster is not limited by the total
476                       number of nodes requested during allocation. However, note
477                       that this will mean each worker should be configured to use
478                       only a proportional fraction of the capacity of the
479                       resources on the node. In general, this feature is only
480                       useful for testing and simulation purposes, and not for
481                       production use.</li>
482       
483</ul>
484<a name="N100AE"></a><a name="3.5+gridservice-hdfs+options"></a>
485<h3 class="h4">3.5 gridservice-hdfs options</h3>
486<ul>
487         
488<li>external: If false, indicates that a HDFS cluster must be
489                      bought up by the HOD system, on the nodes which it
490                      allocates via the allocate command. Note that in that case,
491                      when the cluster is de-allocated, it will bring down the
492                      HDFS cluster, and all the data will be lost.
493                      If true, it will try and connect to an externally configured
494                      HDFS system.
495                      Typically, because input for jobs are placed into HDFS
496                      before jobs are run, and also the output from jobs in HDFS
497                      is required to be persistent, an internal HDFS cluster is
498                      of little value in a production system. However, it allows
499                      for quick testing.</li>
500         
501         
502<li>host: Hostname of the externally configured NameNode, if any</li>
503         
504         
505<li>fs_port: Port to which NameNode RPC server is bound.</li>
506         
507         
508<li>info_port: Port to which the NameNode web UI server is bound.</li>
509         
510         
511<li>pkgs: Installation directory, under which bin/hadoop executable is
512                  located. This can be used to use a pre-installed version of
513                  Hadoop on the cluster.</li>
514         
515         
516<li>server-params: Comma-separated list of hadoop config parameters
517                           specified key-value pairs. These will be used to
518                           generate a hadoop-site.xml that will be used by the
519                           NameNode and DataNodes.</li>
520         
521         
522<li>final-server-params: Same as above, except they will be marked final.</li>
523       
524</ul>
525<a name="N100CD"></a><a name="3.6+gridservice-mapred+options"></a>
526<h3 class="h4">3.6 gridservice-mapred options</h3>
527<ul>
528         
529<li>external: If false, indicates that a Map/Reduce cluster must be
530                      bought up by the HOD system on the nodes which it allocates
531                      via the allocate command.
532                      If true, if will try and connect to an externally
533                      configured Map/Reduce system.</li>
534         
535         
536<li>host: Hostname of the externally configured JobTracker, if any</li>
537         
538         
539<li>tracker_port: Port to which the JobTracker RPC server is bound</li>
540         
541         
542<li>info_port: Port to which the JobTracker web UI server is bound.</li>
543         
544         
545<li>pkgs: Installation directory, under which bin/hadoop executable is
546                  located</li>
547         
548         
549<li>server-params: Comma-separated list of hadoop config parameters
550                           specified key-value pairs. These will be used to
551                           generate a hadoop-site.xml that will be used by the
552                           JobTracker and TaskTrackers</li>
553         
554         
555<li>final-server-params: Same as above, except they will be marked final.</li>
556       
557</ul>
558<a name="N100EC"></a><a name="3.7+hodring+options"></a>
559<h3 class="h4">3.7 hodring options</h3>
560<ul>
561         
562<li>mapred-system-dir-root: Directory in the DFS under which HOD will
563                                      generate sub-directory names and pass the full path
564                                      as the value of the 'mapred.system.dir' configuration
565                                      parameter to Hadoop daemons. The format of the full
566                                      path will be value-of-this-option/userid/mapredsystem/cluster-id.
567                                      Note that the directory specified here should be such
568                                      that all users can create directories under this, if
569                                      permissions are enabled in HDFS. Setting the value of
570                                      this option to /user will make HOD use the user's
571                                      home directory to generate the mapred.system.dir value.</li>
572
573         
574<li>log-destination-uri: URL describing a path in an external, static DFS or the
575                                   cluster node's local file system where HOD will upload
576                                   Hadoop logs when a cluster is deallocated. To specify a
577                                   DFS path, use the format 'hdfs://path'. To specify a
578                                   cluster node's local file path, use the format 'file://path'.
579
580                                   When clusters are deallocated by HOD, the hadoop logs will
581                                   be deleted as part of HOD's cleanup process. To ensure these
582                                   logs persist, you can use this configuration option.
583
584                                   The format of the path is
585                                   value-of-this-option/userid/hod-logs/cluster-id
586
587                                   Note that the directory you specify here must be such that all
588                                   users can create sub-directories under this. Setting this value
589                                   to hdfs://user will make the logs come in the user's home directory
590                                   in DFS.</li>
591
592         
593<li>pkgs: Installation directory, under which bin/hadoop executable is located. This will
594                    be used by HOD to upload logs if a HDFS URL is specified in log-destination-uri
595                    option. Note that this is useful if the users are using a tarball whose version
596                    may differ from the external, static HDFS version.</li>
597                                     
598       
599</ul>
600</div>
601 
602</div>
603<!--+
604    |end content
605    +-->
606<div class="clearboth">&nbsp;</div>
607</div>
608<div id="footer">
609<!--+
610    |start bottomstrip
611    +-->
612<div class="lastmodified">
613<script type="text/javascript"><!--
614document.write("Last Published: " + document.lastModified);
615//  --></script>
616</div>
617<div class="copyright">
618        Copyright &copy;
619         2008 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.</a>
620</div>
621<!--+
622    |end bottomstrip
623    +-->
624</div>
625</body>
626</html>
Note: See TracBrowser for help on using the repository browser.