source: proiecte/HadoopJUnit/hadoop-0.20.1/contrib/hod/bin/hodring @ 120

Last change on this file since 120 was 120, checked in by (none), 14 years ago

Added the mail files for the Hadoop JUNit Project
  • Property svn:executable set to *
File size: 10.0 KB
Line 
1#!/bin/sh
2
3# Licensed to the Apache Software Foundation (ASF) under one or more
4# contributor license agreements.  See the NOTICE file distributed with
5# this work for additional information regarding copyright ownership.
6# The ASF licenses this file to You under the Apache License, Version 2.0
7# (the "License"); you may not use this file except in compliance with
8# the License.  You may obtain a copy of the License at
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17
18
19""":"
20work_dir=$(dirname $0)
21base_name=$(basename $0)
22cd $work_dir
23
24if [ $HOD_PYTHON_HOME ]; then
25    exec $HOD_PYTHON_HOME -OO $base_name ${1+"$@"}
26elif [ -e /usr/bin/python ]; then
27    exec /usr/bin/python -OO $base_name ${1+"$@"}
28elif [ -e /usr/local/bin/python ]; then
29    exec /usr/local/bin/python -OO $base_name ${1+"$@"}
30else
31    exec python -OO $base_name ${1+"$@"}
32fi
33":"""
34
35"""The executable to be used by the user"""
36import sys, os, re
37
38
39myName          = os.path.basename(sys.argv[0])
40myName          = re.sub(".*/", "", myName)
41binDirectory    = os.path.realpath(sys.argv[0])
42rootDirectory   = re.sub("/bin/.*", "", binDirectory)
43libDirectory    = rootDirectory
44
45sys.path.append(libDirectory)
46
47from hodlib.HodRing.hodRing import HodRing
48from hodlib.Common.setup import *
49from hodlib.Common.util import filter_warnings, get_exception_string, \
50                get_exception_error_string, getMapredSystemDirectory, \
51                to_http_url, local_fqdn
52from hodlib.Common.logger import getLogger, ensureLogDir
53from hodlib.Common.xmlrpc import hodXRClient
54
55filter_warnings()
56
57reVersion = re.compile(".*(\d+_\d+).*")
58
59VERSION = '$HeadURL$'
60
61reMatch = reVersion.match(VERSION)
62if reMatch:
63    VERSION = reMatch.group(1)
64    VERSION = re.sub("_", ".", VERSION)
65else:
66    VERSION = 'DEV'
67
68# Definition tuple is of the form:
69#  (name, type, description, default value, required?, validate?)
70#
71defList = { 'hodring' : (
72             ('temp-dir', 'directory', 'hod work directories',
73              False, None, True, False),
74             
75             ('log-dir', 'directory', 'hod logging directory.',
76              False, os.path.join(rootDirectory, 'logs'), False, True),
77
78             ('log-destination-uri', 'string',
79              'URI to store logs to, local://some_path or '
80              + 'hdfs://host:port/some_path',
81              False, None, False, True),
82
83             ('pkgs', 'directory', 'Path to Hadoop to use in case of uploading to HDFS',
84              False, None, False, True),
85             
86             ('syslog-address', 'address', 'Syslog address.',
87              False, None, False, True),
88         
89             ('java-home', 'directory', 'Java home directory.',
90              False, None, True, True),
91             
92             ('debug', 'pos_int', 'Debugging level, 0-4.',
93              False, 3, True, True),
94               
95             ('register', 'bool', 'Register with service registry?',
96              False, True, True, True),
97               
98             ('stream', 'bool', 'Output to stderr.',
99              False, False, False, True),
100
101             ('userid', 'user_account',
102              'User ID the hod shell is running under.',
103              False, None, True, False),
104
105             ('xrs-port-range', 'range', 'XML-RPC port range n-m.',
106              False, None, True, True),
107               
108             ('http-port-range', 'range', 'HTTP port range n-m.',
109              False, None, True, True),
110               
111             ('command', 'string', 'Command for hodring to run.',
112              False, None, False, True),
113             
114             ('service-id', 'string', 'Service ID.',
115              False, None, False, True),
116             
117             ('download-addr', 'string', 'Download HTTP address.',
118              False, None, False, True),
119               
120             ('svcrgy-addr', 'address', 'Service registry XMLRPC address.',
121              False, None, True, True),
122   
123             ('ringmaster-xrs-addr', 'address', 'Ringmaster XML-RPC address.',
124              False, None, False, True),
125 
126             ('tarball-retry-initial-time', 'pos_float','initial retry time for tarball download',
127              False, 1, False, True),
128             
129             ('tarball-retry-interval', 'pos_float','interval to spread retries for tarball download',
130              False, 3, False, True),
131             
132             ('cmd-retry-initial-time', 'pos_float','initial retry time for getting commands',
133              False, 2, False, True),
134             
135             ('cmd-retry-interval', 'pos_float','interval to spread retries for getting commands',
136              False, 2, False, True),
137
138             ('mapred-system-dir-root', 'string', 'Root under which mapreduce system directory names are generated by HOD.',
139              False, '/mapredsystem', False, False))
140            }           
141
142if __name__ == '__main__':
143
144  confDef = definition()
145  confDef.add_defs(defList)
146  hodRingOptions = options(confDef, "./%s [OPTIONS]" % myName, VERSION)
147  ensureLogDir(hodRingOptions['hodring']['log-dir'])
148  service = None
149  try:
150    (status, statusMsgs) = hodRingOptions.verify()
151    if not status:
152      raise Exception("%s" % statusMsgs)
153    hodRingOptions['hodring']['base-dir'] = rootDirectory
154    service = HodRing(hodRingOptions)
155    service.start()
156    service.wait()
157   
158    if service.log:
159      log = service.log
160    else:
161      log = getLogger(hodRingOptions['hodring'],'hodring')
162
163    list = []
164   
165    runningHadoops = service.getRunningValues()
166
167    mrSysDirManager = None     
168    for cmd in runningHadoops:
169      if cmd.name == 'jobtracker':
170        mrSysDirManager = cmd.getMRSystemDirectoryManager()
171      log.debug("addding %s to cleanup list..." % cmd)
172      cmd.addCleanup(list)
173   
174    list.append(service.getTempDir())
175    log.debug(list)
176       
177    # archive_logs now
178    cmdString = os.path.join(rootDirectory, "bin", "hodcleanup") # same python
179
180    if (len(runningHadoops) == 0):
181      log.info("len(runningHadoops) == 0, No running cluster?")
182      log.info("Skipping __copy_archive_to_dfs")
183      hadoopString = ""
184    else: hadoopString=runningHadoops[0].path
185
186    #construct the arguments
187    if hodRingOptions['hodring'].has_key('log-destination-uri'):
188      cmdString = cmdString + " --log-destination-uri " \
189                    + hodRingOptions['hodring']['log-destination-uri']
190
191    hadoopLogDirs = service.getHadoopLogDirs()
192    if hadoopLogDirs:
193      cmdString = cmdString \
194                    + " --hadoop-log-dirs " \
195                    + ",".join(hadoopLogDirs)
196
197    cmdString = cmdString \
198                  + " --temp-dir " \
199                  + service._cfg['temp-dir'] \
200                  + " --hadoop-command-string " \
201                  + hadoopString \
202                  + " --user-id " \
203                  + service._cfg['userid'] \
204                  + " --service-id " \
205                  + service._cfg['service-id'] \
206                  + " --hodring-debug " \
207                  + str(hodRingOptions['hodring']['debug']) \
208                  + " --hodring-log-dir " \
209                  + hodRingOptions['hodring']['log-dir'] \
210                  + " --hodring-cleanup-list " \
211                  + ",".join(list)
212
213    if hodRingOptions['hodring'].has_key('syslog-address'):
214      syslogAddr = hodRingOptions['hodring']['syslog-address'][0] + \
215                   ':' + str(hodRingOptions['hodring']['syslog-address'][1])
216      cmdString = cmdString + " --hodring-syslog-address " + syslogAddr
217    if service._cfg.has_key('pkgs'):
218      cmdString = cmdString + " --pkgs " + service._cfg['pkgs']
219
220    if mrSysDirManager is not None:
221      cmdString = "%s %s" % (cmdString, mrSysDirManager.toCleanupArgs())
222
223    log.info("cleanup commandstring : ")
224    log.info(cmdString)
225
226    # clean up
227    cmd = ['/bin/sh', '-c', cmdString]
228
229    mswindows = (sys.platform == "win32")
230    originalcwd = os.getcwd()
231
232    if not mswindows:
233      try:
234        pid = os.fork() 
235        if pid > 0:
236          # exit first parent
237          log.info("child(pid: %s) is now doing cleanup" % pid)
238          sys.exit(0) 
239      except OSError, e:
240        log.error("fork failed: %d (%s)" % (e.errno, e.strerror)) 
241        sys.exit(1)
242
243      # decouple from parent environment
244      os.chdir("/") 
245      os.setsid() 
246      os.umask(0) 
247 
248    MAXFD = 128 # more than enough file descriptors to close. Just in case.
249    for i in xrange(0, MAXFD):
250      try:
251        os.close(i)
252      except OSError:
253        pass
254 
255    try:
256      os.execvp(cmd[0], cmd)
257    finally:
258      log.critical("exec failed")
259      os._exit(1)
260
261  except Exception, e:
262    if service:
263      if service.log:
264        log = service.log
265    else:
266      log = getLogger(hodRingOptions['hodring'], 'hodring')
267    log.error("Error in bin/hodring %s. \nStack trace:\n%s" %(get_exception_error_string(),get_exception_string()))
268   
269    log.info("now trying informing to ringmaster")
270    log.info(hodRingOptions['hodring']['ringmaster-xrs-addr'])
271    log.info(hodRingOptions.normalizeValue('hodring', 'ringmaster-xrs-addr'))
272    log.info(to_http_url(hodRingOptions.normalizeValue( \
273            'hodring', 'ringmaster-xrs-addr')))
274    # Report errors to the Ringmaster if possible
275    try:
276      ringXRAddress = to_http_url(hodRingOptions.normalizeValue( \
277                                     'hodring', 'ringmaster-xrs-addr'))
278      log.debug("Creating ringmaster XML-RPC client.")
279      ringClient = hodXRClient(ringXRAddress)   
280      if ringClient is not None:
281        addr = local_fqdn() + "_" + str(os.getpid())
282        ringClient.setHodRingErrors(addr, str(e))
283        log.info("Reported errors to ringmaster at %s" % ringXRAddress)
284    except Exception, e:
285      log.error("Failed to report errors to ringmaster at %s" % ringXRAddress)
286      log.error("Reason : %s" % get_exception_string())
287    # End of reporting errors to the client
Note: See TracBrowser for help on using the repository browser.