1 | #!/bin/sh |
---|
2 | |
---|
3 | # Licensed to the Apache Software Foundation (ASF) under one or more |
---|
4 | # contributor license agreements. See the NOTICE file distributed with |
---|
5 | # this work for additional information regarding copyright ownership. |
---|
6 | # The ASF licenses this file to You under the Apache License, Version 2.0 |
---|
7 | # (the "License"); you may not use this file except in compliance with |
---|
8 | # the License. You may obtain a copy of the License at |
---|
9 | # |
---|
10 | # http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | # |
---|
12 | # Unless required by applicable law or agreed to in writing, software |
---|
13 | # distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | # See the License for the specific language governing permissions and |
---|
16 | # limitations under the License. |
---|
17 | |
---|
18 | |
---|
19 | """:" |
---|
20 | work_dir=$(dirname $0) |
---|
21 | base_name=$(basename $0) |
---|
22 | original_dir=$PWD |
---|
23 | cd $work_dir |
---|
24 | |
---|
25 | if [ $HOD_PYTHON_HOME ]; then |
---|
26 | exec $HOD_PYTHON_HOME -u -OO $base_name ${1+"$@"} |
---|
27 | elif [ -e /usr/bin/python ]; then |
---|
28 | exec /usr/bin/python -u -OO $base_name ${1+"$@"} |
---|
29 | elif [ -e /usr/local/bin/python ]; then |
---|
30 | exec /usr/local/bin/python -u -OO $base_name ${1+"$@"} |
---|
31 | else |
---|
32 | exec python -u -OO $base_name ${1+"$@"} |
---|
33 | fi |
---|
34 | ":""" |
---|
35 | |
---|
36 | """The executable to be used by the user""" |
---|
37 | import sys, os, re, pwd, threading, sys, random, time, pprint, shutil, time, re |
---|
38 | from pprint import pformat |
---|
39 | from optparse import OptionParser |
---|
40 | |
---|
41 | myName = os.path.basename(sys.argv[0]) |
---|
42 | myName = re.sub(".*/", "", myName) |
---|
43 | binDirectory = os.path.realpath(sys.argv[0]) |
---|
44 | rootDirectory = re.sub("/bin/.*", "", binDirectory) |
---|
45 | libDirectory = rootDirectory |
---|
46 | |
---|
47 | sys.path.append(libDirectory) |
---|
48 | |
---|
49 | from hodlib.Common.threads import simpleCommand |
---|
50 | from hodlib.Common.util import local_fqdn, tar, filter_warnings,\ |
---|
51 | get_exception_string, get_exception_error_string |
---|
52 | from hodlib.Common.logger import hodLog |
---|
53 | from hodlib.Common.logger import getLogger |
---|
54 | from hodlib.HodRing.hodRing import createMRSystemDirectoryManager |
---|
55 | |
---|
56 | filter_warnings() |
---|
57 | |
---|
58 | reVersion = re.compile(".*(\d+_\d+).*") |
---|
59 | reHdfsURI = re.compile("(hdfs://.*?:\d+)(.*)") |
---|
60 | |
---|
61 | VERSION = None |
---|
62 | if os.path.exists("./VERSION"): |
---|
63 | vFile = open("./VERSION", 'r') |
---|
64 | VERSION = vFile.readline() |
---|
65 | vFile.close() |
---|
66 | |
---|
67 | def __archive_logs(conf, log): |
---|
68 | # need log-destination-uri, __hadoopLogDirs, temp-dir |
---|
69 | status = True |
---|
70 | logUri = conf['log-destination-uri'] |
---|
71 | hadoopLogDirs = conf['hadoop-log-dirs'] |
---|
72 | if logUri: |
---|
73 | try: |
---|
74 | if hadoopLogDirs: |
---|
75 | date = time.localtime() |
---|
76 | for logDir in hadoopLogDirs: |
---|
77 | (head, tail) = os.path.split(logDir) |
---|
78 | (head, logType) = os.path.split(head) |
---|
79 | tarBallFile = "%s-%s-%04d%02d%02d%02d%02d%02d-%s.tar.gz" % ( |
---|
80 | logType, local_fqdn(), date[0], date[1], date[2], date[3], |
---|
81 | date[4], date[5], random.randint(0,1000)) |
---|
82 | |
---|
83 | if logUri.startswith('file://'): |
---|
84 | tarBallFile = os.path.join(logUri[7:], |
---|
85 | tarBallFile) |
---|
86 | else: |
---|
87 | tarBallFile = os.path.join(conf['temp-dir'], tarBallFile) |
---|
88 | |
---|
89 | log.debug('archiving log files to: %s' % tarBallFile) |
---|
90 | status = tar(tarBallFile, logDir, ['*',]) |
---|
91 | log.info('archive %s status: %s' % (tarBallFile, status)) |
---|
92 | if status and \ |
---|
93 | logUri.startswith('hdfs://'): |
---|
94 | __copy_archive_to_dfs(conf, tarBallFile) |
---|
95 | log.info("copying archive to dfs finished") |
---|
96 | dict = {} |
---|
97 | except: |
---|
98 | log.error(get_exception_string()) |
---|
99 | status = False |
---|
100 | return status |
---|
101 | |
---|
102 | |
---|
103 | def __copy_archive_to_dfs(conf, archiveFile): |
---|
104 | # need log-destination-uri, hadoopCommandstring and/or pkgs |
---|
105 | hdfsURIMatch = reHdfsURI.match(conf['log-destination-uri']) |
---|
106 | |
---|
107 | (head, tail) = os.path.split(archiveFile) |
---|
108 | destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], 'hod-logs', conf['service-id'], tail) |
---|
109 | |
---|
110 | log.info("copying archive %s to DFS %s ..." % (archiveFile, destFile)) |
---|
111 | |
---|
112 | hadoopCmd = conf['hadoop-command-string'] |
---|
113 | if conf['pkgs']: |
---|
114 | hadoopCmd = os.path.join(conf['pkgs'], 'bin', 'hadoop') |
---|
115 | |
---|
116 | copyCommand = "%s dfs -fs %s -copyFromLocal %s %s" % (hadoopCmd, |
---|
117 | hdfsURIMatch.group(1), archiveFile, destFile) |
---|
118 | |
---|
119 | log.debug(copyCommand) |
---|
120 | |
---|
121 | copyThread = simpleCommand('hadoop', copyCommand) |
---|
122 | copyThread.start() |
---|
123 | copyThread.wait() |
---|
124 | copyThread.join() |
---|
125 | log.debug(pprint.pformat(copyThread.output())) |
---|
126 | |
---|
127 | os.unlink(archiveFile) |
---|
128 | |
---|
129 | def unpack(): |
---|
130 | parser = OptionParser() |
---|
131 | option_list=["--log-destination-uri", "--hadoop-log-dirs", \ |
---|
132 | "--temp-dir", "--hadoop-command-string", "--pkgs", "--user-id", \ |
---|
133 | "--service-id", "--hodring-debug", "--hodring-log-dir", \ |
---|
134 | "--hodring-syslog-address", "--hodring-cleanup-list", \ |
---|
135 | "--jt-pid", "--mr-sys-dir", "--fs-name", "--hadoop-path"] |
---|
136 | regexp = re.compile("^--") |
---|
137 | for opt in option_list: |
---|
138 | parser.add_option(opt,dest=regexp.sub("",opt),action="store") |
---|
139 | option_list.append("--hodring-stream") |
---|
140 | parser.add_option("--hodring-stream",dest="hodring-stream",metavar="bool",\ |
---|
141 | action="store_true") |
---|
142 | (options, args) = parser.parse_args() |
---|
143 | _options= {} |
---|
144 | _options['hodring'] = {} |
---|
145 | for opt in dir(options): |
---|
146 | if "--"+opt in option_list: |
---|
147 | _options[opt] = getattr(options,opt) |
---|
148 | if _options.has_key('hadoop-log-dirs') and _options['hadoop-log-dirs']: |
---|
149 | _options['hadoop-log-dirs'] = _options['hadoop-log-dirs'].split(",") |
---|
150 | if _options.has_key('hodring-syslog-address') and _options['hodring-syslog-address']: |
---|
151 | _options['hodring']['syslog-address'] = \ |
---|
152 | _options['hodring-syslog-address'].split(':') |
---|
153 | _options['hodring']['debug'] = int(_options['hodring-debug']) |
---|
154 | _options['hodring']['log-dir'] = _options['hodring-log-dir'] |
---|
155 | _options['hodring']['stream'] = _options['hodring-stream'] |
---|
156 | _options['hodring']['userid'] = _options['user-id'] |
---|
157 | os.putenv('PBS_JOBID', _options['service-id'] ) |
---|
158 | return _options |
---|
159 | |
---|
160 | if __name__ == '__main__': |
---|
161 | log = None |
---|
162 | try: |
---|
163 | conf = unpack() |
---|
164 | # Use the same log as hodring |
---|
165 | log = getLogger(conf['hodring'],'hodring') |
---|
166 | log.debug("Logger initialised successfully") |
---|
167 | mrSysDirManager = createMRSystemDirectoryManager(conf, log) |
---|
168 | if mrSysDirManager is not None: |
---|
169 | mrSysDirManager.removeMRSystemDirectory() |
---|
170 | |
---|
171 | status = __archive_logs(conf,log) |
---|
172 | log.info("Archive status : %s" % status) |
---|
173 | list = conf['hodring-cleanup-list'].split(',') |
---|
174 | log.info("now removing %s" % list) |
---|
175 | for dir in list: |
---|
176 | if os.path.exists(dir): |
---|
177 | log.debug('removing %s' % (dir)) |
---|
178 | shutil.rmtree(dir, True) |
---|
179 | log.debug("done") |
---|
180 | log.info("Cleanup successfully completed") |
---|
181 | except Exception, e: |
---|
182 | if log: |
---|
183 | log.info("Stack trace:\n%s\n%s" %(get_exception_error_string(),get_exception_string())) |
---|