[120] | 1 | #!/bin/sh |
---|
| 2 | |
---|
| 3 | # Licensed to the Apache Software Foundation (ASF) under one or more |
---|
| 4 | # contributor license agreements. See the NOTICE file distributed with |
---|
| 5 | # this work for additional information regarding copyright ownership. |
---|
| 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 |
---|
| 7 | # (the "License"); you may not use this file except in compliance with |
---|
| 8 | # the License. You may obtain a copy of the License at |
---|
| 9 | # |
---|
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | # |
---|
| 12 | # Unless required by applicable law or agreed to in writing, software |
---|
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | # See the License for the specific language governing permissions and |
---|
| 16 | # limitations under the License. |
---|
| 17 | |
---|
| 18 | |
---|
| 19 | """:" |
---|
| 20 | work_dir=$(dirname $0) |
---|
| 21 | base_name=$(basename $0) |
---|
| 22 | original_dir=$PWD |
---|
| 23 | cd $work_dir |
---|
| 24 | |
---|
| 25 | if [ $HOD_PYTHON_HOME ]; then |
---|
| 26 | exec $HOD_PYTHON_HOME -u -OO $base_name ${1+"$@"} |
---|
| 27 | elif [ -e /usr/bin/python ]; then |
---|
| 28 | exec /usr/bin/python -u -OO $base_name ${1+"$@"} |
---|
| 29 | elif [ -e /usr/local/bin/python ]; then |
---|
| 30 | exec /usr/local/bin/python -u -OO $base_name ${1+"$@"} |
---|
| 31 | else |
---|
| 32 | exec python -u -OO $base_name ${1+"$@"} |
---|
| 33 | fi |
---|
| 34 | ":""" |
---|
| 35 | |
---|
| 36 | """The executable to be used by the user""" |
---|
| 37 | import sys, os, re, pwd, threading, sys, random, time, pprint, shutil, time, re |
---|
| 38 | from pprint import pformat |
---|
| 39 | from optparse import OptionParser |
---|
| 40 | |
---|
| 41 | myName = os.path.basename(sys.argv[0]) |
---|
| 42 | myName = re.sub(".*/", "", myName) |
---|
| 43 | binDirectory = os.path.realpath(sys.argv[0]) |
---|
| 44 | rootDirectory = re.sub("/bin/.*", "", binDirectory) |
---|
| 45 | libDirectory = rootDirectory |
---|
| 46 | |
---|
| 47 | sys.path.append(libDirectory) |
---|
| 48 | |
---|
| 49 | from hodlib.Common.threads import simpleCommand |
---|
| 50 | from hodlib.Common.util import local_fqdn, tar, filter_warnings,\ |
---|
| 51 | get_exception_string, get_exception_error_string |
---|
| 52 | from hodlib.Common.logger import hodLog |
---|
| 53 | from hodlib.Common.logger import getLogger |
---|
| 54 | from hodlib.HodRing.hodRing import createMRSystemDirectoryManager |
---|
| 55 | |
---|
| 56 | filter_warnings() |
---|
| 57 | |
---|
| 58 | reVersion = re.compile(".*(\d+_\d+).*") |
---|
| 59 | reHdfsURI = re.compile("(hdfs://.*?:\d+)(.*)") |
---|
| 60 | |
---|
| 61 | VERSION = None |
---|
| 62 | if os.path.exists("./VERSION"): |
---|
| 63 | vFile = open("./VERSION", 'r') |
---|
| 64 | VERSION = vFile.readline() |
---|
| 65 | vFile.close() |
---|
| 66 | |
---|
| 67 | def __archive_logs(conf, log): |
---|
| 68 | # need log-destination-uri, __hadoopLogDirs, temp-dir |
---|
| 69 | status = True |
---|
| 70 | logUri = conf['log-destination-uri'] |
---|
| 71 | hadoopLogDirs = conf['hadoop-log-dirs'] |
---|
| 72 | if logUri: |
---|
| 73 | try: |
---|
| 74 | if hadoopLogDirs: |
---|
| 75 | date = time.localtime() |
---|
| 76 | for logDir in hadoopLogDirs: |
---|
| 77 | (head, tail) = os.path.split(logDir) |
---|
| 78 | (head, logType) = os.path.split(head) |
---|
| 79 | tarBallFile = "%s-%s-%04d%02d%02d%02d%02d%02d-%s.tar.gz" % ( |
---|
| 80 | logType, local_fqdn(), date[0], date[1], date[2], date[3], |
---|
| 81 | date[4], date[5], random.randint(0,1000)) |
---|
| 82 | |
---|
| 83 | if logUri.startswith('file://'): |
---|
| 84 | tarBallFile = os.path.join(logUri[7:], |
---|
| 85 | tarBallFile) |
---|
| 86 | else: |
---|
| 87 | tarBallFile = os.path.join(conf['temp-dir'], tarBallFile) |
---|
| 88 | |
---|
| 89 | log.debug('archiving log files to: %s' % tarBallFile) |
---|
| 90 | status = tar(tarBallFile, logDir, ['*',]) |
---|
| 91 | log.info('archive %s status: %s' % (tarBallFile, status)) |
---|
| 92 | if status and \ |
---|
| 93 | logUri.startswith('hdfs://'): |
---|
| 94 | __copy_archive_to_dfs(conf, tarBallFile) |
---|
| 95 | log.info("copying archive to dfs finished") |
---|
| 96 | dict = {} |
---|
| 97 | except: |
---|
| 98 | log.error(get_exception_string()) |
---|
| 99 | status = False |
---|
| 100 | return status |
---|
| 101 | |
---|
| 102 | |
---|
| 103 | def __copy_archive_to_dfs(conf, archiveFile): |
---|
| 104 | # need log-destination-uri, hadoopCommandstring and/or pkgs |
---|
| 105 | hdfsURIMatch = reHdfsURI.match(conf['log-destination-uri']) |
---|
| 106 | |
---|
| 107 | (head, tail) = os.path.split(archiveFile) |
---|
| 108 | destFile = os.path.join(hdfsURIMatch.group(2), conf['user-id'], 'hod-logs', conf['service-id'], tail) |
---|
| 109 | |
---|
| 110 | log.info("copying archive %s to DFS %s ..." % (archiveFile, destFile)) |
---|
| 111 | |
---|
| 112 | hadoopCmd = conf['hadoop-command-string'] |
---|
| 113 | if conf['pkgs']: |
---|
| 114 | hadoopCmd = os.path.join(conf['pkgs'], 'bin', 'hadoop') |
---|
| 115 | |
---|
| 116 | copyCommand = "%s dfs -fs %s -copyFromLocal %s %s" % (hadoopCmd, |
---|
| 117 | hdfsURIMatch.group(1), archiveFile, destFile) |
---|
| 118 | |
---|
| 119 | log.debug(copyCommand) |
---|
| 120 | |
---|
| 121 | copyThread = simpleCommand('hadoop', copyCommand) |
---|
| 122 | copyThread.start() |
---|
| 123 | copyThread.wait() |
---|
| 124 | copyThread.join() |
---|
| 125 | log.debug(pprint.pformat(copyThread.output())) |
---|
| 126 | |
---|
| 127 | os.unlink(archiveFile) |
---|
| 128 | |
---|
| 129 | def unpack(): |
---|
| 130 | parser = OptionParser() |
---|
| 131 | option_list=["--log-destination-uri", "--hadoop-log-dirs", \ |
---|
| 132 | "--temp-dir", "--hadoop-command-string", "--pkgs", "--user-id", \ |
---|
| 133 | "--service-id", "--hodring-debug", "--hodring-log-dir", \ |
---|
| 134 | "--hodring-syslog-address", "--hodring-cleanup-list", \ |
---|
| 135 | "--jt-pid", "--mr-sys-dir", "--fs-name", "--hadoop-path"] |
---|
| 136 | regexp = re.compile("^--") |
---|
| 137 | for opt in option_list: |
---|
| 138 | parser.add_option(opt,dest=regexp.sub("",opt),action="store") |
---|
| 139 | option_list.append("--hodring-stream") |
---|
| 140 | parser.add_option("--hodring-stream",dest="hodring-stream",metavar="bool",\ |
---|
| 141 | action="store_true") |
---|
| 142 | (options, args) = parser.parse_args() |
---|
| 143 | _options= {} |
---|
| 144 | _options['hodring'] = {} |
---|
| 145 | for opt in dir(options): |
---|
| 146 | if "--"+opt in option_list: |
---|
| 147 | _options[opt] = getattr(options,opt) |
---|
| 148 | if _options.has_key('hadoop-log-dirs') and _options['hadoop-log-dirs']: |
---|
| 149 | _options['hadoop-log-dirs'] = _options['hadoop-log-dirs'].split(",") |
---|
| 150 | if _options.has_key('hodring-syslog-address') and _options['hodring-syslog-address']: |
---|
| 151 | _options['hodring']['syslog-address'] = \ |
---|
| 152 | _options['hodring-syslog-address'].split(':') |
---|
| 153 | _options['hodring']['debug'] = int(_options['hodring-debug']) |
---|
| 154 | _options['hodring']['log-dir'] = _options['hodring-log-dir'] |
---|
| 155 | _options['hodring']['stream'] = _options['hodring-stream'] |
---|
| 156 | _options['hodring']['userid'] = _options['user-id'] |
---|
| 157 | os.putenv('PBS_JOBID', _options['service-id'] ) |
---|
| 158 | return _options |
---|
| 159 | |
---|
| 160 | if __name__ == '__main__': |
---|
| 161 | log = None |
---|
| 162 | try: |
---|
| 163 | conf = unpack() |
---|
| 164 | # Use the same log as hodring |
---|
| 165 | log = getLogger(conf['hodring'],'hodring') |
---|
| 166 | log.debug("Logger initialised successfully") |
---|
| 167 | mrSysDirManager = createMRSystemDirectoryManager(conf, log) |
---|
| 168 | if mrSysDirManager is not None: |
---|
| 169 | mrSysDirManager.removeMRSystemDirectory() |
---|
| 170 | |
---|
| 171 | status = __archive_logs(conf,log) |
---|
| 172 | log.info("Archive status : %s" % status) |
---|
| 173 | list = conf['hodring-cleanup-list'].split(',') |
---|
| 174 | log.info("now removing %s" % list) |
---|
| 175 | for dir in list: |
---|
| 176 | if os.path.exists(dir): |
---|
| 177 | log.debug('removing %s' % (dir)) |
---|
| 178 | shutil.rmtree(dir, True) |
---|
| 179 | log.debug("done") |
---|
| 180 | log.info("Cleanup successfully completed") |
---|
| 181 | except Exception, e: |
---|
| 182 | if log: |
---|
| 183 | log.info("Stack trace:\n%s\n%s" %(get_exception_error_string(),get_exception_string())) |
---|