[154] | 1 | /** |
---|
| 2 | * Frequent (>1000) failure cause for different categories of events grouped by duration (short, medium, long) |
---|
| 3 | * |
---|
| 4 | * @author cristina |
---|
| 5 | */ |
---|
| 6 | |
---|
| 7 | package org.PP; |
---|
| 8 | |
---|
| 9 | import java.io.IOException; |
---|
| 10 | import java.util.*; |
---|
| 11 | |
---|
| 12 | import org.apache.hadoop.fs.Path; |
---|
| 13 | import org.apache.hadoop.conf.*; |
---|
| 14 | import org.apache.hadoop.io.*; |
---|
| 15 | import org.apache.hadoop.mapred.*; |
---|
| 16 | import org.apache.hadoop.util.*; |
---|
| 17 | |
---|
| 18 | public class FailureCause { |
---|
| 19 | |
---|
| 20 | public static class Map extends MapReduceBase implements |
---|
| 21 | Mapper<LongWritable, Text, Text, LongWritable> { |
---|
| 22 | |
---|
| 23 | // Usage: a value that is greater than the previous key in a map but not |
---|
| 24 | // greater than the current key is classified as the current value |
---|
| 25 | public static HashMap<Integer, String> failureCause = new HashMap<Integer, String>(); |
---|
| 26 | public static HashMap<Integer, String> durationGroup = new HashMap<Integer, String>(); |
---|
| 27 | |
---|
| 28 | public static void init() { |
---|
| 29 | // failure cause |
---|
| 30 | failureCause.put(new Integer(-1), "not reported"); |
---|
| 31 | failureCause.put(new Integer(0), "reported as undetermined"); |
---|
| 32 | failureCause.put(new Integer(999), "infrastructure"); |
---|
| 33 | failureCause.put(new Integer(1999), "hardware"); |
---|
| 34 | failureCause.put(new Integer(2999), "IO"); |
---|
| 35 | failureCause.put(new Integer(3999), "network"); |
---|
| 36 | failureCause.put(new Integer(4999), "software"); |
---|
| 37 | failureCause.put(new Integer(5999), "human error"); |
---|
| 38 | failureCause.put(new Integer(6999), "user"); |
---|
| 39 | failureCause.put(new Integer(7000), "end of measurement"); |
---|
| 40 | |
---|
| 41 | // group by duration |
---|
| 42 | durationGroup.put(new Integer(1000), "short"); |
---|
| 43 | durationGroup.put(new Integer(100000), "medium"); |
---|
| 44 | durationGroup.put(Integer.MAX_VALUE, "long"); |
---|
| 45 | } |
---|
| 46 | |
---|
| 47 | public void map(LongWritable key, Text value, |
---|
| 48 | OutputCollector<Text, LongWritable> output, Reporter reporter) |
---|
| 49 | throws IOException { |
---|
| 50 | |
---|
| 51 | Map.init(); |
---|
| 52 | |
---|
| 53 | Text label = new Text(); |
---|
| 54 | String line = value.toString(); |
---|
| 55 | |
---|
| 56 | // skip comments |
---|
| 57 | if (!line.startsWith("#")) { |
---|
| 58 | Double start = new Double(0), end = new Double(0), duration = new Double( |
---|
| 59 | 0); |
---|
| 60 | Integer fault = new Integer(0); |
---|
| 61 | int count = 0; |
---|
| 62 | |
---|
| 63 | // collons 7, 8 and 9 are start, end time of job, failure code |
---|
| 64 | // respectively |
---|
| 65 | StringTokenizer tokenizer = new StringTokenizer(line); |
---|
| 66 | while (tokenizer.hasMoreTokens()) { |
---|
| 67 | count++; |
---|
| 68 | String token = tokenizer.nextToken(); |
---|
| 69 | if (count == 7) |
---|
| 70 | start = Double.parseDouble(token); |
---|
| 71 | if (count == 8) |
---|
| 72 | end = Double.parseDouble(token); |
---|
| 73 | if (count == 9) { |
---|
| 74 | if (token.compareTo("NULL") == 0) { |
---|
| 75 | fault = -1; |
---|
| 76 | } else |
---|
| 77 | fault = Integer.parseInt(token); |
---|
| 78 | } |
---|
| 79 | } |
---|
| 80 | |
---|
| 81 | String durationClass = "", faultClass = ""; |
---|
| 82 | |
---|
| 83 | // get duration class |
---|
| 84 | duration = end - start; |
---|
| 85 | LinkedList<Integer> ll1 = new LinkedList<Integer>( |
---|
| 86 | Map.durationGroup.keySet()); |
---|
| 87 | Collections.sort(ll1); |
---|
| 88 | Iterator iterator1 = ll1.iterator(); |
---|
| 89 | while (iterator1.hasNext()) { |
---|
| 90 | int mkey = (Integer) iterator1.next(); |
---|
| 91 | if (duration <= mkey) { |
---|
| 92 | String mvalue = (String) Map.durationGroup.get(mkey); |
---|
| 93 | durationClass = mvalue; |
---|
| 94 | break; |
---|
| 95 | } |
---|
| 96 | } |
---|
| 97 | |
---|
| 98 | // get fault class |
---|
| 99 | LinkedList<Integer> ll2 = new LinkedList<Integer>( |
---|
| 100 | Map.failureCause.keySet()); |
---|
| 101 | Collections.sort(ll2); |
---|
| 102 | Iterator iterator2 = ll2.iterator(); |
---|
| 103 | while (iterator2.hasNext()) { |
---|
| 104 | int mkey = (Integer) iterator2.next(); |
---|
| 105 | if (fault <= mkey) { |
---|
| 106 | String mvalue = (String) Map.failureCause.get(mkey); |
---|
| 107 | faultClass = mvalue; |
---|
| 108 | break; |
---|
| 109 | } |
---|
| 110 | } |
---|
| 111 | |
---|
| 112 | if(faultClass.compareTo("") == 0) |
---|
| 113 | faultClass = "TYPING"; |
---|
| 114 | |
---|
| 115 | label = new Text(durationClass + " event - " + faultClass |
---|
| 116 | + " fault"); |
---|
| 117 | |
---|
| 118 | // map |
---|
| 119 | output.collect(label, new LongWritable(1)); |
---|
| 120 | } |
---|
| 121 | } |
---|
| 122 | } |
---|
| 123 | |
---|
| 124 | public static class Combine extends MapReduceBase implements Reducer<Text, LongWritable, Text, LongWritable> { |
---|
| 125 | |
---|
| 126 | public void reduce(Text key, Iterator<LongWritable> values, |
---|
| 127 | OutputCollector<Text, LongWritable> output, Reporter reporter) |
---|
| 128 | throws IOException { |
---|
| 129 | |
---|
| 130 | Long sum = new Long(0); |
---|
| 131 | |
---|
| 132 | // sum them |
---|
| 133 | while (values.hasNext()) { |
---|
| 134 | sum += values.next().get(); |
---|
| 135 | } |
---|
| 136 | |
---|
| 137 | output.collect(key, new LongWritable(sum)); |
---|
| 138 | } |
---|
| 139 | } |
---|
| 140 | |
---|
| 141 | public static class Reduce extends MapReduceBase implements |
---|
| 142 | Reducer<Text, LongWritable, Text, LongWritable> { |
---|
| 143 | |
---|
| 144 | public void reduce(Text key, Iterator<LongWritable> values, |
---|
| 145 | OutputCollector<Text, LongWritable> output, Reporter reporter) |
---|
| 146 | throws IOException { |
---|
| 147 | |
---|
| 148 | Long sum = new Long(0); |
---|
| 149 | |
---|
| 150 | // sum them |
---|
| 151 | while (values.hasNext()) { |
---|
| 152 | sum += values.next().get(); |
---|
| 153 | } |
---|
| 154 | |
---|
| 155 | if(sum > 1000) |
---|
| 156 | output.collect(key, new LongWritable(sum)); |
---|
| 157 | } |
---|
| 158 | } |
---|
| 159 | |
---|
| 160 | public static void main(String[] args) throws Exception { |
---|
| 161 | |
---|
| 162 | JobConf conf = new JobConf(FailureCause.class); |
---|
| 163 | conf.setJobName("failureCause"); |
---|
| 164 | |
---|
| 165 | conf.setOutputKeyClass(Text.class); |
---|
| 166 | conf.setOutputValueClass(LongWritable.class); |
---|
| 167 | |
---|
| 168 | conf.setMapperClass(Map.class); |
---|
| 169 | conf.setCombinerClass(Combine.class); |
---|
| 170 | conf.setReducerClass(Reduce.class); |
---|
| 171 | |
---|
| 172 | conf.setInputFormat(TextInputFormat.class); |
---|
| 173 | conf.setOutputFormat(TextOutputFormat.class); |
---|
| 174 | |
---|
| 175 | FileInputFormat.setInputPaths(conf, new Path(args[0])); |
---|
| 176 | FileOutputFormat.setOutputPath(conf, new Path(args[1])); |
---|
| 177 | |
---|
| 178 | conf.setNumReduceTasks(5); |
---|
| 179 | |
---|
| 180 | JobClient.runJob(conf); |
---|
| 181 | } |
---|
| 182 | } |
---|