source: proiecte/PDAD/trunk/endreasonstat/src/ro/pub/cs/pp/pdad/EndReasonStat.java @ 154

package ro.pub.cs.pp.pdad;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.HashMap;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class EndReasonStat {

        private static int[] domains = {-1, 0, 1, 1000, 2000, 3000, 4000, 5000, 6000, 7000};
        private static String[] reasonName = {"not reported", "undetermined", "infrastructure", "hardware", "IO", "network", "software", "human error", "user"}; 
        
        public static class IntPair implements WritableComparable<IntPair> {
                private int first = 0;
                private int second = 0;

                /**
                 * Set the left and right values.
                 */
                public void set(int left, int right) {
                        first = left;
                        second = right;
                }

                public int getFirst() {
                        return first;
                }

                public int getSecond() {
                        return second;
                }

                /**
                 * Read the two integers. Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE,
                 * MAX_VALUE-> -1
                 */
                @Override
                public void readFields(DataInput in) throws IOException {
                        first = in.readInt() + Integer.MIN_VALUE;
                        second = in.readInt() + Integer.MIN_VALUE;
                }

                @Override
                public void write(DataOutput out) throws IOException {
                        out.writeInt(first - Integer.MIN_VALUE);
                        out.writeInt(second - Integer.MIN_VALUE);
                }

                @Override
                public int hashCode() {
                        return first * 157 + second;
                }

                @Override
                public boolean equals(Object right) {
                        if (right instanceof IntPair) {
                                IntPair r = (IntPair) right;
                                return r.first == first && r.second == second;
                        } else {
                                return false;
                        }
                }

                /** A Comparator that compares serialized IntPair. */
                public static class Comparator extends WritableComparator {
                        public Comparator() {
                                super(IntPair.class);
                        }

                        public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
                                        int l2) {
                                return compareBytes(b1, s1, l1, b2, s2, l2);
                        }
                }

                static { // register this comparator
                        WritableComparator.define(IntPair.class, new Comparator());
                }

                @Override
                public int compareTo(IntPair o) {
                        if (first != o.first) {
                                return first < o.first ? -1 : 1;
                        } else if (second != o.second) {
                                return second < o.second ? -1 : 1;
                        } else {
                                return 0;
                        }
                }
        }

          public static class Map extends Mapper<Object, Text, IntWritable, IntPair> {
                
                private static String null_reason = "NULL"; 
                
                private final static IntPair onePair = new IntPair(); 
                private final static IntWritable zero = new IntWritable(0); 
                private final static int fieldNo = 9; 
                
                private int findDomainCode(int reasonCode) {
                        for (int domainCode = 0; domainCode < domains.length - 1; domainCode++) {
                                if (domains[domainCode] <= reasonCode && reasonCode < domains[domainCode + 1])
                                        return domainCode; 
                        }
                        return 0; 
                }
                
                private int extractReasonCode(String line) {
                        String[] pieces = line.split("\\s+");
                        if (pieces.length < fieldNo + 1) {
                                return 0; 
                        }
                        String reason = pieces[fieldNo];
                        
                        if (reason.equals(null_reason)) {
                                return 0;                               
                        } else {
                                int reasonCode = Integer.parseInt(reason);
                                return reasonCode; 
                        }
                }
                
                public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
                        String line = value.toString(); 
                        try { 
                                int reasonCode = extractReasonCode(line);
                                int reasonDomainCode = findDomainCode(reasonCode); 
                                onePair.set(reasonCode, 1); 
                                context.write(new IntWritable(reasonDomainCode), onePair);
                        } catch (NumberFormatException ex) {
                                onePair.set(0, 0);
                                context.write(zero, onePair); 
                        }
                        
                }
        }
        
        
        public static class Reduce extends Reducer <IntWritable, IntPair, Text, IntWritable> {
                
                private Text reason = new Text(); 
                
                public void reduce(IntWritable key, Iterable<IntPair> values, Context context) throws IOException, InterruptedException {
                        /* write the domain name */
                        reason.set(reasonName[key.get()]); 
                        context.write(reason, null);
                        
                        HashMap<Integer, Integer> reasonSum = new HashMap<Integer, Integer>();
                        for (IntPair pair : values) {
                                int reasonCode = pair.getFirst();
                                if (reasonSum.containsKey(reasonCode)) {
                                        int newSum = reasonSum.get(reasonCode) + pair.getSecond(); 
                                        reasonSum.put(reasonCode, newSum); 
                                } else {
                                        reasonSum.put(reasonCode, pair.getSecond()); 
                                }
                        }
                        
                        for (int reasonCode : reasonSum.keySet()) {
                                reason.set(new String("\t" +reasonCode));
                                context.write(reason, new IntWritable(reasonSum.get(reasonCode))); 
                        }
                }
        }
        
        public static void main(String[] args) throws Exception {
                Configuration conf = new Configuration(); 
                Job job = new Job(conf, "EndReasonStat"); 
                job.setNumReduceTasks(5);
                job.setJarByClass(EndReasonStat.class);
                
                /* set map output key/value classes */
                job.setMapOutputKeyClass(IntWritable.class); 
                job.setMapOutputValueClass(IntPair.class); 
                
                /* set application output key/value classes */
                job.setOutputKeyClass(Text.class); 
                job.setOutputValueClass(IntWritable.class);
                
                job.setMapperClass(Map.class);
                job.setReducerClass(Reduce.class);              
                /* don't set any combiner class */
                
                FileInputFormat.addInputPath(job, new Path(args[0])); 
                FileOutputFormat.setOutputPath(job, new Path(args[1])); 
                
                System.exit(job.waitForCompletion(true) ? 0 : 1);
        }
        
}