source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/mapred/TestMapRed.java @ 120

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.mapred;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.Random;

import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;

/**********************************************************
 * MapredLoadTest generates a bunch of work that exercises
 * a Hadoop Map-Reduce system (and DFS, too).  It goes through
 * the following steps:
 *
 * 1) Take inputs 'range' and 'counts'.
 * 2) Generate 'counts' random integers between 0 and range-1.
 * 3) Create a file that lists each integer between 0 and range-1,
 *    and lists the number of times that integer was generated.
 * 4) Emit a (very large) file that contains all the integers
 *    in the order generated.
 * 5) After the file has been generated, read it back and count
 *    how many times each int was generated.
 * 6) Compare this big count-map against the original one.  If
 *    they match, then SUCCESS!  Otherwise, FAILURE!
 *
 * OK, that's how we can think about it.  What are the map-reduce
 * steps that get the job done?
 *
 * 1) In a non-mapred thread, take the inputs 'range' and 'counts'.
 * 2) In a non-mapread thread, generate the answer-key and write to disk.
 * 3) In a mapred job, divide the answer key into K jobs.
 * 4) A mapred 'generator' task consists of K map jobs.  Each reads
 *    an individual "sub-key", and generates integers according to
 *    to it (though with a random ordering).
 * 5) The generator's reduce task agglomerates all of those files
 *    into a single one.
 * 6) A mapred 'reader' task consists of M map jobs.  The output
 *    file is cut into M pieces. Each of the M jobs counts the 
 *    individual ints in its chunk and creates a map of all seen ints.
 * 7) A mapred job integrates all the count files into a single one.
 *
 **********************************************************/
public class TestMapRed extends TestCase {
  /**
   * Modified to make it a junit test.
   * The RandomGen Job does the actual work of creating
   * a huge file of assorted numbers.  It receives instructions
   * as to how many times each number should be counted.  Then
   * it emits those numbers in a crazy order.
   *
   * The map() function takes a key/val pair that describes
   * a value-to-be-emitted (the key) and how many times it 
   * should be emitted (the value), aka "numtimes".  map() then
   * emits a series of intermediate key/val pairs.  It emits
   * 'numtimes' of these.  The key is a random number and the
   * value is the 'value-to-be-emitted'.
   *
   * The system collates and merges these pairs according to
   * the random number.  reduce() function takes in a key/value
   * pair that consists of a crazy random number and a series
   * of values that should be emitted.  The random number key
   * is now dropped, and reduce() emits a pair for every intermediate value.
   * The emitted key is an intermediate value.  The emitted value
   * is just a blank string.  Thus, we've created a huge file
   * of numbers in random order, but where each number appears
   * as many times as we were instructed.
   */
  static class RandomGenMapper
    implements Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
    
    public void configure(JobConf job) {
    }

    public void map(IntWritable key, IntWritable val,
                    OutputCollector<IntWritable, IntWritable> out,
                    Reporter reporter) throws IOException {
      int randomVal = key.get();
      int randomCount = val.get();

      for (int i = 0; i < randomCount; i++) {
        out.collect(new IntWritable(Math.abs(r.nextInt())), new IntWritable(randomVal));
      }
    }
    public void close() {
    }
  }
  /**
   */
  static class RandomGenReducer
    implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
    
    public void configure(JobConf job) {
    }

    public void reduce(IntWritable key, Iterator<IntWritable> it,
                       OutputCollector<IntWritable, IntWritable> out,
                       Reporter reporter) throws IOException {
      while (it.hasNext()) {
        out.collect(it.next(), null);
      }
    }
    public void close() {
    }
  }

  /**
   * The RandomCheck Job does a lot of our work.  It takes
   * in a num/string keyspace, and transforms it into a
   * key/count(int) keyspace.
   *
   * The map() function just emits a num/1 pair for every
   * num/string input pair.
   *
   * The reduce() function sums up all the 1s that were
   * emitted for a single key.  It then emits the key/total
   * pair.
   *
   * This is used to regenerate the random number "answer key".
   * Each key here is a random number, and the count is the
   * number of times the number was emitted.
   */
  static class RandomCheckMapper
    implements Mapper<WritableComparable, Text, IntWritable, IntWritable> {
    
    public void configure(JobConf job) {
    }

    public void map(WritableComparable key, Text val,
                    OutputCollector<IntWritable, IntWritable> out,
                    Reporter reporter) throws IOException {
      out.collect(new IntWritable(Integer.parseInt(val.toString().trim())), new IntWritable(1));
    }
    public void close() {
    }
  }
  /**
   */
  static class RandomCheckReducer
      implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
    public void configure(JobConf job) {
    }
        
    public void reduce(IntWritable key, Iterator<IntWritable> it,
                       OutputCollector<IntWritable, IntWritable> out,
                       Reporter reporter) throws IOException {
      int keyint = key.get();
      int count = 0;
      while (it.hasNext()) {
        it.next();
        count++;
      }
      out.collect(new IntWritable(keyint), new IntWritable(count));
    }
    public void close() {
    }
  }

  /**
   * The Merge Job is a really simple one.  It takes in
   * an int/int key-value set, and emits the same set.
   * But it merges identical keys by adding their values.
   *
   * Thus, the map() function is just the identity function
   * and reduce() just sums.  Nothing to see here!
   */
  static class MergeMapper
    implements Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
    
    public void configure(JobConf job) {
    }

    public void map(IntWritable key, IntWritable val,
                    OutputCollector<IntWritable, IntWritable> out,
                    Reporter reporter) throws IOException {
      int keyint = key.get();
      int valint = val.get();

      out.collect(new IntWritable(keyint), new IntWritable(valint));
    }
    public void close() {
    }
  }
  static class MergeReducer
    implements Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
    public void configure(JobConf job) {
    }
        
    public void reduce(IntWritable key, Iterator<IntWritable> it,
                       OutputCollector<IntWritable, IntWritable> out,
                       Reporter reporter) throws IOException {
      int keyint = key.get();
      int total = 0;
      while (it.hasNext()) {
        total += it.next().get();
      }
      out.collect(new IntWritable(keyint), new IntWritable(total));
    }
    public void close() {
    }
  }

  private static int range = 10;
  private static int counts = 100;
  private static Random r = new Random();

  /**
     public TestMapRed(int range, int counts, Configuration conf) throws IOException {
     this.range = range;
     this.counts = counts;
     this.conf = conf;
     }
  **/

  public void testMapred() throws Exception {
    launch();
  }

  private static class MyMap
    implements Mapper<WritableComparable, Text, Text, Text> {
      
    public void configure(JobConf conf) {
    }
      
    public void map(WritableComparable key, Text value,
                    OutputCollector<Text, Text> output,
                    Reporter reporter) throws IOException {
      String str = value.toString().toLowerCase();
      output.collect(new Text(str), value);
    }

    public void close() throws IOException {
    }
  }
    
  private static class MyReduce extends IdentityReducer {
    private JobConf conf;
    private boolean compressInput;
    private TaskAttemptID taskId;
    private boolean first = true;
      
    @Override
    public void configure(JobConf conf) {
      this.conf = conf;
      compressInput = conf.getCompressMapOutput();
      taskId = TaskAttemptID.forName(conf.get("mapred.task.id"));
    }
      
    public void reduce(WritableComparable key, Iterator values,
                       OutputCollector output, Reporter reporter
                       ) throws IOException {
      if (first) {
        first = false;
        MapOutputFile mapOutputFile = new MapOutputFile(taskId.getJobID());
        mapOutputFile.setConf(conf);
        Path input = mapOutputFile.getInputFile(0, taskId);
        FileSystem fs = FileSystem.get(conf);
        assertTrue("reduce input exists " + input, fs.exists(input));
        SequenceFile.Reader rdr = 
          new SequenceFile.Reader(fs, input, conf);
        assertEquals("is reduce input compressed " + input, 
                     compressInput, 
                     rdr.isCompressed());
        rdr.close();          
      }
    }
      
  }

  private static class BadPartitioner
      implements Partitioner<LongWritable,Text> {
    boolean low;
    public void configure(JobConf conf) {
      low = conf.getBoolean("test.testmapred.badpartition", true);
    }
    public int getPartition(LongWritable k, Text v, int numPartitions) {
      return low ? -1 : numPartitions;
    }
  }

  public void testPartitioner() throws Exception {
    JobConf conf = new JobConf(TestMapRed.class);
    conf.setPartitionerClass(BadPartitioner.class);
    FileSystem fs = FileSystem.getLocal(conf);
    Path testdir = new Path(
        System.getProperty("test.build.data","/tmp")).makeQualified(fs);
    Path inFile = new Path(testdir, "blah/blah");
    DataOutputStream f = fs.create(inFile);
    f.writeBytes("blah blah blah\n");
    f.close();
    FileInputFormat.setInputPaths(conf, inFile);
    FileOutputFormat.setOutputPath(conf, new Path(testdir, "out"));
    conf.setMapperClass(IdentityMapper.class);
    conf.setReducerClass(IdentityReducer.class);
    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    // partition too low
    conf.setBoolean("test.testmapred.badpartition", true);
    boolean pass = true;
    try {
      JobClient.runJob(conf);
    } catch (IOException e) {
      pass = false;
    }
    assertFalse("should fail for partition < 0", pass);

    // partition too high
    conf.setBoolean("test.testmapred.badpartition", false);
    pass = true;
    try {
      JobClient.runJob(conf);
    } catch (IOException e) {
      pass = false;
    }
    assertFalse("should fail for partition >= numPartitions", pass);
  }

  public static class NullMapper
      implements Mapper<NullWritable,Text,NullWritable,Text> {
    public void map(NullWritable key, Text val,
        OutputCollector<NullWritable,Text> output, Reporter reporter)
        throws IOException {
      output.collect(NullWritable.get(), val);
    }
    public void configure(JobConf conf) { }
    public void close() { }
  }

  public void testNullKeys() throws Exception {
    JobConf conf = new JobConf(TestMapRed.class);
    FileSystem fs = FileSystem.getLocal(conf);
    Path testdir = new Path(
        System.getProperty("test.build.data","/tmp")).makeQualified(fs);
    fs.delete(testdir, true);
    Path inFile = new Path(testdir, "nullin/blah");
    SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile,
        NullWritable.class, Text.class, SequenceFile.CompressionType.NONE);
    Text t = new Text();
    t.set("AAAAAAAAAAAAAA"); w.append(NullWritable.get(), t);
    t.set("BBBBBBBBBBBBBB"); w.append(NullWritable.get(), t);
    t.set("CCCCCCCCCCCCCC"); w.append(NullWritable.get(), t);
    t.set("DDDDDDDDDDDDDD"); w.append(NullWritable.get(), t);
    t.set("EEEEEEEEEEEEEE"); w.append(NullWritable.get(), t);
    t.set("FFFFFFFFFFFFFF"); w.append(NullWritable.get(), t);
    t.set("GGGGGGGGGGGGGG"); w.append(NullWritable.get(), t);
    t.set("HHHHHHHHHHHHHH"); w.append(NullWritable.get(), t);
    w.close();
    FileInputFormat.setInputPaths(conf, inFile);
    FileOutputFormat.setOutputPath(conf, new Path(testdir, "nullout"));
    conf.setMapperClass(NullMapper.class);
    conf.setReducerClass(IdentityReducer.class);
    conf.setOutputKeyClass(NullWritable.class);
    conf.setOutputValueClass(Text.class);
    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    conf.setNumReduceTasks(1);

    JobClient.runJob(conf);

    SequenceFile.Reader r = new SequenceFile.Reader(fs,
        new Path(testdir, "nullout/part-00000"), conf);
    String m = "AAAAAAAAAAAAAA";
    for (int i = 1; r.next(NullWritable.get(), t); ++i) {
      assertTrue(t.toString() + " doesn't match " + m, m.equals(t.toString()));
      m = m.replace((char)('A' + i - 1), (char)('A' + i));
    }
  }

  private void checkCompression(boolean compressMapOutputs,
                                CompressionType redCompression,
                                boolean includeCombine
                                ) throws Exception {
    JobConf conf = new JobConf(TestMapRed.class);
    Path testdir = new Path("build/test/test.mapred.compress");
    Path inDir = new Path(testdir, "in");
    Path outDir = new Path(testdir, "out");
    FileSystem fs = FileSystem.get(conf);
    fs.delete(testdir, true);
    FileInputFormat.setInputPaths(conf, inDir);
    FileOutputFormat.setOutputPath(conf, outDir);
    conf.setMapperClass(MyMap.class);
    conf.setReducerClass(MyReduce.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);
    conf.setOutputFormat(SequenceFileOutputFormat.class);
    if (includeCombine) {
      conf.setCombinerClass(IdentityReducer.class);
    }
    conf.setCompressMapOutput(compressMapOutputs);
    SequenceFileOutputFormat.setOutputCompressionType(conf, redCompression);
    try {
      if (!fs.mkdirs(testdir)) {
        throw new IOException("Mkdirs failed to create " + testdir.toString());
      }
      if (!fs.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
      }
      Path inFile = new Path(inDir, "part0");
      DataOutputStream f = fs.create(inFile);
      f.writeBytes("Owen was here\n");
      f.writeBytes("Hadoop is fun\n");
      f.writeBytes("Is this done, yet?\n");
      f.close();
      RunningJob rj = JobClient.runJob(conf);
      assertTrue("job was complete", rj.isComplete());
      assertTrue("job was successful", rj.isSuccessful());
      Path output = new Path(outDir,
                             Task.getOutputName(0));
      assertTrue("reduce output exists " + output, fs.exists(output));
      SequenceFile.Reader rdr = 
        new SequenceFile.Reader(fs, output, conf);
      assertEquals("is reduce output compressed " + output, 
                   redCompression != CompressionType.NONE, 
                   rdr.isCompressed());
      rdr.close();
    } finally {
      fs.delete(testdir, true);
    }
  }
    
  public void testCompression() throws Exception {
    EnumSet<SequenceFile.CompressionType> seq =
      EnumSet.allOf(SequenceFile.CompressionType.class);
    for (CompressionType redCompression : seq) {
      for(int combine=0; combine < 2; ++combine) {
        checkCompression(false, redCompression, combine == 1);
        checkCompression(true, redCompression, combine == 1);
      }
    }
  }
    
    
  /**
   * 
   */
  public static void launch() throws Exception {
    //
    // Generate distribution of ints.  This is the answer key.
    //
    JobConf conf = new JobConf(TestMapRed.class);
    int countsToGo = counts;
    int dist[] = new int[range];
    for (int i = 0; i < range; i++) {
      double avgInts = (1.0 * countsToGo) / (range - i);
      dist[i] = (int) Math.max(0, Math.round(avgInts + (Math.sqrt(avgInts) * r.nextGaussian())));
      countsToGo -= dist[i];
    }
    if (countsToGo > 0) {
      dist[dist.length-1] += countsToGo;
    }

    //
    // Write the answer key to a file.  
    //
    FileSystem fs = FileSystem.get(conf);
    Path testdir = new Path("mapred.loadtest");
    if (!fs.mkdirs(testdir)) {
      throw new IOException("Mkdirs failed to create " + testdir.toString());
    }

    Path randomIns = new Path(testdir, "genins");
    if (!fs.mkdirs(randomIns)) {
      throw new IOException("Mkdirs failed to create " + randomIns.toString());
    }

    Path answerkey = new Path(randomIns, "answer.key");
    SequenceFile.Writer out = 
      SequenceFile.createWriter(fs, conf, answerkey, IntWritable.class,
                                IntWritable.class, 
                                SequenceFile.CompressionType.NONE);
    try {
      for (int i = 0; i < range; i++) {
        out.append(new IntWritable(i), new IntWritable(dist[i]));
      }
    } finally {
      out.close();
    }
    //printFiles(randomIns, conf);

    //
    // Now we need to generate the random numbers according to
    // the above distribution.
    //
    // We create a lot of map tasks, each of which takes at least
    // one "line" of the distribution.  (That is, a certain number
    // X is to be generated Y number of times.)
    //
    // A map task emits Y key/val pairs.  The val is X.  The key
    // is a randomly-generated number.
    //
    // The reduce task gets its input sorted by key.  That is, sorted
    // in random order.  It then emits a single line of text that
    // for the given values.  It does not emit the key.
    //
    // Because there's just one reduce task, we emit a single big
    // file of random numbers.
    //
    Path randomOuts = new Path(testdir, "genouts");
    fs.delete(randomOuts, true);


    JobConf genJob = new JobConf(conf, TestMapRed.class);
    FileInputFormat.setInputPaths(genJob, randomIns);
    genJob.setInputFormat(SequenceFileInputFormat.class);
    genJob.setMapperClass(RandomGenMapper.class);

    FileOutputFormat.setOutputPath(genJob, randomOuts);
    genJob.setOutputKeyClass(IntWritable.class);
    genJob.setOutputValueClass(IntWritable.class);
    genJob.setOutputFormat(TextOutputFormat.class);
    genJob.setReducerClass(RandomGenReducer.class);
    genJob.setNumReduceTasks(1);

    JobClient.runJob(genJob);
    //printFiles(randomOuts, conf);

    //
    // Next, we read the big file in and regenerate the 
    // original map.  It's split into a number of parts.
    // (That number is 'intermediateReduces'.)
    //
    // We have many map tasks, each of which read at least one
    // of the output numbers.  For each number read in, the
    // map task emits a key/value pair where the key is the
    // number and the value is "1".
    //
    // We have a single reduce task, which receives its input
    // sorted by the key emitted above.  For each key, there will
    // be a certain number of "1" values.  The reduce task sums
    // these values to compute how many times the given key was
    // emitted.
    //
    // The reduce task then emits a key/val pair where the key
    // is the number in question, and the value is the number of
    // times the key was emitted.  This is the same format as the
    // original answer key (except that numbers emitted zero times
    // will not appear in the regenerated key.)  The answer set
    // is split into a number of pieces.  A final MapReduce job
    // will merge them.
    //
    // There's not really a need to go to 10 reduces here 
    // instead of 1.  But we want to test what happens when
    // you have multiple reduces at once.
    //
    int intermediateReduces = 10;
    Path intermediateOuts = new Path(testdir, "intermediateouts");
    fs.delete(intermediateOuts, true);
    JobConf checkJob = new JobConf(conf, TestMapRed.class);
    FileInputFormat.setInputPaths(checkJob, randomOuts);
    checkJob.setInputFormat(TextInputFormat.class);
    checkJob.setMapperClass(RandomCheckMapper.class);

    FileOutputFormat.setOutputPath(checkJob, intermediateOuts);
    checkJob.setOutputKeyClass(IntWritable.class);
    checkJob.setOutputValueClass(IntWritable.class);
    checkJob.setOutputFormat(MapFileOutputFormat.class);
    checkJob.setReducerClass(RandomCheckReducer.class);
    checkJob.setNumReduceTasks(intermediateReduces);

    JobClient.runJob(checkJob);
    //printFiles(intermediateOuts, conf); 

    //
    // OK, now we take the output from the last job and
    // merge it down to a single file.  The map() and reduce()
    // functions don't really do anything except reemit tuples.
    // But by having a single reduce task here, we end up merging
    // all the files.
    //
    Path finalOuts = new Path(testdir, "finalouts");
    fs.delete(finalOuts, true);
    JobConf mergeJob = new JobConf(conf, TestMapRed.class);
    FileInputFormat.setInputPaths(mergeJob, intermediateOuts);
    mergeJob.setInputFormat(SequenceFileInputFormat.class);
    mergeJob.setMapperClass(MergeMapper.class);
        
    FileOutputFormat.setOutputPath(mergeJob, finalOuts);
    mergeJob.setOutputKeyClass(IntWritable.class);
    mergeJob.setOutputValueClass(IntWritable.class);
    mergeJob.setOutputFormat(SequenceFileOutputFormat.class);
    mergeJob.setReducerClass(MergeReducer.class);
    mergeJob.setNumReduceTasks(1);
        
    JobClient.runJob(mergeJob);
    //printFiles(finalOuts, conf); 
 
    //
    // Finally, we compare the reconstructed answer key with the
    // original one.  Remember, we need to ignore zero-count items
    // in the original key.
    //
    boolean success = true;
    Path recomputedkey = new Path(finalOuts, "part-00000");
    SequenceFile.Reader in = new SequenceFile.Reader(fs, recomputedkey, conf);
    int totalseen = 0;
    try {
      IntWritable key = new IntWritable();
      IntWritable val = new IntWritable();            
      for (int i = 0; i < range; i++) {
        if (dist[i] == 0) {
          continue;
        }
        if (!in.next(key, val)) {
          System.err.println("Cannot read entry " + i);
          success = false;
          break;
        } else {
          if (!((key.get() == i) && (val.get() == dist[i]))) {
            System.err.println("Mismatch!  Pos=" + key.get() + ", i=" + i + 
                               ", val=" + val.get() + ", dist[i]=" + dist[i]);
            success = false;
          }
          totalseen += val.get();
        }
      }
      if (success) {
        if (in.next(key, val)) {
          System.err.println("Unnecessary lines in recomputed key!");
          success = false;
        }
      }
    } finally {
      in.close();
    }
    int originalTotal = 0;
    for (int i = 0; i < dist.length; i++) {
      originalTotal += dist[i];
    }
    System.out.println("Original sum: " + originalTotal);
    System.out.println("Recomputed sum: " + totalseen);

    //
    // Write to "results" whether the test succeeded or not.
    //
    Path resultFile = new Path(testdir, "results");
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.create(resultFile)));
    try {
      bw.write("Success=" + success + "\n");
      System.out.println("Success=" + success);
    } finally {
      bw.close();
    }
    assertTrue("testMapRed failed", success);
    fs.delete(testdir, true);
  }

  private static void printTextFile(FileSystem fs, Path p) throws IOException {
    BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(p)));
    String line;
    while ((line = in.readLine()) != null) {
      System.out.println("  Row: " + line);
    }
    in.close();
  }

  private static void printSequenceFile(FileSystem fs, Path p, 
                                        Configuration conf) throws IOException {
    SequenceFile.Reader r = new SequenceFile.Reader(fs, p, conf);
    Object key = null;
    Object value = null;
    while ((key = r.next(key)) != null) {
      value = r.getCurrentValue(value);
      System.out.println("  Row: " + key + ", " + value);
    }
    r.close();    
  }

  private static boolean isSequenceFile(FileSystem fs,
                                        Path f) throws IOException {
    DataInputStream in = fs.open(f);
    byte[] seq = "SEQ".getBytes();
    for(int i=0; i < seq.length; ++i) {
      if (seq[i] != in.read()) {
        return false;
      }
    }
    return true;
  }

  private static void printFiles(Path dir, 
                                 Configuration conf) throws IOException {
    FileSystem fs = dir.getFileSystem(conf);
    for(FileStatus f: fs.listStatus(dir)) {
      System.out.println("Reading " + f.getPath() + ": ");
      if (f.isDir()) {
        System.out.println("  it is a map file.");
        printSequenceFile(fs, new Path(f.getPath(), "data"), conf);
      } else if (isSequenceFile(fs, f.getPath())) {
        System.out.println("  it is a sequence file.");
        printSequenceFile(fs, f.getPath(), conf);
      } else {
        System.out.println("  it is a text file.");
        printTextFile(fs, f.getPath());
      }
    }
  }

  /**
   * Launches all the tasks in order.
   */
  public static void main(String[] argv) throws Exception {
    if (argv.length < 2) {
      System.err.println("Usage: TestMapRed <range> <counts>");
      System.err.println();
      System.err.println("Note: a good test will have a <counts> value that is substantially larger than the <range>");
      return;
    }

    int i = 0;
    range = Integer.parseInt(argv[i++]);
    counts = Integer.parseInt(argv[i++]);
    launch();
  }
    
  public void testSmallInput(){
    runJob(100);
  }

  public void testBiggerInput(){
    runJob(1000);
  }

  public void runJob(int items) {
    try {
      JobConf conf = new JobConf(TestMapRed.class);
      Path testdir = new Path("build/test/test.mapred.spill");
      Path inDir = new Path(testdir, "in");
      Path outDir = new Path(testdir, "out");
      FileSystem fs = FileSystem.get(conf);
      fs.delete(testdir, true);
      conf.setInt("io.sort.mb", 1);
      conf.setInputFormat(SequenceFileInputFormat.class);
      FileInputFormat.setInputPaths(conf, inDir);
      FileOutputFormat.setOutputPath(conf, outDir);
      conf.setMapperClass(IdentityMapper.class);
      conf.setReducerClass(IdentityReducer.class);
      conf.setOutputKeyClass(Text.class);
      conf.setOutputValueClass(Text.class);
      conf.setOutputFormat(SequenceFileOutputFormat.class);
      if (!fs.mkdirs(testdir)) {
        throw new IOException("Mkdirs failed to create " + testdir.toString());
      }
      if (!fs.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
      }
      Path inFile = new Path(inDir, "part0");
      SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inFile,
                                                             Text.class, Text.class);

      StringBuffer content = new StringBuffer();

      for (int i = 0; i < 1000; i++) {
        content.append(i).append(": This is one more line of content\n");
      }

      Text text = new Text(content.toString());

      for (int i = 0; i < items; i++) {
        writer.append(new Text("rec:" + i), text);
      }
      writer.close();

      JobClient.runJob(conf);
    } catch (Exception e) {
      fail("Threw exception:" + e);
    }
  }
}