source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/mapred/ControlledMapReduceJob.java @ 120

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Random;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * A Controlled Map/Reduce Job. The tasks are controlled by the presence of
 * particularly named files in the directory signalFileDir on the file-system
 * that the job is configured to work with. Tasks get scheduled by the
 * scheduler, occupy the slots on the TaskTrackers and keep running till the
 * user gives a signal via files whose names are of the form MAPS_[0-9]* and
 * REDUCES_[0-9]*. For e.g., whenever the map tasks see that a file name MAPS_5
 * is created in the singalFileDir, all the maps whose TaskAttemptIDs are below
 * 4 get finished. At any time, there should be only one MAPS_[0-9]* file and
 * only one REDUCES_[0-9]* file in the singnalFileDir. In the beginning MAPS_0
 * and REDUCE_0 files are present, and further signals are given by renaming
 * these files.
 * 
 */
class ControlledMapReduceJob extends Configured implements Tool,
    Mapper<NullWritable, NullWritable, IntWritable, NullWritable>,
    Reducer<IntWritable, NullWritable, NullWritable, NullWritable>,
    Partitioner<IntWritable, NullWritable>,
    InputFormat<NullWritable, NullWritable> {

  static final Log LOG = LogFactory.getLog(ControlledMapReduceJob.class);

  private FileSystem fs = null;
  private int taskNumber;

  private static ArrayList<Path> signalFileDirCache = new ArrayList<Path>();

  private Path signalFileDir;
  {
    Random random = new Random();
    signalFileDir = new Path("signalFileDir-" + random.nextLong());
    while (signalFileDirCache.contains(signalFileDir)) {
      signalFileDir = new Path("signalFileDir-" + random.nextLong());
    }
    signalFileDirCache.add(signalFileDir);
  }

  private long mapsFinished = 0;
  private long reducesFinished = 0;

  private RunningJob rJob = null;

  private int numMappers;
  private int numReducers;

  private final String MAP_SIGFILE_PREFIX = "MAPS_";
  private final String REDUCE_SIGFILE_PREFIX = "REDUCES_";

  private void initialize()
      throws IOException {
    fs = FileSystem.get(getConf());
    fs.mkdirs(signalFileDir);
    writeFile(new Path(signalFileDir, MAP_SIGFILE_PREFIX + mapsFinished));
    writeFile(new Path(signalFileDir, REDUCE_SIGFILE_PREFIX + reducesFinished));
  }

  /**
   * Finish N number of maps/reduces.
   * 
   * @param isMap
   * @param noOfTasksToFinish
   * @throws IOException
   */
  public void finishNTasks(boolean isMap, int noOfTasksToFinish)
      throws IOException {
    if (noOfTasksToFinish < 0) {
      throw new IOException(
          "Negative values for noOfTasksToFinish not acceptable");
    }

    if (noOfTasksToFinish == 0) {
      return;
    }

    LOG.info("Going to finish off " + noOfTasksToFinish);
    String PREFIX = isMap ? MAP_SIGFILE_PREFIX : REDUCE_SIGFILE_PREFIX;
    long tasksFinished = isMap ? mapsFinished : reducesFinished;
    Path oldSignalFile =
        new Path(signalFileDir, PREFIX + String.valueOf(tasksFinished));
    Path newSignalFile =
        new Path(signalFileDir, PREFIX
            + String.valueOf(tasksFinished + noOfTasksToFinish));
    fs.rename(oldSignalFile, newSignalFile);
    if (isMap) {
      mapsFinished += noOfTasksToFinish;
    } else {
      reducesFinished += noOfTasksToFinish;
    }
    LOG.info("Successfully sent signal to finish off " + noOfTasksToFinish);
  }

  /**
   * Finished all tasks of type determined by isMap
   * 
   * @param isMap
   * @throws IOException
   */
  public void finishAllTasks(boolean isMap)
      throws IOException {
    finishNTasks(isMap, (isMap ? numMappers : numReducers));
  }

  /**
   * Finish the job
   * 
   * @throws IOException
   */
  public void finishJob()
      throws IOException {
    finishAllTasks(true);
    finishAllTasks(false);
  }

  /**
   * Wait till noOfTasksToBeRunning number of tasks of type specified by isMap
   * started running. This currently uses a jip object and directly uses its api
   * to determine the number of tasks running.
   * 
   * <p>
   * 
   * TODO: It should eventually use a JobID and then get the information from
   * the JT to check the number of running tasks.
   * 
   * @param jip
   * @param isMap
   * @param noOfTasksToBeRunning
   */
  static void waitTillNTasksStartRunning(JobInProgress jip, boolean isMap,
      int noOfTasksToBeRunning)
      throws InterruptedException {
    int numTasks = 0;
    while (numTasks != noOfTasksToBeRunning) {
      Thread.sleep(1000);
      numTasks = isMap ? jip.runningMaps() : jip.runningReduces();
      LOG.info("Waiting till " + noOfTasksToBeRunning
          + (isMap ? " map" : " reduce") + " tasks of the job "
          + jip.getJobID() + " start running. " + numTasks
          + " tasks already started running.");
    }
  }

  /**
   * Make sure that the number of tasks of type specified by isMap running in
   * the given job is the same as noOfTasksToBeRunning
   * 
   * <p>
   * 
   * TODO: It should eventually use a JobID and then get the information from
   * the JT to check the number of running tasks.
   * 
   * @param jip
   * @param isMap
   * @param noOfTasksToBeRunning
   */
  static void assertNumTasksRunning(JobInProgress jip, boolean isMap,
      int noOfTasksToBeRunning)
      throws Exception {
    if ((isMap ? jip.runningMaps() : jip.runningReduces()) != noOfTasksToBeRunning) {
      throw new Exception("Number of tasks running is not "
          + noOfTasksToBeRunning);
    }
  }

  /**
   * Wait till noOfTasksToFinish number of tasks of type specified by isMap
   * are finished. This currently uses a jip object and directly uses its api to
   * determine the number of tasks finished.
   * 
   * <p>
   * 
   * TODO: It should eventually use a JobID and then get the information from
   * the JT to check the number of finished tasks.
   * 
   * @param jip
   * @param isMap
   * @param noOfTasksToFinish
   * @throws InterruptedException
   */
  static void waitTillNTotalTasksFinish(JobInProgress jip, boolean isMap,
      int noOfTasksToFinish)
      throws InterruptedException {
    int noOfTasksAlreadyFinished = 0;
    while (noOfTasksAlreadyFinished < noOfTasksToFinish) {
      Thread.sleep(1000);
      noOfTasksAlreadyFinished =
          (isMap ? jip.finishedMaps() : jip.finishedReduces());
      LOG.info("Waiting till " + noOfTasksToFinish
          + (isMap ? " map" : " reduce") + " tasks of the job "
          + jip.getJobID() + " finish. " + noOfTasksAlreadyFinished
          + " tasks already got finished.");
    }
  }

  /**
   * Have all the tasks of type specified by isMap finished in this job?
   * 
   * @param jip
   * @param isMap
   * @return true if finished, false otherwise
   */
  static boolean haveAllTasksFinished(JobInProgress jip, boolean isMap) {
    return ((isMap ? jip.runningMaps() : jip.runningReduces()) == 0);
  }

  private void writeFile(Path name)
      throws IOException {
    Configuration conf = new Configuration(false);
    SequenceFile.Writer writer =
        SequenceFile.createWriter(fs, conf, name, BytesWritable.class,
            BytesWritable.class, CompressionType.NONE);
    writer.append(new BytesWritable(), new BytesWritable());
    writer.close();
  }

  @Override
  public void configure(JobConf conf) {
    try {
      signalFileDir = new Path(conf.get("signal.dir.path"));
      numReducers = conf.getNumReduceTasks();
      fs = FileSystem.get(conf);
      String taskAttemptId = conf.get("mapred.task.id");
      if (taskAttemptId != null) {
        TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptId);
        taskNumber = taskAttemptID.getTaskID().getId();
      }
    } catch (IOException ioe) {
      LOG.warn("Caught exception " + ioe);
    }
  }

  private FileStatus[] listSignalFiles(FileSystem fileSys, final boolean isMap)
      throws IOException {
    return fileSys.globStatus(new Path(signalFileDir.toString() + "/*"),
        new PathFilter() {
          @Override
          public boolean accept(Path path) {
            if (isMap && path.getName().startsWith(MAP_SIGFILE_PREFIX)) {
              LOG.debug("Found signal file : " + path.getName());
              return true;
            } else if (!isMap
                && path.getName().startsWith(REDUCE_SIGFILE_PREFIX)) {
              LOG.debug("Found signal file : " + path.getName());
              return true;
            }
            LOG.info("Didn't find any relevant signal files.");
            return false;
          }
        });
  }

  @Override
  public void map(NullWritable key, NullWritable value,
      OutputCollector<IntWritable, NullWritable> output, Reporter reporter)
      throws IOException {
    LOG.info(taskNumber + " has started.");
    FileStatus[] files = listSignalFiles(fs, true);
    String[] sigFileComps = files[0].getPath().getName().split("_");
    String signalType = sigFileComps[0];
    int noOfTasks = Integer.parseInt(sigFileComps[1]);

    while (!signalType.equals("MAPS") || taskNumber + 1 > noOfTasks) {
      LOG.info("Signal type found : " + signalType
          + " .Number of tasks to be finished by this signal : " + noOfTasks
          + " . My id : " + taskNumber);
      LOG.info(taskNumber + " is still alive.");
      try {
        reporter.progress();
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        LOG.info(taskNumber + " is still alive.");
        break;
      }
      files = listSignalFiles(fs, true);
      sigFileComps = files[0].getPath().getName().split("_");
      signalType = sigFileComps[0];
      noOfTasks = Integer.parseInt(sigFileComps[1]);
    }
    LOG.info("Signal type found : " + signalType
        + " .Number of tasks to be finished by this signal : " + noOfTasks
        + " . My id : " + taskNumber);
    // output numReduce number of random values, so that
    // each reducer will get one key each.
    for (int i = 0; i < numReducers; i++) {
      output.collect(new IntWritable(i), NullWritable.get());
    }

    LOG.info(taskNumber + " is finished.");
  }

  @Override
  public void reduce(IntWritable key, Iterator<NullWritable> values,
      OutputCollector<NullWritable, NullWritable> output, Reporter reporter)
      throws IOException {
    LOG.info(taskNumber + " has started.");
    FileStatus[] files = listSignalFiles(fs, false);
    String[] sigFileComps = files[0].getPath().getName().split("_");
    String signalType = sigFileComps[0];
    int noOfTasks = Integer.parseInt(sigFileComps[1]);

    while (!signalType.equals("REDUCES") || taskNumber + 1 > noOfTasks) {
      LOG.info("Signal type found : " + signalType
          + " .Number of tasks to be finished by this signal : " + noOfTasks
          + " . My id : " + taskNumber);
      LOG.info(taskNumber + " is still alive.");
      try {
        reporter.progress();
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        LOG.info(taskNumber + " is still alive.");
        break;
      }
      files = listSignalFiles(fs, false);
      sigFileComps = files[0].getPath().getName().split("_");
      signalType = sigFileComps[0];
      noOfTasks = Integer.parseInt(sigFileComps[1]);
    }
    LOG.info("Signal type found : " + signalType
        + " .Number of tasks to be finished by this signal : " + noOfTasks
        + " . My id : " + taskNumber);
    LOG.info(taskNumber + " is finished.");
  }

  @Override
  public void close()
      throws IOException {
    // nothing
  }

  public JobID getJobId() {
    if (rJob == null) {
      return null;
    }
    return rJob.getID();
  }

  public int run(int numMapper, int numReducer)
      throws IOException {
    JobConf conf =
        getControlledMapReduceJobConf(getConf(), numMapper, numReducer);
    JobClient client = new JobClient(conf);
    rJob = client.submitJob(conf);
    while (!rJob.isComplete()) {
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
        break;
      }
    }
    if (rJob.isSuccessful()) {
      return 0;
    }
    return 1;
  }

  private JobConf getControlledMapReduceJobConf(Configuration clusterConf,
      int numMapper, int numReducer)
      throws IOException {
    setConf(clusterConf);
    initialize();
    JobConf conf = new JobConf(getConf(), ControlledMapReduceJob.class);
    conf.setJobName("ControlledJob");
    conf.set("signal.dir.path", signalFileDir.toString());
    conf.setNumMapTasks(numMapper);
    conf.setNumReduceTasks(numReducer);
    conf.setMapperClass(ControlledMapReduceJob.class);
    conf.setMapOutputKeyClass(IntWritable.class);
    conf.setMapOutputValueClass(NullWritable.class);
    conf.setReducerClass(ControlledMapReduceJob.class);
    conf.setOutputKeyClass(NullWritable.class);
    conf.setOutputValueClass(NullWritable.class);
    conf.setInputFormat(ControlledMapReduceJob.class);
    FileInputFormat.addInputPath(conf, new Path("ignored"));
    conf.setOutputFormat(NullOutputFormat.class);
    conf.setMapSpeculativeExecution(false);
    conf.setReduceSpeculativeExecution(false);

    // Set the following for reduce tasks to be able to be started running
    // immediately along with maps.
    conf.set("mapred.reduce.slowstart.completed.maps", String.valueOf(0));

    return conf;
  }

  @Override
  public int run(String[] args)
      throws Exception {
    numMappers = Integer.parseInt(args[0]);
    numReducers = Integer.parseInt(args[1]);
    return run(numMappers, numReducers);
  }

  @Override
  public int getPartition(IntWritable k, NullWritable v, int numPartitions) {
    return k.get() % numPartitions;
  }

  @Override
  public RecordReader<NullWritable, NullWritable> getRecordReader(
      InputSplit split, JobConf job, Reporter reporter) {
    LOG.debug("Inside RecordReader.getRecordReader");
    return new RecordReader<NullWritable, NullWritable>() {
      private int pos = 0;

      public void close() {
        // nothing
      }

      public NullWritable createKey() {
        return NullWritable.get();
      }

      public NullWritable createValue() {
        return NullWritable.get();
      }

      public long getPos() {
        return pos;
      }

      public float getProgress() {
        return pos * 100;
      }

      public boolean next(NullWritable key, NullWritable value) {
        if (pos++ == 0) {
          LOG.debug("Returning the next record");
          return true;
        }
        LOG.debug("No more records. Returning none.");
        return false;
      }
    };
  }

  @Override
  public InputSplit[] getSplits(JobConf job, int numSplits) {
    LOG.debug("Inside InputSplit.getSplits");
    InputSplit[] ret = new InputSplit[numSplits];
    for (int i = 0; i < numSplits; ++i) {
      ret[i] = new EmptySplit();
    }
    return ret;
  }

  public static class EmptySplit implements InputSplit {
    public void write(DataOutput out)
        throws IOException {
    }

    public void readFields(DataInput in)
        throws IOException {
    }

    public long getLength() {
      return 0L;
    }

    public String[] getLocations() {
      return new String[0];
    }
  }

  static class ControlledMapReduceJobRunner extends Thread {
    private JobConf conf;
    private ControlledMapReduceJob job;
    private JobID jobID;

    private int numMappers;
    private int numReducers;

    public ControlledMapReduceJobRunner() {
      this(new JobConf(), 5, 5);
    }

    public ControlledMapReduceJobRunner(JobConf cnf, int numMap, int numRed) {
      this.conf = cnf;
      this.numMappers = numMap;
      this.numReducers = numRed;
    }

    public ControlledMapReduceJob getJob() {
      while (job == null) {
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ie) {
          LOG.info(ControlledMapReduceJobRunner.class.getName()
              + " is interrupted.");
          break;
        }
      }
      return job;
    }

    public JobID getJobID()
        throws IOException {
      ControlledMapReduceJob job = getJob();
      JobID id = job.getJobId();
      while (id == null) {
        id = job.getJobId();
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ie) {
          LOG.info(ControlledMapReduceJobRunner.class.getName()
              + " is interrupted.");
          break;
        }
      }
      return id;
    }

    @Override
    public void run() {
      if (job != null) {
        LOG.warn("Job is already running.");
        return;
      }
      try {
        job = new ControlledMapReduceJob();
        int ret =
            ToolRunner.run(this.conf, job, new String[] {
                String.valueOf(numMappers), String.valueOf(numReducers) });
        LOG.info("Return value for the job : " + ret);
      } catch (Exception e) {
        LOG.warn("Caught exception : " + StringUtils.stringifyException(e));
      }
    }

    static ControlledMapReduceJobRunner getControlledMapReduceJobRunner(
        JobConf conf, int numMappers, int numReducers) {
      return new ControlledMapReduceJobRunner(conf, numMappers, numReducers);
    }
  }
}