source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/mapred/TestJobTrackerSafeMode.java @ 120

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.mapred;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.MiniDFSCluster;

import junit.framework.TestCase;
import java.io.*;
import java.util.HashSet;
import java.util.Set;

/** 
 * This test checks jobtracker in safe mode. In safe mode the jobtracker upon 
 * restart doesnt schedule any new tasks and waits for the (old) trackers to 
 * join back.
 */
public class TestJobTrackerSafeMode extends TestCase {
  final Path testDir = 
    new Path(System.getProperty("test.build.data", "/tmp"), "jt-safemode");
  final Path inDir = new Path(testDir, "input");
  final Path shareDir = new Path(testDir, "share");
  final Path outputDir = new Path(testDir, "output");
  final int numDir = 1;
  final int numTrackers = 2;
  
  private static final Log LOG = 
    LogFactory.getLog(TestJobTrackerSafeMode.class);
  
  private JobConf configureJob(JobConf conf, int maps, int reduces,
                               String mapSignal, String redSignal) 
  throws IOException {
    UtilsForTests.configureWaitingJobConf(conf, inDir, outputDir, 
        maps, reduces, "test-jobtracker-safemode", 
        mapSignal, redSignal);
    return conf;
  }
  
  /**
   * Tests the jobtracker's safemode. The test is as follows : 
   *   - starts a cluster with 2 trackers
   *   - submits a job with large (40) maps to make sure that all the trackers 
   *     are logged to the job history
   *   - wait for the job to be 50% done
   *   - stop the jobtracker
   *   - wait for the trackers to be done with all the tasks
   *   - kill a task tracker
   *   - start the jobtracker
   *   - start 2 more trackers
   *   - now check that while all the tracker are detected (or lost) the 
   *     scheduling window is closed
   *   - check that after all the trackers are recovered, scheduling is opened 
   */
  private void testSafeMode(MiniDFSCluster dfs, MiniMRCluster mr) 
  throws IOException {
    FileSystem fileSys = dfs.getFileSystem();
    JobConf jobConf = mr.createJobConf();
    String mapSignalFile = UtilsForTests.getMapSignalFile(shareDir);
    String redSignalFile = UtilsForTests.getReduceSignalFile(shareDir);
    JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
    int numTracker = jobtracker.getClusterStatus(false).getTaskTrackers();
    
    // Configure the jobs
    JobConf job = configureJob(jobConf, 40, 0, mapSignalFile, redSignalFile);
      
    fileSys.delete(shareDir, true);
    
    // Submit a master job   
    JobClient jobClient = new JobClient(job);
    RunningJob rJob = jobClient.submitJob(job);
    JobID id = rJob.getID();
    
    // wait for the job to be inited
    mr.initializeJob(id);
    
    // Make sure that the master job is 50% completed
    while (UtilsForTests.getJobStatus(jobClient, id).mapProgress() 
           < 0.5f) {
      LOG.info("Waiting for the job to be 50% done");
      UtilsForTests.waitFor(100);
    }

    // Kill the jobtracker
    mr.stopJobTracker();

    // Enable recovery on restart
    mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover", 
                                      true);
    
    // Signal the maps to complete
    UtilsForTests.signalTasks(dfs, fileSys, true, mapSignalFile, redSignalFile);
    
    // Signal the reducers to complete
    UtilsForTests.signalTasks(dfs, fileSys, false, mapSignalFile, 
                              redSignalFile);
    
    // wait for the tasks to complete at the tracker
    Set<String> trackers = new HashSet<String>();
    for (int i = 0 ; i < numTracker; ++i) {
      TaskTracker t = mr.getTaskTrackerRunner(i).getTaskTracker();
      trackers.add(t.getName());
      int runningCount = t.getRunningTaskStatuses().size();
      while (runningCount != 0) {
        LOG.info("Waiting for tracker " + t.getName() + " to stabilize");
        UtilsForTests.waitFor(100);
        runningCount = 0;
        for (TaskStatus status : t.getRunningTaskStatuses()) {
          if (status.getIsMap() 
              && (status.getRunState() == TaskStatus.State.UNASSIGNED 
                  || status.getRunState() == TaskStatus.State.RUNNING)) {
            ++runningCount;
          }
        }
      }
    }

    LOG.info("Trackers have stabilized");
    
    // Kill a tasktracker
    int trackerToKill = --numTracker;
    TaskTracker t = mr.getTaskTrackerRunner(trackerToKill).getTaskTracker();
    
    trackers.remove(t.getName()); // remove this from the set to check
    
    Set<String> lostTrackers = new HashSet<String>();
    lostTrackers.add(t.getName());
    
    // get the attempt-id's to ignore
    // stop the tracker
    LOG.info("Stopping tracker : " + t.getName());
    mr.getTaskTrackerRunner(trackerToKill).getTaskTracker().shutdown();
    mr.stopTaskTracker(trackerToKill);

    // Restart the jobtracker
    mr.startJobTracker();

    // Wait for the JT to be ready
    UtilsForTests.waitForJobTracker(jobClient);

    jobtracker = mr.getJobTrackerRunner().getJobTracker();

    // Start a tracker
    LOG.info("Start a new tracker");
    mr.startTaskTracker(null, null, ++numTracker, numDir);
    
    // Start a tracker
    LOG.info("Start a new tracker");
    mr.startTaskTracker(null, null, ++numTracker, numDir);

    // Check if the jobs are still running
    
    // Wait for the tracker to be lost
    boolean shouldSchedule = jobtracker.recoveryManager.shouldSchedule();
    while (!checkTrackers(jobtracker, trackers, lostTrackers)) {
      assertFalse("JobTracker has opened up scheduling before all the" 
                  + " trackers were recovered", shouldSchedule);
      UtilsForTests.waitFor(100);
      
      // snapshot jobtracker's scheduling status
      shouldSchedule = jobtracker.recoveryManager.shouldSchedule();
    }

    assertTrue("JobTracker hasnt opened up scheduling even all the" 
               + " trackers were recovered", 
               jobtracker.recoveryManager.shouldSchedule());
    
    assertEquals("Recovery manager is in inconsistent state", 
                 0, jobtracker.recoveryManager.recoveredTrackers.size());
    
    // wait for the job to be complete
    UtilsForTests.waitTillDone(jobClient);
  }

  private boolean checkTrackers(JobTracker jobtracker, Set<String> present, 
                                Set<String> absent) {
    long jobtrackerRecoveryFinishTime = 
      jobtracker.getStartTime() + jobtracker.getRecoveryDuration();
    for (String trackerName : present) {
      TaskTrackerStatus status = jobtracker.getTaskTracker(trackerName);
      // check if the status is present and also the tracker has contacted back
      // after restart
      if (status == null 
          || status.getLastSeen() < jobtrackerRecoveryFinishTime) {
        return false;
      }
    }
    for (String trackerName : absent) {
      TaskTrackerStatus status = jobtracker.getTaskTracker(trackerName);
      // check if the status is still present
      if ( status != null) {
        return false;
      }
    }
    return true;
  }

  /**
   * Test {@link JobTracker}'s safe mode.
   */
  public void testJobTrackerSafeMode() throws IOException {
    String namenode = null;
    MiniDFSCluster dfs = null;
    MiniMRCluster mr = null;
    FileSystem fileSys = null;

    try {
      Configuration conf = new Configuration();
      conf.setBoolean("dfs.replication.considerLoad", false);
      dfs = new MiniDFSCluster(conf, 1, true, null, null);
      dfs.waitActive();
      fileSys = dfs.getFileSystem();
      
      // clean up
      fileSys.delete(testDir, true);
      
      if (!fileSys.mkdirs(inDir)) {
        throw new IOException("Mkdirs failed to create " + inDir.toString());
      }

      // Write the input file
      UtilsForTests.writeFile(dfs.getNameNode(), conf, 
                              new Path(inDir + "/file"), (short)1);

      dfs.startDataNodes(conf, 1, true, null, null, null, null);
      dfs.waitActive();

      namenode = (dfs.getFileSystem()).getUri().getHost() + ":" 
                 + (dfs.getFileSystem()).getUri().getPort();

      // Make sure that jobhistory leads to a proper job restart
      // So keep the blocksize and the buffer size small
      JobConf jtConf = new JobConf();
      jtConf.set("mapred.jobtracker.job.history.block.size", "512");
      jtConf.set("mapred.jobtracker.job.history.buffer.size", "512");
      jtConf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
      jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
      jtConf.setLong("mapred.tasktracker.expiry.interval", 5000);
      jtConf.setInt("mapred.reduce.copy.backoff", 4);
      jtConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
      
      mr = new MiniMRCluster(numTrackers, namenode, numDir, null, null, jtConf);
      
      // Test Lost tracker case
      testSafeMode(dfs, mr);
    } finally {
      if (mr != null) {
        try {
          mr.shutdown();
        } catch (Exception e) {}
      }
      if (dfs != null) {
        try {
          dfs.shutdown();
        } catch (Exception e) {}
      }
    }
  }

  public static void main(String[] args) throws IOException {
    new TestJobTrackerSafeMode().testJobTrackerSafeMode();
  }
}