source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @ 120

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.server.datanode;

import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import junit.framework.TestCase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.util.DataChecksum;

/**
 * this class tests the methods of the  SimulatedFSDataset.
 *
 */

public class TestSimulatedFSDataset extends TestCase {
  
  Configuration conf = null;
  

  
  static final int NUMBLOCKS = 20;
  static final int BLOCK_LENGTH_MULTIPLIER = 79;

  protected void setUp() throws Exception {
    super.setUp();
      conf = new Configuration();
      conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
 
  }

  protected void tearDown() throws Exception {
    super.tearDown();
  }
  
  long blockIdToLen(long blkid) {
    return blkid*BLOCK_LENGTH_MULTIPLIER;
  }
  
  int addSomeBlocks(FSDatasetInterface fsdataset, int startingBlockId) throws IOException {
    int bytesAdded = 0;
    for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) {
      Block b = new Block(i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual data written
      OutputStream dataOut  = fsdataset.writeToBlock(b, false).dataOut;
      assertEquals(0, fsdataset.getLength(b));
      for (int j=1; j <= blockIdToLen(i); ++j) {
        dataOut.write(j);
        assertEquals(j, fsdataset.getLength(b)); // correct length even as we write
        bytesAdded++;
      }
      dataOut.close();
      b.setNumBytes(blockIdToLen(i));
      fsdataset.finalizeBlock(b);
      assertEquals(blockIdToLen(i), fsdataset.getLength(b));
    }
    return bytesAdded;  
  }
  int addSomeBlocks(FSDatasetInterface fsdataset ) throws IOException {
    return addSomeBlocks(fsdataset, 1);
  }

  public void testGetMetaData() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block b = new Block(1, 5, 0);
    try {
      assertFalse(fsdataset.metaFileExists(b));
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
    addSomeBlocks(fsdataset); // Only need to add one but ....
    b = new Block(1, 0, 0);
    InputStream metaInput = fsdataset.getMetaDataInputStream(b);
    DataInputStream metaDataInput = new DataInputStream(metaInput);
    short version = metaDataInput.readShort();
    assertEquals(FSDataset.METADATA_VERSION, version);
    DataChecksum checksum = DataChecksum.newDataChecksum(metaDataInput);
    assertEquals(DataChecksum.CHECKSUM_NULL, checksum.getChecksumType());
    assertEquals(0, checksum.getChecksumSize());  
  }


  public void testStorageUsage() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    assertEquals(fsdataset.getDfsUsed(), 0);
    assertEquals(fsdataset.getRemaining(), fsdataset.getCapacity());
    int bytesAdded = addSomeBlocks(fsdataset);
    assertEquals(bytesAdded, fsdataset.getDfsUsed());
    assertEquals(fsdataset.getCapacity()-bytesAdded,  fsdataset.getRemaining());
    
  }



  void  checkBlockDataAndSize(FSDatasetInterface fsdataset, 
              Block b, long expectedLen) throws IOException { 
    InputStream input = fsdataset.getBlockInputStream(b);
    long lengthRead = 0;
    int data;
    while ((data = input.read()) != -1) {
      assertEquals(SimulatedFSDataset.DEFAULT_DATABYTE, data);
      lengthRead++;
    }
    assertEquals(expectedLen, lengthRead);
  }
  
  public void testWriteRead() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    addSomeBlocks(fsdataset);
    for (int i=1; i <= NUMBLOCKS; ++i) {
      Block b = new Block(i, 0, 0);
      assertTrue(fsdataset.isValidBlock(b));
      assertEquals(blockIdToLen(i), fsdataset.getLength(b));
      checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
    }
  }



  public void testGetBlockReport() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
  }
  public void testInjectionEmpty() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    
    // Inject blocks into an empty fsdataset
    //  - injecting the blocks we got above.
  
   
    SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
    sfsdataset.injectBlocks(blockReport);
    blockReport = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
  }

  public void testInjectionNonEmpty() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    
    Block[] blockReport = fsdataset.getBlockReport();
    assertEquals(0, blockReport.length);
    int bytesAdded = addSomeBlocks(fsdataset);
    blockReport = fsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
    }
    fsdataset = null;
    
    // Inject blocks into an non-empty fsdataset
    //  - injecting the blocks we got above.
  
   
    SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
    // Add come blocks whose block ids do not conflict with
    // the ones we are going to inject.
    bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1);
    Block[] blockReport2 = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    blockReport2 = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS, blockReport.length);
    sfsdataset.injectBlocks(blockReport);
    blockReport = sfsdataset.getBlockReport();
    assertEquals(NUMBLOCKS*2, blockReport.length);
    for (Block b: blockReport) {
      assertNotNull(b);
      assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
      assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
    }
    assertEquals(bytesAdded, sfsdataset.getDfsUsed());
    assertEquals(sfsdataset.getCapacity()-bytesAdded,  sfsdataset.getRemaining());
    
    
    // Now test that the dataset cannot be created if it does not have sufficient cap

    conf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY, 10);
 
    try {
      sfsdataset = new SimulatedFSDataset(conf);
      sfsdataset.injectBlocks(blockReport);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }

  }

  public void checkInvalidBlock(Block b) throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    assertFalse(fsdataset.isValidBlock(b));
    try {
      fsdataset.getLength(b);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
    
    try {
      fsdataset.getBlockInputStream(b);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
    
    try {
      fsdataset.finalizeBlock(b);
      assertTrue("Expected an IO exception", false);
    } catch (IOException e) {
      // ok - as expected
    }
    
  }
  
  public void testInValidBlocks() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    Block b = new Block(1, 5, 0);
    checkInvalidBlock(b);
    
    // Now check invlaid after adding some blocks
    addSomeBlocks(fsdataset);
    b = new Block(NUMBLOCKS + 99, 5, 0);
    checkInvalidBlock(b);
    
  }

  public void testInvalidate() throws IOException {
    FSDatasetInterface fsdataset = new SimulatedFSDataset(conf); 
    int bytesAdded = addSomeBlocks(fsdataset);
    Block[] deleteBlocks = new Block[2];
    deleteBlocks[0] = new Block(1, 0, 0);
    deleteBlocks[1] = new Block(2, 0, 0);
    fsdataset.invalidate(deleteBlocks);
    checkInvalidBlock(deleteBlocks[0]);
    checkInvalidBlock(deleteBlocks[1]);
    long sizeDeleted = blockIdToLen(1) + blockIdToLen(2);
    assertEquals(bytesAdded-sizeDeleted, fsdataset.getDfsUsed());
    assertEquals(fsdataset.getCapacity()-bytesAdded+sizeDeleted,  fsdataset.getRemaining());
    
    
    
    // Now make sure the rest of the blocks are valid
    for (int i=3; i <= NUMBLOCKS; ++i) {
      Block b = new Block(i, 0, 0);
      assertTrue(fsdataset.isValidBlock(b));
    }
  }

}