source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @ 120

Last change on this file since 120 was 120, checked in by (none), 14 years ago

Added the mail files for the Hadoop JUNit Project
  • Property svn:executable set to *
File size: 13.0 KB
Line 
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.hdfs.server.namenode;
20
21import java.io.ByteArrayOutputStream;
22import java.io.PrintStream;
23import java.net.InetSocketAddress;
24import java.io.File;
25import java.io.RandomAccessFile;
26import java.lang.Exception;
27import java.io.IOException;
28import java.nio.channels.FileChannel;
29import java.util.Random;
30
31import junit.framework.TestCase;
32
33import org.apache.commons.logging.impl.Log4JLogger;
34import org.apache.log4j.Level;
35import org.apache.hadoop.conf.Configuration;
36import org.apache.hadoop.fs.FileSystem;
37import org.apache.hadoop.fs.Path;
38import org.apache.hadoop.fs.FSDataOutputStream;
39import org.apache.hadoop.util.ToolRunner;
40import org.apache.hadoop.hdfs.DFSClient;
41import org.apache.hadoop.hdfs.DFSTestUtil;
42import org.apache.hadoop.hdfs.MiniDFSCluster;
43import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
44import org.apache.hadoop.hdfs.tools.DFSck;
45import org.apache.hadoop.io.IOUtils;
46
47/**
48 * A JUnit test for doing fsck
49 */
50public class TestFsck extends TestCase {
51  static String runFsck(Configuration conf, int expectedErrCode, 
52                        boolean checkErrorCode,String... path) 
53                        throws Exception {
54    PrintStream oldOut = System.out;
55    ByteArrayOutputStream bStream = new ByteArrayOutputStream();
56    PrintStream newOut = new PrintStream(bStream, true);
57    System.setOut(newOut);
58    ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.ALL);
59    int errCode = ToolRunner.run(new DFSck(conf), path);
60    if (checkErrorCode)
61      assertEquals(expectedErrCode, errCode);
62    ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.INFO);
63    System.setOut(oldOut);
64    return bStream.toString();
65  }
66
67  /** do fsck */
68  public void testFsck() throws Exception {
69    DFSTestUtil util = new DFSTestUtil("TestFsck", 20, 3, 8*1024);
70    MiniDFSCluster cluster = null;
71    FileSystem fs = null;
72    try {
73      Configuration conf = new Configuration();
74      conf.setLong("dfs.blockreport.intervalMsec", 10000L);
75      cluster = new MiniDFSCluster(conf, 4, true, null);
76      fs = cluster.getFileSystem();
77      util.createFiles(fs, "/srcdat");
78      util.waitReplication(fs, "/srcdat", (short)3);
79      String outStr = runFsck(conf, 0, true, "/");
80      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
81      System.out.println(outStr);
82      if (fs != null) {try{fs.close();} catch(Exception e){}}
83      cluster.shutdown();
84     
85      // restart the cluster; bring up namenode but not the data nodes
86      cluster = new MiniDFSCluster(conf, 0, false, null);
87      outStr = runFsck(conf, 1, true, "/");
88      // expect the result is corrupt
89      assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
90      System.out.println(outStr);
91     
92      // bring up data nodes & cleanup cluster
93      cluster.startDataNodes(conf, 4, true, null, null);
94      cluster.waitActive();
95      cluster.waitClusterUp();
96      fs = cluster.getFileSystem();
97      util.cleanup(fs, "/srcdat");
98    } finally {
99      if (fs != null) {try{fs.close();} catch(Exception e){}}
100      if (cluster != null) { cluster.shutdown(); }
101    }
102  }
103
104  public void testFsckNonExistent() throws Exception {
105    DFSTestUtil util = new DFSTestUtil("TestFsck", 20, 3, 8*1024);
106    MiniDFSCluster cluster = null;
107    FileSystem fs = null;
108    try {
109      Configuration conf = new Configuration();
110      conf.setLong("dfs.blockreport.intervalMsec", 10000L);
111      cluster = new MiniDFSCluster(conf, 4, true, null);
112      fs = cluster.getFileSystem();
113      util.createFiles(fs, "/srcdat");
114      util.waitReplication(fs, "/srcdat", (short)3);
115      String outStr = runFsck(conf, 0, true, "/non-existent");
116      assertEquals(-1, outStr.indexOf(NamenodeFsck.HEALTHY_STATUS));
117      System.out.println(outStr);
118      util.cleanup(fs, "/srcdat");
119    } finally {
120      if (fs != null) {try{fs.close();} catch(Exception e){}}
121      if (cluster != null) { cluster.shutdown(); }
122    }
123  }
124
125  public void testFsckMove() throws Exception {
126    DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, 8*1024);
127    MiniDFSCluster cluster = null;
128    FileSystem fs = null;
129    try {
130      Configuration conf = new Configuration();
131      conf.setLong("dfs.blockreport.intervalMsec", 10000L);
132      cluster = new MiniDFSCluster(conf, 4, true, null);
133      String topDir = "/srcdat";
134      fs = cluster.getFileSystem();
135      cluster.waitActive();
136      util.createFiles(fs, topDir);
137      util.waitReplication(fs, topDir, (short)3);
138      String outStr = runFsck(conf, 0, true, "/");
139      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
140     
141      // Corrupt a block by deleting it
142      String[] fileNames = util.getFileNames(topDir);
143      DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
144                                          cluster.getNameNodePort()), conf);
145      String block = dfsClient.namenode.
146                      getBlockLocations(fileNames[0], 0, Long.MAX_VALUE).
147                      get(0).getBlock().getBlockName();
148      File baseDir = new File(System.getProperty("test.build.data",
149                                                 "build/test/data"),"dfs/data");
150      for (int i=0; i<8; i++) {
151        File blockFile = new File(baseDir, "data" +(i+1)+ "/current/" + block);
152        if(blockFile.exists()) {
153          assertTrue(blockFile.delete());
154        }
155      }
156
157      // We excpect the filesystem to be corrupted
158      outStr = runFsck(conf, 1, false, "/");
159      while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) {
160        try {
161          Thread.sleep(100);
162        } catch (InterruptedException ignore) {
163        }
164        outStr = runFsck(conf, 1, false, "/");
165      } 
166     
167      // Fix the filesystem by moving corrupted files to lost+found
168      outStr = runFsck(conf, 1, true, "/", "-move");
169      assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
170     
171      // Check to make sure we have healthy filesystem
172      outStr = runFsck(conf, 0, true, "/");
173      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); 
174      util.cleanup(fs, topDir);
175      if (fs != null) {try{fs.close();} catch(Exception e){}}
176      cluster.shutdown();
177    } finally {
178      if (fs != null) {try{fs.close();} catch(Exception e){}}
179      if (cluster != null) { cluster.shutdown(); }
180    }
181  }
182 
183  public void testFsckOpenFiles() throws Exception {
184    DFSTestUtil util = new DFSTestUtil("TestFsck", 4, 3, 8*1024); 
185    MiniDFSCluster cluster = null;
186    FileSystem fs = null;
187    try {
188      Configuration conf = new Configuration();
189      conf.setLong("dfs.blockreport.intervalMsec", 10000L);
190      cluster = new MiniDFSCluster(conf, 4, true, null);
191      String topDir = "/srcdat";
192      String randomString = "HADOOP  ";
193      fs = cluster.getFileSystem();
194      cluster.waitActive();
195      util.createFiles(fs, topDir);
196      util.waitReplication(fs, topDir, (short)3);
197      String outStr = runFsck(conf, 0, true, "/");
198      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
199      // Open a file for writing and do not close for now
200      Path openFile = new Path(topDir + "/openFile");
201      FSDataOutputStream out = fs.create(openFile);
202      int writeCount = 0;
203      while (writeCount != 100) {
204        out.write(randomString.getBytes());
205        writeCount++;                 
206      }
207      // We expect the filesystem to be HEALTHY and show one open file
208      outStr = runFsck(conf, 0, true, topDir);
209      System.out.println(outStr);
210      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
211      assertFalse(outStr.contains("OPENFORWRITE")); 
212      // Use -openforwrite option to list open files
213      outStr = runFsck(conf, 0, true, topDir, "-openforwrite");
214      System.out.println(outStr);
215      assertTrue(outStr.contains("OPENFORWRITE"));
216      assertTrue(outStr.contains("openFile"));
217      // Close the file
218      out.close(); 
219      // Now, fsck should show HEALTHY fs and should not show any open files
220      outStr = runFsck(conf, 0, true, topDir);
221      System.out.println(outStr);
222      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
223      assertFalse(outStr.contains("OPENFORWRITE"));
224      util.cleanup(fs, topDir);
225      if (fs != null) {try{fs.close();} catch(Exception e){}}
226      cluster.shutdown();
227    } finally {
228      if (fs != null) {try{fs.close();} catch(Exception e){}}
229      if (cluster != null) { cluster.shutdown(); }
230    }
231  }
232
233  public void testCorruptBlock() throws Exception {
234    Configuration conf = new Configuration();
235    conf.setLong("dfs.blockreport.intervalMsec", 1000);
236    FileSystem fs = null;
237    DFSClient dfsClient = null;
238    LocatedBlocks blocks = null;
239    int replicaCount = 0;
240    Random random = new Random();
241    String outStr = null;
242
243    MiniDFSCluster cluster = null;
244    try {
245    cluster = new MiniDFSCluster(conf, 3, true, null);
246    cluster.waitActive();
247    fs = cluster.getFileSystem();
248    Path file1 = new Path("/testCorruptBlock");
249    DFSTestUtil.createFile(fs, file1, 1024, (short)3, 0);
250    // Wait until file replication has completed
251    DFSTestUtil.waitReplication(fs, file1, (short)3);
252    String block = DFSTestUtil.getFirstBlock(fs, file1).getBlockName();
253
254    // Make sure filesystem is in healthy state
255    outStr = runFsck(conf, 0, true, "/");
256    System.out.println(outStr);
257    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
258   
259    // corrupt replicas
260    File baseDir = new File(System.getProperty("test.build.data",
261                                               "build/test/data"),"dfs/data");
262    for (int i=0; i < 6; i++) {
263      File blockFile = new File(baseDir, "data" + (i+1) + "/current/" +
264                                block);
265      if (blockFile.exists()) {
266        RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
267        FileChannel channel = raFile.getChannel();
268        String badString = "BADBAD";
269        int rand = random.nextInt((int)channel.size()/2);
270        raFile.seek(rand);
271        raFile.write(badString.getBytes());
272        raFile.close();
273      }
274    }
275    // Read the file to trigger reportBadBlocks
276    try {
277      IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf,
278                        true);
279    } catch (IOException ie) {
280      // Ignore exception
281    }
282
283    dfsClient = new DFSClient(new InetSocketAddress("localhost",
284                               cluster.getNameNodePort()), conf);
285    blocks = dfsClient.namenode.
286               getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
287    replicaCount = blocks.get(0).getLocations().length;
288    while (replicaCount != 3) {
289      try {
290        Thread.sleep(100);
291      } catch (InterruptedException ignore) {
292      }
293      blocks = dfsClient.namenode.
294                getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
295      replicaCount = blocks.get(0).getLocations().length;
296    }
297    assertTrue (blocks.get(0).isCorrupt());
298
299    // Check if fsck reports the same
300    outStr = runFsck(conf, 1, true, "/");
301    System.out.println(outStr);
302    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
303    assertTrue(outStr.contains("testCorruptBlock"));
304    } finally {
305      if (cluster != null) {cluster.shutdown();}
306    }
307  }
308 
309  /** Test if fsck can return -1 in case of failure
310   *
311   * @throws Exception
312   */
313  public void testFsckError() throws Exception {
314    MiniDFSCluster cluster = null;
315    try {
316      // bring up a one-node cluster
317      Configuration conf = new Configuration();
318      cluster = new MiniDFSCluster(conf, 1, true, null);
319      String fileName = "/test.txt";
320      Path filePath = new Path(fileName);
321      FileSystem fs = cluster.getFileSystem();
322     
323      // create a one-block file
324      DFSTestUtil.createFile(fs, filePath, 1L, (short)1, 1L);
325      DFSTestUtil.waitReplication(fs, filePath, (short)1);
326     
327      // intentionally corrupt NN data structure
328      INodeFile node = (INodeFile)cluster.getNameNode().namesystem.dir.rootDir.getNode(fileName);
329      assertEquals(node.blocks.length, 1);
330      node.blocks[0].setNumBytes(-1L);  // set the block length to be negative
331     
332      // run fsck and expect a failure with -1 as the error code
333      String outStr = runFsck(conf, -1, true, fileName);
334      System.out.println(outStr);
335      assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS));
336     
337      // clean up file system
338      fs.delete(filePath, true);
339    } finally {
340      if (cluster != null) {cluster.shutdown();}
341    }
342  }
343}
Note: See TracBrowser for help on using the repository browser.