[120] | 1 | /** |
---|
| 2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
| 3 | * or more contributor license agreements. See the NOTICE file |
---|
| 4 | * distributed with this work for additional information |
---|
| 5 | * regarding copyright ownership. The ASF licenses this file |
---|
| 6 | * to you under the Apache License, Version 2.0 (the |
---|
| 7 | * "License"); you may not use this file except in compliance |
---|
| 8 | * with the License. You may obtain a copy of the License at |
---|
| 9 | * |
---|
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | * |
---|
| 12 | * Unless required by applicable law or agreed to in writing, software |
---|
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | * See the License for the specific language governing permissions and |
---|
| 16 | * limitations under the License. |
---|
| 17 | */ |
---|
| 18 | package org.apache.hadoop.hdfs; |
---|
| 19 | |
---|
| 20 | import java.io.IOException; |
---|
| 21 | |
---|
| 22 | import org.apache.commons.logging.impl.Log4JLogger; |
---|
| 23 | import org.apache.hadoop.conf.Configuration; |
---|
| 24 | import org.apache.hadoop.fs.FSDataOutputStream; |
---|
| 25 | import org.apache.hadoop.fs.FileSystem; |
---|
| 26 | import org.apache.hadoop.fs.Path; |
---|
| 27 | import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; |
---|
| 28 | import org.apache.hadoop.hdfs.server.namenode.LeaseManager; |
---|
| 29 | import org.apache.hadoop.hdfs.server.namenode.NameNode; |
---|
| 30 | import org.apache.log4j.Level; |
---|
| 31 | |
---|
| 32 | public class TestRenameWhileOpen extends junit.framework.TestCase { |
---|
| 33 | { |
---|
| 34 | ((Log4JLogger)NameNode.stateChangeLog).getLogger().setLevel(Level.ALL); |
---|
| 35 | ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL); |
---|
| 36 | ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL); |
---|
| 37 | } |
---|
| 38 | |
---|
| 39 | /** |
---|
| 40 | * open /user/dir1/file1 /user/dir2/file2 |
---|
| 41 | * mkdir /user/dir3 |
---|
| 42 | * move /user/dir1 /user/dir3 |
---|
| 43 | */ |
---|
| 44 | public void testWhileOpenRenameParent() throws IOException { |
---|
| 45 | Configuration conf = new Configuration(); |
---|
| 46 | final int MAX_IDLE_TIME = 2000; // 2s |
---|
| 47 | conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); |
---|
| 48 | conf.setInt("heartbeat.recheck.interval", 1000); |
---|
| 49 | conf.setInt("dfs.heartbeat.interval", 1); |
---|
| 50 | conf.setInt("dfs.safemode.threshold.pct", 1); |
---|
| 51 | conf.setBoolean("dfs.support.append", true); |
---|
| 52 | |
---|
| 53 | // create cluster |
---|
| 54 | System.out.println("Test 1*****************************"); |
---|
| 55 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
| 56 | FileSystem fs = null; |
---|
| 57 | try { |
---|
| 58 | cluster.waitActive(); |
---|
| 59 | fs = cluster.getFileSystem(); |
---|
| 60 | final int nnport = cluster.getNameNodePort(); |
---|
| 61 | |
---|
| 62 | // create file1. |
---|
| 63 | Path dir1 = new Path("/user/a+b/dir1"); |
---|
| 64 | Path file1 = new Path(dir1, "file1"); |
---|
| 65 | FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); |
---|
| 66 | System.out.println("testFileCreationDeleteParent: " |
---|
| 67 | + "Created file " + file1); |
---|
| 68 | TestFileCreation.writeFile(stm1); |
---|
| 69 | stm1.sync(); |
---|
| 70 | |
---|
| 71 | // create file2. |
---|
| 72 | Path dir2 = new Path("/user/dir2"); |
---|
| 73 | Path file2 = new Path(dir2, "file2"); |
---|
| 74 | FSDataOutputStream stm2 = TestFileCreation.createFile(fs, file2, 1); |
---|
| 75 | System.out.println("testFileCreationDeleteParent: " |
---|
| 76 | + "Created file " + file2); |
---|
| 77 | TestFileCreation.writeFile(stm2); |
---|
| 78 | stm2.sync(); |
---|
| 79 | |
---|
| 80 | // move dir1 while file1 is open |
---|
| 81 | Path dir3 = new Path("/user/dir3"); |
---|
| 82 | fs.mkdirs(dir3); |
---|
| 83 | fs.rename(dir1, dir3); |
---|
| 84 | |
---|
| 85 | // create file3 |
---|
| 86 | Path file3 = new Path(dir3, "file3"); |
---|
| 87 | FSDataOutputStream stm3 = TestFileCreation.createFile(fs, file3, 1); |
---|
| 88 | TestFileCreation.writeFile(stm3); |
---|
| 89 | // rename file3 to some bad name |
---|
| 90 | try { |
---|
| 91 | fs.rename(file3, new Path(dir3, "$ ")); |
---|
| 92 | } catch(Exception e) { |
---|
| 93 | e.printStackTrace(); |
---|
| 94 | } |
---|
| 95 | |
---|
| 96 | // restart cluster with the same namenode port as before. |
---|
| 97 | // This ensures that leases are persisted in fsimage. |
---|
| 98 | cluster.shutdown(); |
---|
| 99 | try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} |
---|
| 100 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 101 | null, null, null); |
---|
| 102 | cluster.waitActive(); |
---|
| 103 | |
---|
| 104 | // restart cluster yet again. This triggers the code to read in |
---|
| 105 | // persistent leases from fsimage. |
---|
| 106 | cluster.shutdown(); |
---|
| 107 | try {Thread.sleep(5000);} catch (InterruptedException e) {} |
---|
| 108 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 109 | null, null, null); |
---|
| 110 | cluster.waitActive(); |
---|
| 111 | fs = cluster.getFileSystem(); |
---|
| 112 | |
---|
| 113 | Path newfile = new Path("/user/dir3/dir1", "file1"); |
---|
| 114 | assertTrue(!fs.exists(file1)); |
---|
| 115 | assertTrue(fs.exists(file2)); |
---|
| 116 | assertTrue(fs.exists(newfile)); |
---|
| 117 | TestFileCreation.checkFullFile(fs, newfile); |
---|
| 118 | } finally { |
---|
| 119 | fs.close(); |
---|
| 120 | cluster.shutdown(); |
---|
| 121 | } |
---|
| 122 | } |
---|
| 123 | |
---|
| 124 | /** |
---|
| 125 | * open /user/dir1/file1 /user/dir2/file2 |
---|
| 126 | * move /user/dir1 /user/dir3 |
---|
| 127 | */ |
---|
| 128 | public void testWhileOpenRenameParentToNonexistentDir() throws IOException { |
---|
| 129 | Configuration conf = new Configuration(); |
---|
| 130 | final int MAX_IDLE_TIME = 2000; // 2s |
---|
| 131 | conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); |
---|
| 132 | conf.setInt("heartbeat.recheck.interval", 1000); |
---|
| 133 | conf.setInt("dfs.heartbeat.interval", 1); |
---|
| 134 | conf.setInt("dfs.safemode.threshold.pct", 1); |
---|
| 135 | conf.setBoolean("dfs.support.append", true); |
---|
| 136 | System.out.println("Test 2************************************"); |
---|
| 137 | |
---|
| 138 | // create cluster |
---|
| 139 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
| 140 | FileSystem fs = null; |
---|
| 141 | try { |
---|
| 142 | cluster.waitActive(); |
---|
| 143 | fs = cluster.getFileSystem(); |
---|
| 144 | final int nnport = cluster.getNameNodePort(); |
---|
| 145 | |
---|
| 146 | // create file1. |
---|
| 147 | Path dir1 = new Path("/user/dir1"); |
---|
| 148 | Path file1 = new Path(dir1, "file1"); |
---|
| 149 | FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); |
---|
| 150 | System.out.println("testFileCreationDeleteParent: " |
---|
| 151 | + "Created file " + file1); |
---|
| 152 | TestFileCreation.writeFile(stm1); |
---|
| 153 | stm1.sync(); |
---|
| 154 | |
---|
| 155 | // create file2. |
---|
| 156 | Path dir2 = new Path("/user/dir2"); |
---|
| 157 | Path file2 = new Path(dir2, "file2"); |
---|
| 158 | FSDataOutputStream stm2 = TestFileCreation.createFile(fs, file2, 1); |
---|
| 159 | System.out.println("testFileCreationDeleteParent: " |
---|
| 160 | + "Created file " + file2); |
---|
| 161 | TestFileCreation.writeFile(stm2); |
---|
| 162 | stm2.sync(); |
---|
| 163 | |
---|
| 164 | // move dir1 while file1 is open |
---|
| 165 | Path dir3 = new Path("/user/dir3"); |
---|
| 166 | fs.rename(dir1, dir3); |
---|
| 167 | |
---|
| 168 | // restart cluster with the same namenode port as before. |
---|
| 169 | // This ensures that leases are persisted in fsimage. |
---|
| 170 | cluster.shutdown(); |
---|
| 171 | try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} |
---|
| 172 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 173 | null, null, null); |
---|
| 174 | cluster.waitActive(); |
---|
| 175 | |
---|
| 176 | // restart cluster yet again. This triggers the code to read in |
---|
| 177 | // persistent leases from fsimage. |
---|
| 178 | cluster.shutdown(); |
---|
| 179 | try {Thread.sleep(5000);} catch (InterruptedException e) {} |
---|
| 180 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 181 | null, null, null); |
---|
| 182 | cluster.waitActive(); |
---|
| 183 | fs = cluster.getFileSystem(); |
---|
| 184 | |
---|
| 185 | Path newfile = new Path("/user/dir3", "file1"); |
---|
| 186 | assertTrue(!fs.exists(file1)); |
---|
| 187 | assertTrue(fs.exists(file2)); |
---|
| 188 | assertTrue(fs.exists(newfile)); |
---|
| 189 | TestFileCreation.checkFullFile(fs, newfile); |
---|
| 190 | } finally { |
---|
| 191 | fs.close(); |
---|
| 192 | cluster.shutdown(); |
---|
| 193 | } |
---|
| 194 | } |
---|
| 195 | |
---|
| 196 | /** |
---|
| 197 | * open /user/dir1/file1 |
---|
| 198 | * mkdir /user/dir2 |
---|
| 199 | * move /user/dir1/file1 /user/dir2/ |
---|
| 200 | */ |
---|
| 201 | public void testWhileOpenRenameToExistentDirectory() throws IOException { |
---|
| 202 | Configuration conf = new Configuration(); |
---|
| 203 | final int MAX_IDLE_TIME = 2000; // 2s |
---|
| 204 | conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); |
---|
| 205 | conf.setInt("heartbeat.recheck.interval", 1000); |
---|
| 206 | conf.setInt("dfs.heartbeat.interval", 1); |
---|
| 207 | conf.setInt("dfs.safemode.threshold.pct", 1); |
---|
| 208 | conf.setBoolean("dfs.support.append", true); |
---|
| 209 | System.out.println("Test 3************************************"); |
---|
| 210 | |
---|
| 211 | // create cluster |
---|
| 212 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
| 213 | FileSystem fs = null; |
---|
| 214 | try { |
---|
| 215 | cluster.waitActive(); |
---|
| 216 | fs = cluster.getFileSystem(); |
---|
| 217 | final int nnport = cluster.getNameNodePort(); |
---|
| 218 | |
---|
| 219 | // create file1. |
---|
| 220 | Path dir1 = new Path("/user/dir1"); |
---|
| 221 | Path file1 = new Path(dir1, "file1"); |
---|
| 222 | FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); |
---|
| 223 | System.out.println("testFileCreationDeleteParent: " + |
---|
| 224 | "Created file " + file1); |
---|
| 225 | TestFileCreation.writeFile(stm1); |
---|
| 226 | stm1.sync(); |
---|
| 227 | |
---|
| 228 | Path dir2 = new Path("/user/dir2"); |
---|
| 229 | fs.mkdirs(dir2); |
---|
| 230 | |
---|
| 231 | fs.rename(file1, dir2); |
---|
| 232 | |
---|
| 233 | // restart cluster with the same namenode port as before. |
---|
| 234 | // This ensures that leases are persisted in fsimage. |
---|
| 235 | cluster.shutdown(); |
---|
| 236 | try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} |
---|
| 237 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 238 | null, null, null); |
---|
| 239 | cluster.waitActive(); |
---|
| 240 | |
---|
| 241 | // restart cluster yet again. This triggers the code to read in |
---|
| 242 | // persistent leases from fsimage. |
---|
| 243 | cluster.shutdown(); |
---|
| 244 | try {Thread.sleep(5000);} catch (InterruptedException e) {} |
---|
| 245 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 246 | null, null, null); |
---|
| 247 | cluster.waitActive(); |
---|
| 248 | fs = cluster.getFileSystem(); |
---|
| 249 | |
---|
| 250 | Path newfile = new Path("/user/dir2", "file1"); |
---|
| 251 | assertTrue(!fs.exists(file1)); |
---|
| 252 | assertTrue(fs.exists(newfile)); |
---|
| 253 | TestFileCreation.checkFullFile(fs, newfile); |
---|
| 254 | } finally { |
---|
| 255 | fs.close(); |
---|
| 256 | cluster.shutdown(); |
---|
| 257 | } |
---|
| 258 | } |
---|
| 259 | |
---|
| 260 | /** |
---|
| 261 | * open /user/dir1/file1 |
---|
| 262 | * move /user/dir1/file1 /user/dir2/ |
---|
| 263 | */ |
---|
| 264 | public void testWhileOpenRenameToNonExistentDirectory() throws IOException { |
---|
| 265 | Configuration conf = new Configuration(); |
---|
| 266 | final int MAX_IDLE_TIME = 2000; // 2s |
---|
| 267 | conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); |
---|
| 268 | conf.setInt("heartbeat.recheck.interval", 1000); |
---|
| 269 | conf.setInt("dfs.heartbeat.interval", 1); |
---|
| 270 | conf.setInt("dfs.safemode.threshold.pct", 1); |
---|
| 271 | conf.setBoolean("dfs.support.append", true); |
---|
| 272 | System.out.println("Test 4************************************"); |
---|
| 273 | |
---|
| 274 | // create cluster |
---|
| 275 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
| 276 | FileSystem fs = null; |
---|
| 277 | try { |
---|
| 278 | cluster.waitActive(); |
---|
| 279 | fs = cluster.getFileSystem(); |
---|
| 280 | final int nnport = cluster.getNameNodePort(); |
---|
| 281 | |
---|
| 282 | // create file1. |
---|
| 283 | Path dir1 = new Path("/user/dir1"); |
---|
| 284 | Path file1 = new Path(dir1, "file1"); |
---|
| 285 | FSDataOutputStream stm1 = TestFileCreation.createFile(fs, file1, 1); |
---|
| 286 | System.out.println("testFileCreationDeleteParent: " |
---|
| 287 | + "Created file " + file1); |
---|
| 288 | TestFileCreation.writeFile(stm1); |
---|
| 289 | stm1.sync(); |
---|
| 290 | |
---|
| 291 | Path dir2 = new Path("/user/dir2"); |
---|
| 292 | |
---|
| 293 | fs.rename(file1, dir2); |
---|
| 294 | |
---|
| 295 | // restart cluster with the same namenode port as before. |
---|
| 296 | // This ensures that leases are persisted in fsimage. |
---|
| 297 | cluster.shutdown(); |
---|
| 298 | try {Thread.sleep(2*MAX_IDLE_TIME);} catch (InterruptedException e) {} |
---|
| 299 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 300 | null, null, null); |
---|
| 301 | cluster.waitActive(); |
---|
| 302 | |
---|
| 303 | // restart cluster yet again. This triggers the code to read in |
---|
| 304 | // persistent leases from fsimage. |
---|
| 305 | cluster.shutdown(); |
---|
| 306 | try {Thread.sleep(5000);} catch (InterruptedException e) {} |
---|
| 307 | cluster = new MiniDFSCluster(nnport, conf, 1, false, true, |
---|
| 308 | null, null, null); |
---|
| 309 | cluster.waitActive(); |
---|
| 310 | fs = cluster.getFileSystem(); |
---|
| 311 | |
---|
| 312 | Path newfile = new Path("/user", "dir2"); |
---|
| 313 | assertTrue(!fs.exists(file1)); |
---|
| 314 | assertTrue(fs.exists(newfile)); |
---|
| 315 | TestFileCreation.checkFullFile(fs, newfile); |
---|
| 316 | } finally { |
---|
| 317 | fs.close(); |
---|
| 318 | cluster.shutdown(); |
---|
| 319 | } |
---|
| 320 | } |
---|
| 321 | } |
---|