[120] | 1 | /** |
---|
| 2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
| 3 | * or more contributor license agreements. See the NOTICE file |
---|
| 4 | * distributed with this work for additional information |
---|
| 5 | * regarding copyright ownership. The ASF licenses this file |
---|
| 6 | * to you under the Apache License, Version 2.0 (the |
---|
| 7 | * "License"); you may not use this file except in compliance |
---|
| 8 | * with the License. You may obtain a copy of the License at |
---|
| 9 | * |
---|
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | * |
---|
| 12 | * Unless required by applicable law or agreed to in writing, software |
---|
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | * See the License for the specific language governing permissions and |
---|
| 16 | * limitations under the License. |
---|
| 17 | */ |
---|
| 18 | package org.apache.hadoop.hdfs; |
---|
| 19 | |
---|
| 20 | import java.io.ByteArrayOutputStream; |
---|
| 21 | import java.io.DataOutputStream; |
---|
| 22 | import java.io.File; |
---|
| 23 | import java.io.IOException; |
---|
| 24 | import java.io.OutputStream; |
---|
| 25 | import java.io.PrintStream; |
---|
| 26 | import java.io.PrintWriter; |
---|
| 27 | import java.security.Permission; |
---|
| 28 | import java.util.ArrayList; |
---|
| 29 | import java.util.Arrays; |
---|
| 30 | import java.util.List; |
---|
| 31 | import java.util.Random; |
---|
| 32 | import java.util.Scanner; |
---|
| 33 | import java.util.zip.GZIPOutputStream; |
---|
| 34 | |
---|
| 35 | import junit.framework.TestCase; |
---|
| 36 | |
---|
| 37 | import org.apache.hadoop.conf.Configuration; |
---|
| 38 | import org.apache.hadoop.fs.FSInputChecker; |
---|
| 39 | import org.apache.hadoop.fs.FileSystem; |
---|
| 40 | import org.apache.hadoop.fs.FsShell; |
---|
| 41 | import org.apache.hadoop.fs.Path; |
---|
| 42 | import org.apache.hadoop.fs.permission.FsPermission; |
---|
| 43 | import org.apache.hadoop.fs.shell.Count; |
---|
| 44 | import org.apache.hadoop.hdfs.protocol.Block; |
---|
| 45 | import org.apache.hadoop.hdfs.server.datanode.DataNode; |
---|
| 46 | import org.apache.hadoop.hdfs.server.datanode.FSDataset; |
---|
| 47 | import org.apache.hadoop.io.IOUtils; |
---|
| 48 | import org.apache.hadoop.security.UnixUserGroupInformation; |
---|
| 49 | import org.apache.hadoop.security.UserGroupInformation; |
---|
| 50 | import org.apache.hadoop.util.StringUtils; |
---|
| 51 | import org.apache.hadoop.util.ToolRunner; |
---|
| 52 | |
---|
| 53 | /** |
---|
| 54 | * This class tests commands from DFSShell. |
---|
| 55 | */ |
---|
| 56 | public class TestDFSShell extends TestCase { |
---|
| 57 | static final String TEST_ROOT_DIR = |
---|
| 58 | new Path(System.getProperty("test.build.data","/tmp")) |
---|
| 59 | .toString().replace(' ', '+'); |
---|
| 60 | |
---|
| 61 | static Path writeFile(FileSystem fs, Path f) throws IOException { |
---|
| 62 | DataOutputStream out = fs.create(f); |
---|
| 63 | out.writeBytes("dhruba: " + f); |
---|
| 64 | out.close(); |
---|
| 65 | assertTrue(fs.exists(f)); |
---|
| 66 | return f; |
---|
| 67 | } |
---|
| 68 | |
---|
| 69 | static Path mkdir(FileSystem fs, Path p) throws IOException { |
---|
| 70 | assertTrue(fs.mkdirs(p)); |
---|
| 71 | assertTrue(fs.exists(p)); |
---|
| 72 | assertTrue(fs.getFileStatus(p).isDir()); |
---|
| 73 | return p; |
---|
| 74 | } |
---|
| 75 | |
---|
| 76 | static File createLocalFile(File f) throws IOException { |
---|
| 77 | assertTrue(!f.exists()); |
---|
| 78 | PrintWriter out = new PrintWriter(f); |
---|
| 79 | out.print("createLocalFile: " + f.getAbsolutePath()); |
---|
| 80 | out.flush(); |
---|
| 81 | out.close(); |
---|
| 82 | assertTrue(f.exists()); |
---|
| 83 | assertTrue(f.isFile()); |
---|
| 84 | return f; |
---|
| 85 | } |
---|
| 86 | |
---|
| 87 | static void show(String s) { |
---|
| 88 | System.out.println(Thread.currentThread().getStackTrace()[2] + " " + s); |
---|
| 89 | } |
---|
| 90 | |
---|
| 91 | public void testZeroSizeFile() throws IOException { |
---|
| 92 | Configuration conf = new Configuration(); |
---|
| 93 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 94 | FileSystem fs = cluster.getFileSystem(); |
---|
| 95 | assertTrue("Not a HDFS: "+fs.getUri(), |
---|
| 96 | fs instanceof DistributedFileSystem); |
---|
| 97 | final DistributedFileSystem dfs = (DistributedFileSystem)fs; |
---|
| 98 | |
---|
| 99 | try { |
---|
| 100 | //create a zero size file |
---|
| 101 | final File f1 = new File(TEST_ROOT_DIR, "f1"); |
---|
| 102 | assertTrue(!f1.exists()); |
---|
| 103 | assertTrue(f1.createNewFile()); |
---|
| 104 | assertTrue(f1.exists()); |
---|
| 105 | assertTrue(f1.isFile()); |
---|
| 106 | assertEquals(0L, f1.length()); |
---|
| 107 | |
---|
| 108 | //copy to remote |
---|
| 109 | final Path root = mkdir(dfs, new Path("/test/zeroSizeFile")); |
---|
| 110 | final Path remotef = new Path(root, "dst"); |
---|
| 111 | show("copy local " + f1 + " to remote " + remotef); |
---|
| 112 | dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef); |
---|
| 113 | |
---|
| 114 | //getBlockSize() should not throw exception |
---|
| 115 | show("Block size = " + dfs.getFileStatus(remotef).getBlockSize()); |
---|
| 116 | |
---|
| 117 | //copy back |
---|
| 118 | final File f2 = new File(TEST_ROOT_DIR, "f2"); |
---|
| 119 | assertTrue(!f2.exists()); |
---|
| 120 | dfs.copyToLocalFile(remotef, new Path(f2.getPath())); |
---|
| 121 | assertTrue(f2.exists()); |
---|
| 122 | assertTrue(f2.isFile()); |
---|
| 123 | assertEquals(0L, f2.length()); |
---|
| 124 | |
---|
| 125 | f1.delete(); |
---|
| 126 | f2.delete(); |
---|
| 127 | } finally { |
---|
| 128 | try {dfs.close();} catch (Exception e) {} |
---|
| 129 | cluster.shutdown(); |
---|
| 130 | } |
---|
| 131 | } |
---|
| 132 | |
---|
| 133 | public void testRecrusiveRm() throws IOException { |
---|
| 134 | Configuration conf = new Configuration(); |
---|
| 135 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 136 | FileSystem fs = cluster.getFileSystem(); |
---|
| 137 | assertTrue("Not a HDFS: " + fs.getUri(), |
---|
| 138 | fs instanceof DistributedFileSystem); |
---|
| 139 | try { |
---|
| 140 | fs.mkdirs(new Path(new Path("parent"), "child")); |
---|
| 141 | try { |
---|
| 142 | fs.delete(new Path("parent"), false); |
---|
| 143 | assert(false); // should never reach here. |
---|
| 144 | } catch(IOException e) { |
---|
| 145 | //should have thrown an exception |
---|
| 146 | } |
---|
| 147 | try { |
---|
| 148 | fs.delete(new Path("parent"), true); |
---|
| 149 | } catch(IOException e) { |
---|
| 150 | assert(false); |
---|
| 151 | } |
---|
| 152 | } finally { |
---|
| 153 | try { fs.close();}catch(IOException e){}; |
---|
| 154 | cluster.shutdown(); |
---|
| 155 | } |
---|
| 156 | } |
---|
| 157 | |
---|
| 158 | public void testDu() throws IOException { |
---|
| 159 | Configuration conf = new Configuration(); |
---|
| 160 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 161 | FileSystem fs = cluster.getFileSystem(); |
---|
| 162 | assertTrue("Not a HDFS: "+fs.getUri(), |
---|
| 163 | fs instanceof DistributedFileSystem); |
---|
| 164 | final DistributedFileSystem dfs = (DistributedFileSystem)fs; |
---|
| 165 | PrintStream psBackup = System.out; |
---|
| 166 | ByteArrayOutputStream out = new ByteArrayOutputStream(); |
---|
| 167 | PrintStream psOut = new PrintStream(out); |
---|
| 168 | System.setOut(psOut); |
---|
| 169 | FsShell shell = new FsShell(); |
---|
| 170 | shell.setConf(conf); |
---|
| 171 | |
---|
| 172 | try { |
---|
| 173 | Path myPath = new Path("/test/dir"); |
---|
| 174 | assertTrue(fs.mkdirs(myPath)); |
---|
| 175 | assertTrue(fs.exists(myPath)); |
---|
| 176 | Path myFile = new Path("/test/dir/file"); |
---|
| 177 | writeFile(fs, myFile); |
---|
| 178 | assertTrue(fs.exists(myFile)); |
---|
| 179 | Path myFile2 = new Path("/test/dir/file2"); |
---|
| 180 | writeFile(fs, myFile2); |
---|
| 181 | assertTrue(fs.exists(myFile2)); |
---|
| 182 | |
---|
| 183 | String[] args = new String[2]; |
---|
| 184 | args[0] = "-du"; |
---|
| 185 | args[1] = "/test/dir"; |
---|
| 186 | int val = -1; |
---|
| 187 | try { |
---|
| 188 | val = shell.run(args); |
---|
| 189 | } catch (Exception e) { |
---|
| 190 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 191 | e.getLocalizedMessage()); |
---|
| 192 | } |
---|
| 193 | assertTrue(val == 0); |
---|
| 194 | String returnString = out.toString(); |
---|
| 195 | out.reset(); |
---|
| 196 | // Check if size matchs as expected |
---|
| 197 | assertTrue(returnString.contains("22")); |
---|
| 198 | assertTrue(returnString.contains("23")); |
---|
| 199 | |
---|
| 200 | } finally { |
---|
| 201 | try {dfs.close();} catch (Exception e) {} |
---|
| 202 | System.setOut(psBackup); |
---|
| 203 | cluster.shutdown(); |
---|
| 204 | } |
---|
| 205 | |
---|
| 206 | } |
---|
| 207 | public void testPut() throws IOException { |
---|
| 208 | Configuration conf = new Configuration(); |
---|
| 209 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 210 | FileSystem fs = cluster.getFileSystem(); |
---|
| 211 | assertTrue("Not a HDFS: "+fs.getUri(), |
---|
| 212 | fs instanceof DistributedFileSystem); |
---|
| 213 | final DistributedFileSystem dfs = (DistributedFileSystem)fs; |
---|
| 214 | |
---|
| 215 | try { |
---|
| 216 | // remove left over crc files: |
---|
| 217 | new File(TEST_ROOT_DIR, ".f1.crc").delete(); |
---|
| 218 | new File(TEST_ROOT_DIR, ".f2.crc").delete(); |
---|
| 219 | final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1")); |
---|
| 220 | final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2")); |
---|
| 221 | |
---|
| 222 | final Path root = mkdir(dfs, new Path("/test/put")); |
---|
| 223 | final Path dst = new Path(root, "dst"); |
---|
| 224 | |
---|
| 225 | show("begin"); |
---|
| 226 | |
---|
| 227 | final Thread copy2ndFileThread = new Thread() { |
---|
| 228 | public void run() { |
---|
| 229 | try { |
---|
| 230 | show("copy local " + f2 + " to remote " + dst); |
---|
| 231 | dfs.copyFromLocalFile(false, false, new Path(f2.getPath()), dst); |
---|
| 232 | } catch (IOException ioe) { |
---|
| 233 | show("good " + StringUtils.stringifyException(ioe)); |
---|
| 234 | return; |
---|
| 235 | } |
---|
| 236 | //should not be here, must got IOException |
---|
| 237 | assertTrue(false); |
---|
| 238 | } |
---|
| 239 | }; |
---|
| 240 | |
---|
| 241 | //use SecurityManager to pause the copying of f1 and begin copying f2 |
---|
| 242 | SecurityManager sm = System.getSecurityManager(); |
---|
| 243 | System.out.println("SecurityManager = " + sm); |
---|
| 244 | System.setSecurityManager(new SecurityManager() { |
---|
| 245 | private boolean firstTime = true; |
---|
| 246 | |
---|
| 247 | public void checkPermission(Permission perm) { |
---|
| 248 | if (firstTime) { |
---|
| 249 | Thread t = Thread.currentThread(); |
---|
| 250 | if (!t.toString().contains("DataNode")) { |
---|
| 251 | String s = "" + Arrays.asList(t.getStackTrace()); |
---|
| 252 | if (s.contains("FileUtil.copyContent")) { |
---|
| 253 | //pause at FileUtil.copyContent |
---|
| 254 | |
---|
| 255 | firstTime = false; |
---|
| 256 | copy2ndFileThread.start(); |
---|
| 257 | try {Thread.sleep(5000);} catch (InterruptedException e) {} |
---|
| 258 | } |
---|
| 259 | } |
---|
| 260 | } |
---|
| 261 | } |
---|
| 262 | }); |
---|
| 263 | show("copy local " + f1 + " to remote " + dst); |
---|
| 264 | dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst); |
---|
| 265 | show("done"); |
---|
| 266 | |
---|
| 267 | try {copy2ndFileThread.join();} catch (InterruptedException e) { } |
---|
| 268 | System.setSecurityManager(sm); |
---|
| 269 | |
---|
| 270 | // copy multiple files to destination directory |
---|
| 271 | final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple")); |
---|
| 272 | Path[] srcs = new Path[2]; |
---|
| 273 | srcs[0] = new Path(f1.getPath()); |
---|
| 274 | srcs[1] = new Path(f2.getPath()); |
---|
| 275 | dfs.copyFromLocalFile(false, false, srcs, destmultiple); |
---|
| 276 | srcs[0] = new Path(destmultiple,"f1"); |
---|
| 277 | srcs[1] = new Path(destmultiple,"f2"); |
---|
| 278 | assertTrue(dfs.exists(srcs[0])); |
---|
| 279 | assertTrue(dfs.exists(srcs[1])); |
---|
| 280 | |
---|
| 281 | // move multiple files to destination directory |
---|
| 282 | final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple")); |
---|
| 283 | srcs[0] = new Path(f1.getPath()); |
---|
| 284 | srcs[1] = new Path(f2.getPath()); |
---|
| 285 | dfs.moveFromLocalFile(srcs, destmultiple2); |
---|
| 286 | assertFalse(f1.exists()); |
---|
| 287 | assertFalse(f2.exists()); |
---|
| 288 | srcs[0] = new Path(destmultiple2, "f1"); |
---|
| 289 | srcs[1] = new Path(destmultiple2, "f2"); |
---|
| 290 | assertTrue(dfs.exists(srcs[0])); |
---|
| 291 | assertTrue(dfs.exists(srcs[1])); |
---|
| 292 | |
---|
| 293 | f1.delete(); |
---|
| 294 | f2.delete(); |
---|
| 295 | } finally { |
---|
| 296 | try {dfs.close();} catch (Exception e) {} |
---|
| 297 | cluster.shutdown(); |
---|
| 298 | } |
---|
| 299 | } |
---|
| 300 | |
---|
| 301 | |
---|
| 302 | /** check command error outputs and exit statuses. */ |
---|
| 303 | public void testErrOutPut() throws Exception { |
---|
| 304 | Configuration conf = new Configuration(); |
---|
| 305 | MiniDFSCluster cluster = null; |
---|
| 306 | PrintStream bak = null; |
---|
| 307 | try { |
---|
| 308 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 309 | FileSystem srcFs = cluster.getFileSystem(); |
---|
| 310 | Path root = new Path("/nonexistentfile"); |
---|
| 311 | bak = System.err; |
---|
| 312 | ByteArrayOutputStream out = new ByteArrayOutputStream(); |
---|
| 313 | PrintStream tmp = new PrintStream(out); |
---|
| 314 | System.setErr(tmp); |
---|
| 315 | String[] argv = new String[2]; |
---|
| 316 | argv[0] = "-cat"; |
---|
| 317 | argv[1] = root.toUri().getPath(); |
---|
| 318 | int ret = ToolRunner.run(new FsShell(), argv); |
---|
| 319 | assertTrue(" -cat returned -1 ", 0>=ret); |
---|
| 320 | String returned = out.toString(); |
---|
| 321 | assertTrue("cat does not print exceptions ", |
---|
| 322 | (returned.lastIndexOf("Exception") == -1)); |
---|
| 323 | out.reset(); |
---|
| 324 | argv[0] = "-rm"; |
---|
| 325 | argv[1] = root.toString(); |
---|
| 326 | FsShell shell = new FsShell(); |
---|
| 327 | shell.setConf(conf); |
---|
| 328 | ret = ToolRunner.run(shell, argv); |
---|
| 329 | assertTrue(" -rm returned -1 ", 0>=ret); |
---|
| 330 | returned = out.toString(); |
---|
| 331 | out.reset(); |
---|
| 332 | assertTrue("rm prints reasonable error ", |
---|
| 333 | (returned.lastIndexOf("No such file or directory") != -1)); |
---|
| 334 | argv[0] = "-rmr"; |
---|
| 335 | argv[1] = root.toString(); |
---|
| 336 | ret = ToolRunner.run(shell, argv); |
---|
| 337 | assertTrue(" -rmr returned -1", 0>=ret); |
---|
| 338 | returned = out.toString(); |
---|
| 339 | assertTrue("rmr prints reasonable error ", |
---|
| 340 | (returned.lastIndexOf("No such file or directory") != -1)); |
---|
| 341 | out.reset(); |
---|
| 342 | argv[0] = "-du"; |
---|
| 343 | argv[1] = "/nonexistentfile"; |
---|
| 344 | ret = ToolRunner.run(shell, argv); |
---|
| 345 | returned = out.toString(); |
---|
| 346 | assertTrue(" -du prints reasonable error ", |
---|
| 347 | (returned.lastIndexOf("No such file or directory") != -1)); |
---|
| 348 | out.reset(); |
---|
| 349 | argv[0] = "-dus"; |
---|
| 350 | argv[1] = "/nonexistentfile"; |
---|
| 351 | ret = ToolRunner.run(shell, argv); |
---|
| 352 | returned = out.toString(); |
---|
| 353 | assertTrue(" -dus prints reasonable error", |
---|
| 354 | (returned.lastIndexOf("No such file or directory") != -1)); |
---|
| 355 | out.reset(); |
---|
| 356 | argv[0] = "-ls"; |
---|
| 357 | argv[1] = "/nonexistenfile"; |
---|
| 358 | ret = ToolRunner.run(shell, argv); |
---|
| 359 | returned = out.toString(); |
---|
| 360 | assertTrue(" -ls does not return Found 0 items", |
---|
| 361 | (returned.lastIndexOf("Found 0") == -1)); |
---|
| 362 | out.reset(); |
---|
| 363 | argv[0] = "-ls"; |
---|
| 364 | argv[1] = "/nonexistentfile"; |
---|
| 365 | ret = ToolRunner.run(shell, argv); |
---|
| 366 | assertTrue(" -lsr should fail ", |
---|
| 367 | (ret < 0)); |
---|
| 368 | out.reset(); |
---|
| 369 | srcFs.mkdirs(new Path("/testdir")); |
---|
| 370 | argv[0] = "-ls"; |
---|
| 371 | argv[1] = "/testdir"; |
---|
| 372 | ret = ToolRunner.run(shell, argv); |
---|
| 373 | returned = out.toString(); |
---|
| 374 | assertTrue(" -ls does not print out anything ", |
---|
| 375 | (returned.lastIndexOf("Found 0") == -1)); |
---|
| 376 | out.reset(); |
---|
| 377 | argv[0] = "-ls"; |
---|
| 378 | argv[1] = "/user/nonxistant/*"; |
---|
| 379 | ret = ToolRunner.run(shell, argv); |
---|
| 380 | assertTrue(" -ls on nonexistent glob returns -1", |
---|
| 381 | (ret < 0)); |
---|
| 382 | out.reset(); |
---|
| 383 | argv[0] = "-mkdir"; |
---|
| 384 | argv[1] = "/testdir"; |
---|
| 385 | ret = ToolRunner.run(shell, argv); |
---|
| 386 | returned = out.toString(); |
---|
| 387 | assertTrue(" -mkdir returned -1 ", (ret < 0)); |
---|
| 388 | assertTrue(" -mkdir returned File exists", |
---|
| 389 | (returned.lastIndexOf("File exists") != -1)); |
---|
| 390 | Path testFile = new Path("/testfile"); |
---|
| 391 | OutputStream outtmp = srcFs.create(testFile); |
---|
| 392 | outtmp.write(testFile.toString().getBytes()); |
---|
| 393 | outtmp.close(); |
---|
| 394 | out.reset(); |
---|
| 395 | argv[0] = "-mkdir"; |
---|
| 396 | argv[1] = "/testfile"; |
---|
| 397 | ret = ToolRunner.run(shell, argv); |
---|
| 398 | returned = out.toString(); |
---|
| 399 | assertTrue(" -mkdir returned -1", (ret < 0)); |
---|
| 400 | assertTrue(" -mkdir returned this is a file ", |
---|
| 401 | (returned.lastIndexOf("not a directory") != -1)); |
---|
| 402 | out.reset(); |
---|
| 403 | argv = new String[3]; |
---|
| 404 | argv[0] = "-mv"; |
---|
| 405 | argv[1] = "/testfile"; |
---|
| 406 | argv[2] = "file"; |
---|
| 407 | ret = ToolRunner.run(shell, argv); |
---|
| 408 | assertTrue("mv failed to rename", ret == -1); |
---|
| 409 | out.reset(); |
---|
| 410 | argv = new String[3]; |
---|
| 411 | argv[0] = "-mv"; |
---|
| 412 | argv[1] = "/testfile"; |
---|
| 413 | argv[2] = "/testfiletest"; |
---|
| 414 | ret = ToolRunner.run(shell, argv); |
---|
| 415 | returned = out.toString(); |
---|
| 416 | assertTrue("no output from rename", |
---|
| 417 | (returned.lastIndexOf("Renamed") == -1)); |
---|
| 418 | out.reset(); |
---|
| 419 | argv[0] = "-mv"; |
---|
| 420 | argv[1] = "/testfile"; |
---|
| 421 | argv[2] = "/testfiletmp"; |
---|
| 422 | ret = ToolRunner.run(shell, argv); |
---|
| 423 | returned = out.toString(); |
---|
| 424 | assertTrue(" unix like output", |
---|
| 425 | (returned.lastIndexOf("No such file or") != -1)); |
---|
| 426 | out.reset(); |
---|
| 427 | argv = new String[1]; |
---|
| 428 | argv[0] = "-du"; |
---|
| 429 | srcFs.mkdirs(srcFs.getHomeDirectory()); |
---|
| 430 | ret = ToolRunner.run(shell, argv); |
---|
| 431 | returned = out.toString(); |
---|
| 432 | assertTrue(" no error ", (ret == 0)); |
---|
| 433 | assertTrue("empty path specified", |
---|
| 434 | (returned.lastIndexOf("empty string") == -1)); |
---|
| 435 | } finally { |
---|
| 436 | if (bak != null) { |
---|
| 437 | System.setErr(bak); |
---|
| 438 | } |
---|
| 439 | if (cluster != null) { |
---|
| 440 | cluster.shutdown(); |
---|
| 441 | } |
---|
| 442 | } |
---|
| 443 | } |
---|
| 444 | |
---|
| 445 | |
---|
| 446 | public void testURIPaths() throws Exception { |
---|
| 447 | Configuration srcConf = new Configuration(); |
---|
| 448 | Configuration dstConf = new Configuration(); |
---|
| 449 | MiniDFSCluster srcCluster = null; |
---|
| 450 | MiniDFSCluster dstCluster = null; |
---|
| 451 | String bak = System.getProperty("test.build.data"); |
---|
| 452 | try{ |
---|
| 453 | srcCluster = new MiniDFSCluster(srcConf, 2, true, null); |
---|
| 454 | File nameDir = new File(new File(bak), "dfs_tmp_uri/"); |
---|
| 455 | nameDir.mkdirs(); |
---|
| 456 | System.setProperty("test.build.data", nameDir.toString()); |
---|
| 457 | dstCluster = new MiniDFSCluster(dstConf, 2, true, null); |
---|
| 458 | FileSystem srcFs = srcCluster.getFileSystem(); |
---|
| 459 | FileSystem dstFs = dstCluster.getFileSystem(); |
---|
| 460 | FsShell shell = new FsShell(); |
---|
| 461 | shell.setConf(srcConf); |
---|
| 462 | //check for ls |
---|
| 463 | String[] argv = new String[2]; |
---|
| 464 | argv[0] = "-ls"; |
---|
| 465 | argv[1] = dstFs.getUri().toString() + "/"; |
---|
| 466 | int ret = ToolRunner.run(shell, argv); |
---|
| 467 | assertTrue("ls works on remote uri ", (ret==0)); |
---|
| 468 | //check for rm -r |
---|
| 469 | dstFs.mkdirs(new Path("/hadoopdir")); |
---|
| 470 | argv = new String[2]; |
---|
| 471 | argv[0] = "-rmr"; |
---|
| 472 | argv[1] = dstFs.getUri().toString() + "/hadoopdir"; |
---|
| 473 | ret = ToolRunner.run(shell, argv); |
---|
| 474 | assertTrue("-rmr works on remote uri " + argv[1], (ret==0)); |
---|
| 475 | //check du |
---|
| 476 | argv[0] = "-du"; |
---|
| 477 | argv[1] = dstFs.getUri().toString() + "/"; |
---|
| 478 | ret = ToolRunner.run(shell, argv); |
---|
| 479 | assertTrue("du works on remote uri ", (ret ==0)); |
---|
| 480 | //check put |
---|
| 481 | File furi = new File(TEST_ROOT_DIR, "furi"); |
---|
| 482 | createLocalFile(furi); |
---|
| 483 | argv = new String[3]; |
---|
| 484 | argv[0] = "-put"; |
---|
| 485 | argv[1] = furi.toString(); |
---|
| 486 | argv[2] = dstFs.getUri().toString() + "/furi"; |
---|
| 487 | ret = ToolRunner.run(shell, argv); |
---|
| 488 | assertTrue(" put is working ", (ret==0)); |
---|
| 489 | //check cp |
---|
| 490 | argv[0] = "-cp"; |
---|
| 491 | argv[1] = dstFs.getUri().toString() + "/furi"; |
---|
| 492 | argv[2] = srcFs.getUri().toString() + "/furi"; |
---|
| 493 | ret = ToolRunner.run(shell, argv); |
---|
| 494 | assertTrue(" cp is working ", (ret==0)); |
---|
| 495 | assertTrue(srcFs.exists(new Path("/furi"))); |
---|
| 496 | //check cat |
---|
| 497 | argv = new String[2]; |
---|
| 498 | argv[0] = "-cat"; |
---|
| 499 | argv[1] = dstFs.getUri().toString() + "/furi"; |
---|
| 500 | ret = ToolRunner.run(shell, argv); |
---|
| 501 | assertTrue(" cat is working ", (ret == 0)); |
---|
| 502 | //check chown |
---|
| 503 | dstFs.delete(new Path("/furi"), true); |
---|
| 504 | dstFs.delete(new Path("/hadoopdir"), true); |
---|
| 505 | String file = "/tmp/chownTest"; |
---|
| 506 | Path path = new Path(file); |
---|
| 507 | Path parent = new Path("/tmp"); |
---|
| 508 | Path root = new Path("/"); |
---|
| 509 | TestDFSShell.writeFile(dstFs, path); |
---|
| 510 | runCmd(shell, "-chgrp", "-R", "herbivores", dstFs.getUri().toString() +"/*"); |
---|
| 511 | confirmOwner(null, "herbivores", dstFs, parent, path); |
---|
| 512 | runCmd(shell, "-chown", "-R", ":reptiles", dstFs.getUri().toString() + "/"); |
---|
| 513 | confirmOwner(null, "reptiles", dstFs, root, parent, path); |
---|
| 514 | //check if default hdfs:/// works |
---|
| 515 | argv[0] = "-cat"; |
---|
| 516 | argv[1] = "hdfs:///furi"; |
---|
| 517 | ret = ToolRunner.run(shell, argv); |
---|
| 518 | assertTrue(" default works for cat", (ret == 0)); |
---|
| 519 | argv[0] = "-ls"; |
---|
| 520 | argv[1] = "hdfs:///"; |
---|
| 521 | ret = ToolRunner.run(shell, argv); |
---|
| 522 | assertTrue("default works for ls ", (ret == 0)); |
---|
| 523 | argv[0] = "-rmr"; |
---|
| 524 | argv[1] = "hdfs:///furi"; |
---|
| 525 | ret = ToolRunner.run(shell, argv); |
---|
| 526 | assertTrue("default works for rm/rmr", (ret ==0)); |
---|
| 527 | } finally { |
---|
| 528 | System.setProperty("test.build.data", bak); |
---|
| 529 | if (null != srcCluster) { |
---|
| 530 | srcCluster.shutdown(); |
---|
| 531 | } |
---|
| 532 | if (null != dstCluster) { |
---|
| 533 | dstCluster.shutdown(); |
---|
| 534 | } |
---|
| 535 | } |
---|
| 536 | } |
---|
| 537 | |
---|
| 538 | public void testText() throws Exception { |
---|
| 539 | Configuration conf = new Configuration(); |
---|
| 540 | MiniDFSCluster cluster = null; |
---|
| 541 | PrintStream bak = null; |
---|
| 542 | try { |
---|
| 543 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 544 | FileSystem fs = cluster.getFileSystem(); |
---|
| 545 | Path root = new Path("/texttest"); |
---|
| 546 | fs.mkdirs(root); |
---|
| 547 | OutputStream zout = new GZIPOutputStream( |
---|
| 548 | fs.create(new Path(root, "file.gz"))); |
---|
| 549 | Random r = new Random(); |
---|
| 550 | ByteArrayOutputStream file = new ByteArrayOutputStream(); |
---|
| 551 | for (int i = 0; i < 1024; ++i) { |
---|
| 552 | char c = Character.forDigit(r.nextInt(26) + 10, 36); |
---|
| 553 | file.write(c); |
---|
| 554 | zout.write(c); |
---|
| 555 | } |
---|
| 556 | zout.close(); |
---|
| 557 | |
---|
| 558 | bak = System.out; |
---|
| 559 | ByteArrayOutputStream out = new ByteArrayOutputStream(); |
---|
| 560 | System.setOut(new PrintStream(out)); |
---|
| 561 | |
---|
| 562 | String[] argv = new String[2]; |
---|
| 563 | argv[0] = "-text"; |
---|
| 564 | argv[1] = new Path(root, "file.gz").toUri().getPath(); |
---|
| 565 | int ret = ToolRunner.run(new FsShell(), argv); |
---|
| 566 | assertTrue("-text returned -1", 0 >= ret); |
---|
| 567 | file.reset(); |
---|
| 568 | out.reset(); |
---|
| 569 | assertTrue("Output doesn't match input", |
---|
| 570 | Arrays.equals(file.toByteArray(), out.toByteArray())); |
---|
| 571 | |
---|
| 572 | } finally { |
---|
| 573 | if (null != bak) { |
---|
| 574 | System.setOut(bak); |
---|
| 575 | } |
---|
| 576 | if (null != cluster) { |
---|
| 577 | cluster.shutdown(); |
---|
| 578 | } |
---|
| 579 | } |
---|
| 580 | } |
---|
| 581 | |
---|
| 582 | public void testCopyToLocal() throws IOException { |
---|
| 583 | Configuration conf = new Configuration(); |
---|
| 584 | /* This tests some properties of ChecksumFileSystem as well. |
---|
| 585 | * Make sure that we create ChecksumDFS */ |
---|
| 586 | conf.set("fs.hdfs.impl", |
---|
| 587 | "org.apache.hadoop.hdfs.ChecksumDistributedFileSystem"); |
---|
| 588 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 589 | FileSystem fs = cluster.getFileSystem(); |
---|
| 590 | assertTrue("Not a HDFS: "+fs.getUri(), |
---|
| 591 | fs instanceof ChecksumDistributedFileSystem); |
---|
| 592 | ChecksumDistributedFileSystem dfs = (ChecksumDistributedFileSystem)fs; |
---|
| 593 | FsShell shell = new FsShell(); |
---|
| 594 | shell.setConf(conf); |
---|
| 595 | |
---|
| 596 | try { |
---|
| 597 | String root = createTree(dfs, "copyToLocal"); |
---|
| 598 | |
---|
| 599 | // Verify copying the tree |
---|
| 600 | { |
---|
| 601 | try { |
---|
| 602 | assertEquals(0, |
---|
| 603 | runCmd(shell, "-copyToLocal", root + "*", TEST_ROOT_DIR)); |
---|
| 604 | } catch (Exception e) { |
---|
| 605 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 606 | e.getLocalizedMessage()); |
---|
| 607 | } |
---|
| 608 | |
---|
| 609 | File localroot = new File(TEST_ROOT_DIR, "copyToLocal"); |
---|
| 610 | File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2"); |
---|
| 611 | |
---|
| 612 | File f1 = new File(localroot, "f1"); |
---|
| 613 | assertTrue("Copying failed.", f1.isFile()); |
---|
| 614 | |
---|
| 615 | File f2 = new File(localroot, "f2"); |
---|
| 616 | assertTrue("Copying failed.", f2.isFile()); |
---|
| 617 | |
---|
| 618 | File sub = new File(localroot, "sub"); |
---|
| 619 | assertTrue("Copying failed.", sub.isDirectory()); |
---|
| 620 | |
---|
| 621 | File f3 = new File(sub, "f3"); |
---|
| 622 | assertTrue("Copying failed.", f3.isFile()); |
---|
| 623 | |
---|
| 624 | File f4 = new File(sub, "f4"); |
---|
| 625 | assertTrue("Copying failed.", f4.isFile()); |
---|
| 626 | |
---|
| 627 | File f5 = new File(localroot2, "f1"); |
---|
| 628 | assertTrue("Copying failed.", f5.isFile()); |
---|
| 629 | |
---|
| 630 | f1.delete(); |
---|
| 631 | f2.delete(); |
---|
| 632 | f3.delete(); |
---|
| 633 | f4.delete(); |
---|
| 634 | f5.delete(); |
---|
| 635 | sub.delete(); |
---|
| 636 | } |
---|
| 637 | // Verify copying non existing sources do not create zero byte |
---|
| 638 | // destination files |
---|
| 639 | { |
---|
| 640 | String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR}; |
---|
| 641 | try { |
---|
| 642 | assertEquals(-1, shell.run(args)); |
---|
| 643 | } catch (Exception e) { |
---|
| 644 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 645 | e.getLocalizedMessage()); |
---|
| 646 | } |
---|
| 647 | File f6 = new File(TEST_ROOT_DIR, "nosuchfile"); |
---|
| 648 | assertTrue(!f6.exists()); |
---|
| 649 | } |
---|
| 650 | } finally { |
---|
| 651 | try { |
---|
| 652 | dfs.close(); |
---|
| 653 | } catch (Exception e) { |
---|
| 654 | } |
---|
| 655 | cluster.shutdown(); |
---|
| 656 | } |
---|
| 657 | } |
---|
| 658 | |
---|
| 659 | static String createTree(FileSystem fs, String name) throws IOException { |
---|
| 660 | // create a tree |
---|
| 661 | // ROOT |
---|
| 662 | // |- f1 |
---|
| 663 | // |- f2 |
---|
| 664 | // + sub |
---|
| 665 | // |- f3 |
---|
| 666 | // |- f4 |
---|
| 667 | // ROOT2 |
---|
| 668 | // |- f1 |
---|
| 669 | String path = "/test/" + name; |
---|
| 670 | Path root = mkdir(fs, new Path(path)); |
---|
| 671 | Path sub = mkdir(fs, new Path(root, "sub")); |
---|
| 672 | Path root2 = mkdir(fs, new Path(path + "2")); |
---|
| 673 | |
---|
| 674 | writeFile(fs, new Path(root, "f1")); |
---|
| 675 | writeFile(fs, new Path(root, "f2")); |
---|
| 676 | writeFile(fs, new Path(sub, "f3")); |
---|
| 677 | writeFile(fs, new Path(sub, "f4")); |
---|
| 678 | writeFile(fs, new Path(root2, "f1")); |
---|
| 679 | mkdir(fs, new Path(root2, "sub")); |
---|
| 680 | return path; |
---|
| 681 | } |
---|
| 682 | |
---|
| 683 | public void testCount() throws Exception { |
---|
| 684 | Configuration conf = new Configuration(); |
---|
| 685 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 686 | DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); |
---|
| 687 | FsShell shell = new FsShell(); |
---|
| 688 | shell.setConf(conf); |
---|
| 689 | |
---|
| 690 | try { |
---|
| 691 | String root = createTree(dfs, "count"); |
---|
| 692 | |
---|
| 693 | // Verify the counts |
---|
| 694 | runCount(root, 2, 4, conf); |
---|
| 695 | runCount(root + "2", 2, 1, conf); |
---|
| 696 | runCount(root + "2/f1", 0, 1, conf); |
---|
| 697 | runCount(root + "2/sub", 1, 0, conf); |
---|
| 698 | |
---|
| 699 | final FileSystem localfs = FileSystem.getLocal(conf); |
---|
| 700 | Path localpath = new Path(TEST_ROOT_DIR, "testcount"); |
---|
| 701 | localpath = localpath.makeQualified(localfs); |
---|
| 702 | localfs.mkdirs(localpath); |
---|
| 703 | |
---|
| 704 | final String localstr = localpath.toString(); |
---|
| 705 | System.out.println("localstr=" + localstr); |
---|
| 706 | runCount(localstr, 1, 0, conf); |
---|
| 707 | assertEquals(0, new Count(new String[]{root, localstr}, 0, conf).runAll()); |
---|
| 708 | } finally { |
---|
| 709 | try { |
---|
| 710 | dfs.close(); |
---|
| 711 | } catch (Exception e) { |
---|
| 712 | } |
---|
| 713 | cluster.shutdown(); |
---|
| 714 | } |
---|
| 715 | } |
---|
| 716 | private void runCount(String path, long dirs, long files, Configuration conf |
---|
| 717 | ) throws IOException { |
---|
| 718 | ByteArrayOutputStream bytes = new ByteArrayOutputStream(); |
---|
| 719 | PrintStream out = new PrintStream(bytes); |
---|
| 720 | PrintStream oldOut = System.out; |
---|
| 721 | System.setOut(out); |
---|
| 722 | Scanner in = null; |
---|
| 723 | String results = null; |
---|
| 724 | try { |
---|
| 725 | new Count(new String[]{path}, 0, conf).runAll(); |
---|
| 726 | results = bytes.toString(); |
---|
| 727 | in = new Scanner(results); |
---|
| 728 | assertEquals(dirs, in.nextLong()); |
---|
| 729 | assertEquals(files, in.nextLong()); |
---|
| 730 | } finally { |
---|
| 731 | if (in!=null) in.close(); |
---|
| 732 | IOUtils.closeStream(out); |
---|
| 733 | System.setOut(oldOut); |
---|
| 734 | System.out.println("results:\n" + results); |
---|
| 735 | } |
---|
| 736 | } |
---|
| 737 | |
---|
| 738 | //throws IOException instead of Exception as shell.run() does. |
---|
| 739 | private int runCmd(FsShell shell, String... args) throws IOException { |
---|
| 740 | try { |
---|
| 741 | return shell.run(args); |
---|
| 742 | } catch (IOException e) { |
---|
| 743 | throw e; |
---|
| 744 | } catch (RuntimeException e) { |
---|
| 745 | throw e; |
---|
| 746 | } catch (Exception e) { |
---|
| 747 | throw new IOException(StringUtils.stringifyException(e)); |
---|
| 748 | } |
---|
| 749 | } |
---|
| 750 | |
---|
| 751 | /** |
---|
| 752 | * Test chmod. |
---|
| 753 | */ |
---|
| 754 | void testChmod(Configuration conf, FileSystem fs, String chmodDir) |
---|
| 755 | throws IOException { |
---|
| 756 | FsShell shell = new FsShell(); |
---|
| 757 | shell.setConf(conf); |
---|
| 758 | |
---|
| 759 | try { |
---|
| 760 | //first make dir |
---|
| 761 | Path dir = new Path(chmodDir); |
---|
| 762 | fs.delete(dir, true); |
---|
| 763 | fs.mkdirs(dir); |
---|
| 764 | |
---|
| 765 | runCmd(shell, "-chmod", "u+rwx,g=rw,o-rwx", chmodDir); |
---|
| 766 | assertEquals("rwxrw----", |
---|
| 767 | fs.getFileStatus(dir).getPermission().toString()); |
---|
| 768 | |
---|
| 769 | //create an empty file |
---|
| 770 | Path file = new Path(chmodDir, "file"); |
---|
| 771 | TestDFSShell.writeFile(fs, file); |
---|
| 772 | |
---|
| 773 | //test octal mode |
---|
| 774 | runCmd(shell, "-chmod", "644", file.toString()); |
---|
| 775 | assertEquals("rw-r--r--", |
---|
| 776 | fs.getFileStatus(file).getPermission().toString()); |
---|
| 777 | |
---|
| 778 | //test recursive |
---|
| 779 | runCmd(shell, "-chmod", "-R", "a+rwX", chmodDir); |
---|
| 780 | assertEquals("rwxrwxrwx", |
---|
| 781 | fs.getFileStatus(dir).getPermission().toString()); |
---|
| 782 | assertEquals("rw-rw-rw-", |
---|
| 783 | fs.getFileStatus(file).getPermission().toString()); |
---|
| 784 | |
---|
| 785 | fs.delete(dir, true); |
---|
| 786 | } finally { |
---|
| 787 | try { |
---|
| 788 | fs.close(); |
---|
| 789 | shell.close(); |
---|
| 790 | } catch (IOException ignored) {} |
---|
| 791 | } |
---|
| 792 | } |
---|
| 793 | |
---|
| 794 | private void confirmOwner(String owner, String group, |
---|
| 795 | FileSystem fs, Path... paths) throws IOException { |
---|
| 796 | for(Path path : paths) { |
---|
| 797 | if (owner != null) { |
---|
| 798 | assertEquals(owner, fs.getFileStatus(path).getOwner()); |
---|
| 799 | } |
---|
| 800 | if (group != null) { |
---|
| 801 | assertEquals(group, fs.getFileStatus(path).getGroup()); |
---|
| 802 | } |
---|
| 803 | } |
---|
| 804 | } |
---|
| 805 | |
---|
| 806 | public void testFilePermissions() throws IOException { |
---|
| 807 | Configuration conf = new Configuration(); |
---|
| 808 | |
---|
| 809 | //test chmod on local fs |
---|
| 810 | FileSystem fs = FileSystem.getLocal(conf); |
---|
| 811 | testChmod(conf, fs, |
---|
| 812 | (new File(TEST_ROOT_DIR, "chmodTest")).getAbsolutePath()); |
---|
| 813 | |
---|
| 814 | conf.set("dfs.permissions", "true"); |
---|
| 815 | |
---|
| 816 | //test chmod on DFS |
---|
| 817 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 818 | fs = cluster.getFileSystem(); |
---|
| 819 | testChmod(conf, fs, "/tmp/chmodTest"); |
---|
| 820 | |
---|
| 821 | // test chown and chgrp on DFS: |
---|
| 822 | |
---|
| 823 | FsShell shell = new FsShell(); |
---|
| 824 | shell.setConf(conf); |
---|
| 825 | fs = cluster.getFileSystem(); |
---|
| 826 | |
---|
| 827 | /* For dfs, I am the super user and I can change ower of any file to |
---|
| 828 | * anything. "-R" option is already tested by chmod test above. |
---|
| 829 | */ |
---|
| 830 | |
---|
| 831 | String file = "/tmp/chownTest"; |
---|
| 832 | Path path = new Path(file); |
---|
| 833 | Path parent = new Path("/tmp"); |
---|
| 834 | Path root = new Path("/"); |
---|
| 835 | TestDFSShell.writeFile(fs, path); |
---|
| 836 | |
---|
| 837 | runCmd(shell, "-chgrp", "-R", "herbivores", "/*", "unknownFile*"); |
---|
| 838 | confirmOwner(null, "herbivores", fs, parent, path); |
---|
| 839 | |
---|
| 840 | runCmd(shell, "-chgrp", "mammals", file); |
---|
| 841 | confirmOwner(null, "mammals", fs, path); |
---|
| 842 | |
---|
| 843 | runCmd(shell, "-chown", "-R", ":reptiles", "/"); |
---|
| 844 | confirmOwner(null, "reptiles", fs, root, parent, path); |
---|
| 845 | |
---|
| 846 | runCmd(shell, "-chown", "python:", "/nonExistentFile", file); |
---|
| 847 | confirmOwner("python", "reptiles", fs, path); |
---|
| 848 | |
---|
| 849 | runCmd(shell, "-chown", "-R", "hadoop:toys", "unknownFile", "/"); |
---|
| 850 | confirmOwner("hadoop", "toys", fs, root, parent, path); |
---|
| 851 | |
---|
| 852 | // Test different characters in names |
---|
| 853 | |
---|
| 854 | runCmd(shell, "-chown", "hdfs.user", file); |
---|
| 855 | confirmOwner("hdfs.user", null, fs, path); |
---|
| 856 | |
---|
| 857 | runCmd(shell, "-chown", "_Hdfs.User-10:_hadoop.users--", file); |
---|
| 858 | confirmOwner("_Hdfs.User-10", "_hadoop.users--", fs, path); |
---|
| 859 | |
---|
| 860 | runCmd(shell, "-chown", "hdfs/hadoop-core@apache.org:asf-projects", file); |
---|
| 861 | confirmOwner("hdfs/hadoop-core@apache.org", "asf-projects", fs, path); |
---|
| 862 | |
---|
| 863 | runCmd(shell, "-chgrp", "hadoop-core@apache.org/100", file); |
---|
| 864 | confirmOwner(null, "hadoop-core@apache.org/100", fs, path); |
---|
| 865 | |
---|
| 866 | cluster.shutdown(); |
---|
| 867 | } |
---|
| 868 | /** |
---|
| 869 | * Tests various options of DFSShell. |
---|
| 870 | */ |
---|
| 871 | public void testDFSShell() throws IOException { |
---|
| 872 | Configuration conf = new Configuration(); |
---|
| 873 | /* This tests some properties of ChecksumFileSystem as well. |
---|
| 874 | * Make sure that we create ChecksumDFS */ |
---|
| 875 | conf.set("fs.hdfs.impl", |
---|
| 876 | "org.apache.hadoop.hdfs.ChecksumDistributedFileSystem"); |
---|
| 877 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 878 | FileSystem fs = cluster.getFileSystem(); |
---|
| 879 | assertTrue("Not a HDFS: "+fs.getUri(), |
---|
| 880 | fs instanceof ChecksumDistributedFileSystem); |
---|
| 881 | ChecksumDistributedFileSystem fileSys = (ChecksumDistributedFileSystem)fs; |
---|
| 882 | FsShell shell = new FsShell(); |
---|
| 883 | shell.setConf(conf); |
---|
| 884 | |
---|
| 885 | try { |
---|
| 886 | // First create a new directory with mkdirs |
---|
| 887 | Path myPath = new Path("/test/mkdirs"); |
---|
| 888 | assertTrue(fileSys.mkdirs(myPath)); |
---|
| 889 | assertTrue(fileSys.exists(myPath)); |
---|
| 890 | assertTrue(fileSys.mkdirs(myPath)); |
---|
| 891 | |
---|
| 892 | // Second, create a file in that directory. |
---|
| 893 | Path myFile = new Path("/test/mkdirs/myFile"); |
---|
| 894 | writeFile(fileSys, myFile); |
---|
| 895 | assertTrue(fileSys.exists(myFile)); |
---|
| 896 | Path myFile2 = new Path("/test/mkdirs/myFile2"); |
---|
| 897 | writeFile(fileSys, myFile2); |
---|
| 898 | assertTrue(fileSys.exists(myFile2)); |
---|
| 899 | |
---|
| 900 | // Verify that rm with a pattern |
---|
| 901 | { |
---|
| 902 | String[] args = new String[2]; |
---|
| 903 | args[0] = "-rm"; |
---|
| 904 | args[1] = "/test/mkdirs/myFile*"; |
---|
| 905 | int val = -1; |
---|
| 906 | try { |
---|
| 907 | val = shell.run(args); |
---|
| 908 | } catch (Exception e) { |
---|
| 909 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 910 | e.getLocalizedMessage()); |
---|
| 911 | } |
---|
| 912 | assertTrue(val == 0); |
---|
| 913 | assertFalse(fileSys.exists(myFile)); |
---|
| 914 | assertFalse(fileSys.exists(myFile2)); |
---|
| 915 | |
---|
| 916 | //re-create the files for other tests |
---|
| 917 | writeFile(fileSys, myFile); |
---|
| 918 | assertTrue(fileSys.exists(myFile)); |
---|
| 919 | writeFile(fileSys, myFile2); |
---|
| 920 | assertTrue(fileSys.exists(myFile2)); |
---|
| 921 | } |
---|
| 922 | |
---|
| 923 | // Verify that we can read the file |
---|
| 924 | { |
---|
| 925 | String[] args = new String[3]; |
---|
| 926 | args[0] = "-cat"; |
---|
| 927 | args[1] = "/test/mkdirs/myFile"; |
---|
| 928 | args[2] = "/test/mkdirs/myFile2"; |
---|
| 929 | int val = -1; |
---|
| 930 | try { |
---|
| 931 | val = shell.run(args); |
---|
| 932 | } catch (Exception e) { |
---|
| 933 | System.err.println("Exception raised from DFSShell.run: " + |
---|
| 934 | StringUtils.stringifyException(e)); |
---|
| 935 | } |
---|
| 936 | assertTrue(val == 0); |
---|
| 937 | } |
---|
| 938 | fileSys.delete(myFile2, true); |
---|
| 939 | |
---|
| 940 | // Verify that we can get with and without crc |
---|
| 941 | { |
---|
| 942 | File testFile = new File(TEST_ROOT_DIR, "mkdirs/myFile"); |
---|
| 943 | File checksumFile = new File(fileSys.getChecksumFile( |
---|
| 944 | new Path(testFile.getAbsolutePath())).toString()); |
---|
| 945 | testFile.delete(); |
---|
| 946 | checksumFile.delete(); |
---|
| 947 | |
---|
| 948 | String[] args = new String[3]; |
---|
| 949 | args[0] = "-get"; |
---|
| 950 | args[1] = "/test/mkdirs"; |
---|
| 951 | args[2] = TEST_ROOT_DIR; |
---|
| 952 | int val = -1; |
---|
| 953 | try { |
---|
| 954 | val = shell.run(args); |
---|
| 955 | } catch (Exception e) { |
---|
| 956 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 957 | e.getLocalizedMessage()); |
---|
| 958 | } |
---|
| 959 | assertTrue(val == 0); |
---|
| 960 | assertTrue("Copying failed.", testFile.exists()); |
---|
| 961 | assertTrue("Checksum file " + checksumFile+" is copied.", !checksumFile.exists()); |
---|
| 962 | testFile.delete(); |
---|
| 963 | } |
---|
| 964 | { |
---|
| 965 | File testFile = new File(TEST_ROOT_DIR, "mkdirs/myFile"); |
---|
| 966 | File checksumFile = new File(fileSys.getChecksumFile( |
---|
| 967 | new Path(testFile.getAbsolutePath())).toString()); |
---|
| 968 | testFile.delete(); |
---|
| 969 | checksumFile.delete(); |
---|
| 970 | |
---|
| 971 | String[] args = new String[4]; |
---|
| 972 | args[0] = "-get"; |
---|
| 973 | args[1] = "-crc"; |
---|
| 974 | args[2] = "/test/mkdirs"; |
---|
| 975 | args[3] = TEST_ROOT_DIR; |
---|
| 976 | int val = -1; |
---|
| 977 | try { |
---|
| 978 | val = shell.run(args); |
---|
| 979 | } catch (Exception e) { |
---|
| 980 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 981 | e.getLocalizedMessage()); |
---|
| 982 | } |
---|
| 983 | assertTrue(val == 0); |
---|
| 984 | |
---|
| 985 | assertTrue("Copying data file failed.", testFile.exists()); |
---|
| 986 | assertTrue("Checksum file " + checksumFile+" not copied.", checksumFile.exists()); |
---|
| 987 | testFile.delete(); |
---|
| 988 | checksumFile.delete(); |
---|
| 989 | } |
---|
| 990 | // Verify that we get an error while trying to read an nonexistent file |
---|
| 991 | { |
---|
| 992 | String[] args = new String[2]; |
---|
| 993 | args[0] = "-cat"; |
---|
| 994 | args[1] = "/test/mkdirs/myFile1"; |
---|
| 995 | int val = -1; |
---|
| 996 | try { |
---|
| 997 | val = shell.run(args); |
---|
| 998 | } catch (Exception e) { |
---|
| 999 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1000 | e.getLocalizedMessage()); |
---|
| 1001 | } |
---|
| 1002 | assertTrue(val != 0); |
---|
| 1003 | } |
---|
| 1004 | |
---|
| 1005 | // Verify that we get an error while trying to delete an nonexistent file |
---|
| 1006 | { |
---|
| 1007 | String[] args = new String[2]; |
---|
| 1008 | args[0] = "-rm"; |
---|
| 1009 | args[1] = "/test/mkdirs/myFile1"; |
---|
| 1010 | int val = -1; |
---|
| 1011 | try { |
---|
| 1012 | val = shell.run(args); |
---|
| 1013 | } catch (Exception e) { |
---|
| 1014 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1015 | e.getLocalizedMessage()); |
---|
| 1016 | } |
---|
| 1017 | assertTrue(val != 0); |
---|
| 1018 | } |
---|
| 1019 | |
---|
| 1020 | // Verify that we succeed in removing the file we created |
---|
| 1021 | { |
---|
| 1022 | String[] args = new String[2]; |
---|
| 1023 | args[0] = "-rm"; |
---|
| 1024 | args[1] = "/test/mkdirs/myFile"; |
---|
| 1025 | int val = -1; |
---|
| 1026 | try { |
---|
| 1027 | val = shell.run(args); |
---|
| 1028 | } catch (Exception e) { |
---|
| 1029 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1030 | e.getLocalizedMessage()); |
---|
| 1031 | } |
---|
| 1032 | assertTrue(val == 0); |
---|
| 1033 | } |
---|
| 1034 | |
---|
| 1035 | // Verify touch/test |
---|
| 1036 | { |
---|
| 1037 | String[] args = new String[2]; |
---|
| 1038 | args[0] = "-touchz"; |
---|
| 1039 | args[1] = "/test/mkdirs/noFileHere"; |
---|
| 1040 | int val = -1; |
---|
| 1041 | try { |
---|
| 1042 | val = shell.run(args); |
---|
| 1043 | } catch (Exception e) { |
---|
| 1044 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1045 | e.getLocalizedMessage()); |
---|
| 1046 | } |
---|
| 1047 | assertTrue(val == 0); |
---|
| 1048 | |
---|
| 1049 | args = new String[3]; |
---|
| 1050 | args[0] = "-test"; |
---|
| 1051 | args[1] = "-e"; |
---|
| 1052 | args[2] = "/test/mkdirs/noFileHere"; |
---|
| 1053 | val = -1; |
---|
| 1054 | try { |
---|
| 1055 | val = shell.run(args); |
---|
| 1056 | } catch (Exception e) { |
---|
| 1057 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1058 | e.getLocalizedMessage()); |
---|
| 1059 | } |
---|
| 1060 | assertTrue(val == 0); |
---|
| 1061 | } |
---|
| 1062 | |
---|
| 1063 | // Verify that cp from a directory to a subdirectory fails |
---|
| 1064 | { |
---|
| 1065 | String[] args = new String[2]; |
---|
| 1066 | args[0] = "-mkdir"; |
---|
| 1067 | args[1] = "/test/dir1"; |
---|
| 1068 | int val = -1; |
---|
| 1069 | try { |
---|
| 1070 | val = shell.run(args); |
---|
| 1071 | } catch (Exception e) { |
---|
| 1072 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1073 | e.getLocalizedMessage()); |
---|
| 1074 | } |
---|
| 1075 | assertTrue(val == 0); |
---|
| 1076 | |
---|
| 1077 | // this should fail |
---|
| 1078 | String[] args1 = new String[3]; |
---|
| 1079 | args1[0] = "-cp"; |
---|
| 1080 | args1[1] = "/test/dir1"; |
---|
| 1081 | args1[2] = "/test/dir1/dir2"; |
---|
| 1082 | val = 0; |
---|
| 1083 | try { |
---|
| 1084 | val = shell.run(args1); |
---|
| 1085 | } catch (Exception e) { |
---|
| 1086 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1087 | e.getLocalizedMessage()); |
---|
| 1088 | } |
---|
| 1089 | assertTrue(val == -1); |
---|
| 1090 | |
---|
| 1091 | // this should succeed |
---|
| 1092 | args1[0] = "-cp"; |
---|
| 1093 | args1[1] = "/test/dir1"; |
---|
| 1094 | args1[2] = "/test/dir1foo"; |
---|
| 1095 | val = -1; |
---|
| 1096 | try { |
---|
| 1097 | val = shell.run(args1); |
---|
| 1098 | } catch (Exception e) { |
---|
| 1099 | System.err.println("Exception raised from DFSShell.run " + |
---|
| 1100 | e.getLocalizedMessage()); |
---|
| 1101 | } |
---|
| 1102 | assertTrue(val == 0); |
---|
| 1103 | } |
---|
| 1104 | |
---|
| 1105 | } finally { |
---|
| 1106 | try { |
---|
| 1107 | fileSys.close(); |
---|
| 1108 | } catch (Exception e) { |
---|
| 1109 | } |
---|
| 1110 | cluster.shutdown(); |
---|
| 1111 | } |
---|
| 1112 | } |
---|
| 1113 | |
---|
| 1114 | static List<File> getBlockFiles(MiniDFSCluster cluster) throws IOException { |
---|
| 1115 | List<File> files = new ArrayList<File>(); |
---|
| 1116 | List<DataNode> datanodes = cluster.getDataNodes(); |
---|
| 1117 | Block[][] blocks = cluster.getAllBlockReports(); |
---|
| 1118 | for(int i = 0; i < blocks.length; i++) { |
---|
| 1119 | FSDataset ds = (FSDataset)datanodes.get(i).getFSDataset(); |
---|
| 1120 | for(Block b : blocks[i]) { |
---|
| 1121 | files.add(ds.getBlockFile(b)); |
---|
| 1122 | } |
---|
| 1123 | } |
---|
| 1124 | return files; |
---|
| 1125 | } |
---|
| 1126 | |
---|
| 1127 | static void corrupt(List<File> files) throws IOException { |
---|
| 1128 | for(File f : files) { |
---|
| 1129 | StringBuilder content = new StringBuilder(DFSTestUtil.readFile(f)); |
---|
| 1130 | char c = content.charAt(0); |
---|
| 1131 | content.setCharAt(0, ++c); |
---|
| 1132 | PrintWriter out = new PrintWriter(f); |
---|
| 1133 | out.print(content); |
---|
| 1134 | out.flush(); |
---|
| 1135 | out.close(); |
---|
| 1136 | } |
---|
| 1137 | } |
---|
| 1138 | |
---|
| 1139 | static interface TestGetRunner { |
---|
| 1140 | String run(int exitcode, String... options) throws IOException; |
---|
| 1141 | } |
---|
| 1142 | |
---|
| 1143 | public void testRemoteException() throws Exception { |
---|
| 1144 | UnixUserGroupInformation tmpUGI = new UnixUserGroupInformation("tmpname", |
---|
| 1145 | new String[] { |
---|
| 1146 | "mygroup"}); |
---|
| 1147 | MiniDFSCluster dfs = null; |
---|
| 1148 | PrintStream bak = null; |
---|
| 1149 | try { |
---|
| 1150 | Configuration conf = new Configuration(); |
---|
| 1151 | dfs = new MiniDFSCluster(conf, 2, true, null); |
---|
| 1152 | FileSystem fs = dfs.getFileSystem(); |
---|
| 1153 | Path p = new Path("/foo"); |
---|
| 1154 | fs.mkdirs(p); |
---|
| 1155 | fs.setPermission(p, new FsPermission((short)0700)); |
---|
| 1156 | UnixUserGroupInformation.saveToConf(conf, |
---|
| 1157 | UnixUserGroupInformation.UGI_PROPERTY_NAME, tmpUGI); |
---|
| 1158 | FsShell fshell = new FsShell(conf); |
---|
| 1159 | bak = System.err; |
---|
| 1160 | ByteArrayOutputStream out = new ByteArrayOutputStream(); |
---|
| 1161 | PrintStream tmp = new PrintStream(out); |
---|
| 1162 | System.setErr(tmp); |
---|
| 1163 | String[] args = new String[2]; |
---|
| 1164 | args[0] = "-ls"; |
---|
| 1165 | args[1] = "/foo"; |
---|
| 1166 | int ret = ToolRunner.run(fshell, args); |
---|
| 1167 | assertTrue("returned should be -1", (ret == -1)); |
---|
| 1168 | String str = out.toString(); |
---|
| 1169 | assertTrue("permission denied printed", str.indexOf("Permission denied") != -1); |
---|
| 1170 | out.reset(); |
---|
| 1171 | } finally { |
---|
| 1172 | if (bak != null) { |
---|
| 1173 | System.setErr(bak); |
---|
| 1174 | } |
---|
| 1175 | if (dfs != null) { |
---|
| 1176 | dfs.shutdown(); |
---|
| 1177 | } |
---|
| 1178 | } |
---|
| 1179 | } |
---|
| 1180 | |
---|
| 1181 | public void testGet() throws IOException { |
---|
| 1182 | DFSTestUtil.setLogLevel2All(FSInputChecker.LOG); |
---|
| 1183 | final Configuration conf = new Configuration(); |
---|
| 1184 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 1185 | DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); |
---|
| 1186 | |
---|
| 1187 | try { |
---|
| 1188 | final String fname = "testGet.txt"; |
---|
| 1189 | final File localf = createLocalFile(new File(TEST_ROOT_DIR, fname)); |
---|
| 1190 | final String localfcontent = DFSTestUtil.readFile(localf); |
---|
| 1191 | final Path root = mkdir(dfs, new Path("/test/get")); |
---|
| 1192 | final Path remotef = new Path(root, fname); |
---|
| 1193 | dfs.copyFromLocalFile(false, false, new Path(localf.getPath()), remotef); |
---|
| 1194 | |
---|
| 1195 | final FsShell shell = new FsShell(); |
---|
| 1196 | shell.setConf(conf); |
---|
| 1197 | TestGetRunner runner = new TestGetRunner() { |
---|
| 1198 | private int count = 0; |
---|
| 1199 | |
---|
| 1200 | public String run(int exitcode, String... options) throws IOException { |
---|
| 1201 | String dst = TEST_ROOT_DIR + "/" + fname+ ++count; |
---|
| 1202 | String[] args = new String[options.length + 3]; |
---|
| 1203 | args[0] = "-get"; |
---|
| 1204 | args[args.length - 2] = remotef.toString(); |
---|
| 1205 | args[args.length - 1] = dst; |
---|
| 1206 | for(int i = 0; i < options.length; i++) { |
---|
| 1207 | args[i + 1] = options[i]; |
---|
| 1208 | } |
---|
| 1209 | show("args=" + Arrays.asList(args)); |
---|
| 1210 | |
---|
| 1211 | try { |
---|
| 1212 | assertEquals(exitcode, shell.run(args)); |
---|
| 1213 | } catch (Exception e) { |
---|
| 1214 | assertTrue(StringUtils.stringifyException(e), false); |
---|
| 1215 | } |
---|
| 1216 | return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null; |
---|
| 1217 | } |
---|
| 1218 | }; |
---|
| 1219 | |
---|
| 1220 | assertEquals(localfcontent, runner.run(0)); |
---|
| 1221 | assertEquals(localfcontent, runner.run(0, "-ignoreCrc")); |
---|
| 1222 | |
---|
| 1223 | //find and modify the block files |
---|
| 1224 | List<File> files = getBlockFiles(cluster); |
---|
| 1225 | show("files=" + files); |
---|
| 1226 | corrupt(files); |
---|
| 1227 | |
---|
| 1228 | assertEquals(null, runner.run(-1)); |
---|
| 1229 | String corruptedcontent = runner.run(0, "-ignoreCrc"); |
---|
| 1230 | assertEquals(localfcontent.substring(1), corruptedcontent.substring(1)); |
---|
| 1231 | assertEquals(localfcontent.charAt(0)+1, corruptedcontent.charAt(0)); |
---|
| 1232 | |
---|
| 1233 | localf.delete(); |
---|
| 1234 | } finally { |
---|
| 1235 | try {dfs.close();} catch (Exception e) {} |
---|
| 1236 | cluster.shutdown(); |
---|
| 1237 | } |
---|
| 1238 | } |
---|
| 1239 | |
---|
| 1240 | public void testLsr() throws Exception { |
---|
| 1241 | Configuration conf = new Configuration(); |
---|
| 1242 | MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
| 1243 | DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); |
---|
| 1244 | |
---|
| 1245 | try { |
---|
| 1246 | final String root = createTree(dfs, "lsr"); |
---|
| 1247 | dfs.mkdirs(new Path(root, "zzz")); |
---|
| 1248 | |
---|
| 1249 | runLsr(new FsShell(conf), root, 0); |
---|
| 1250 | |
---|
| 1251 | final Path sub = new Path(root, "sub"); |
---|
| 1252 | dfs.setPermission(sub, new FsPermission((short)0)); |
---|
| 1253 | |
---|
| 1254 | final UserGroupInformation ugi = UserGroupInformation.getCurrentUGI(); |
---|
| 1255 | final String tmpusername = ugi.getUserName() + "1"; |
---|
| 1256 | UnixUserGroupInformation tmpUGI = new UnixUserGroupInformation( |
---|
| 1257 | tmpusername, new String[] {tmpusername}); |
---|
| 1258 | UnixUserGroupInformation.saveToConf(conf, |
---|
| 1259 | UnixUserGroupInformation.UGI_PROPERTY_NAME, tmpUGI); |
---|
| 1260 | String results = runLsr(new FsShell(conf), root, -1); |
---|
| 1261 | assertTrue(results.contains("zzz")); |
---|
| 1262 | } finally { |
---|
| 1263 | cluster.shutdown(); |
---|
| 1264 | } |
---|
| 1265 | } |
---|
| 1266 | private static String runLsr(final FsShell shell, String root, int returnvalue |
---|
| 1267 | ) throws Exception { |
---|
| 1268 | System.out.println("root=" + root + ", returnvalue=" + returnvalue); |
---|
| 1269 | final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); |
---|
| 1270 | final PrintStream out = new PrintStream(bytes); |
---|
| 1271 | final PrintStream oldOut = System.out; |
---|
| 1272 | final PrintStream oldErr = System.err; |
---|
| 1273 | System.setOut(out); |
---|
| 1274 | System.setErr(out); |
---|
| 1275 | final String results; |
---|
| 1276 | try { |
---|
| 1277 | assertEquals(returnvalue, shell.run(new String[]{"-lsr", root})); |
---|
| 1278 | results = bytes.toString(); |
---|
| 1279 | } finally { |
---|
| 1280 | IOUtils.closeStream(out); |
---|
| 1281 | System.setOut(oldOut); |
---|
| 1282 | System.setErr(oldErr); |
---|
| 1283 | } |
---|
| 1284 | System.out.println("results:\n" + results); |
---|
| 1285 | return results; |
---|
| 1286 | } |
---|
| 1287 | } |
---|