1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.fs; |
---|
20 | |
---|
21 | import java.io.ByteArrayOutputStream; |
---|
22 | import java.io.FileNotFoundException; |
---|
23 | import java.io.IOException; |
---|
24 | import java.io.PrintStream; |
---|
25 | import java.net.URI; |
---|
26 | import java.util.ArrayList; |
---|
27 | import java.util.List; |
---|
28 | import java.util.Random; |
---|
29 | import java.util.StringTokenizer; |
---|
30 | |
---|
31 | import junit.framework.TestCase; |
---|
32 | |
---|
33 | import org.apache.commons.logging.LogFactory; |
---|
34 | import org.apache.commons.logging.impl.Log4JLogger; |
---|
35 | import org.apache.hadoop.conf.Configuration; |
---|
36 | import org.apache.hadoop.fs.permission.FsPermission; |
---|
37 | import org.apache.hadoop.hdfs.MiniDFSCluster; |
---|
38 | import org.apache.hadoop.hdfs.server.datanode.DataNode; |
---|
39 | import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; |
---|
40 | import org.apache.hadoop.mapred.MiniMRCluster; |
---|
41 | import org.apache.hadoop.security.UnixUserGroupInformation; |
---|
42 | import org.apache.hadoop.security.UserGroupInformation; |
---|
43 | import org.apache.hadoop.tools.DistCp; |
---|
44 | import org.apache.hadoop.util.ToolRunner; |
---|
45 | import org.apache.log4j.Level; |
---|
46 | |
---|
47 | |
---|
48 | /** |
---|
49 | * A JUnit test for copying files recursively. |
---|
50 | */ |
---|
51 | public class TestCopyFiles extends TestCase { |
---|
52 | { |
---|
53 | ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange") |
---|
54 | ).getLogger().setLevel(Level.OFF); |
---|
55 | ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF); |
---|
56 | ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.OFF); |
---|
57 | ((Log4JLogger)DistCp.LOG).getLogger().setLevel(Level.ALL); |
---|
58 | } |
---|
59 | |
---|
60 | static final URI LOCAL_FS = URI.create("file:///"); |
---|
61 | |
---|
62 | private static final Random RAN = new Random(); |
---|
63 | private static final int NFILES = 20; |
---|
64 | private static String TEST_ROOT_DIR = |
---|
65 | new Path(System.getProperty("test.build.data","/tmp")) |
---|
66 | .toString().replace(' ', '+'); |
---|
67 | |
---|
68 | /** class MyFile contains enough information to recreate the contents of |
---|
69 | * a single file. |
---|
70 | */ |
---|
71 | private static class MyFile { |
---|
72 | private static Random gen = new Random(); |
---|
73 | private static final int MAX_LEVELS = 3; |
---|
74 | private static final int MAX_SIZE = 8*1024; |
---|
75 | private static String[] dirNames = { |
---|
76 | "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine" |
---|
77 | }; |
---|
78 | private final String name; |
---|
79 | private int size = 0; |
---|
80 | private long seed = 0L; |
---|
81 | |
---|
82 | MyFile() { |
---|
83 | this(gen.nextInt(MAX_LEVELS)); |
---|
84 | } |
---|
85 | MyFile(int nLevels) { |
---|
86 | String xname = ""; |
---|
87 | if (nLevels != 0) { |
---|
88 | int[] levels = new int[nLevels]; |
---|
89 | for (int idx = 0; idx < nLevels; idx++) { |
---|
90 | levels[idx] = gen.nextInt(10); |
---|
91 | } |
---|
92 | StringBuffer sb = new StringBuffer(); |
---|
93 | for (int idx = 0; idx < nLevels; idx++) { |
---|
94 | sb.append(dirNames[levels[idx]]); |
---|
95 | sb.append("/"); |
---|
96 | } |
---|
97 | xname = sb.toString(); |
---|
98 | } |
---|
99 | long fidx = gen.nextLong() & Long.MAX_VALUE; |
---|
100 | name = xname + Long.toString(fidx); |
---|
101 | reset(); |
---|
102 | } |
---|
103 | void reset() { |
---|
104 | final int oldsize = size; |
---|
105 | do { size = gen.nextInt(MAX_SIZE); } while (oldsize == size); |
---|
106 | final long oldseed = seed; |
---|
107 | do { seed = gen.nextLong() & Long.MAX_VALUE; } while (oldseed == seed); |
---|
108 | } |
---|
109 | String getName() { return name; } |
---|
110 | int getSize() { return size; } |
---|
111 | long getSeed() { return seed; } |
---|
112 | } |
---|
113 | |
---|
114 | private static MyFile[] createFiles(URI fsname, String topdir) |
---|
115 | throws IOException { |
---|
116 | return createFiles(FileSystem.get(fsname, new Configuration()), topdir); |
---|
117 | } |
---|
118 | |
---|
119 | /** create NFILES with random names and directory hierarchies |
---|
120 | * with random (but reproducible) data in them. |
---|
121 | */ |
---|
122 | private static MyFile[] createFiles(FileSystem fs, String topdir) |
---|
123 | throws IOException { |
---|
124 | Path root = new Path(topdir); |
---|
125 | MyFile[] files = new MyFile[NFILES]; |
---|
126 | for (int i = 0; i < NFILES; i++) { |
---|
127 | files[i] = createFile(root, fs); |
---|
128 | } |
---|
129 | return files; |
---|
130 | } |
---|
131 | |
---|
132 | static MyFile createFile(Path root, FileSystem fs, int levels) |
---|
133 | throws IOException { |
---|
134 | MyFile f = levels < 0 ? new MyFile() : new MyFile(levels); |
---|
135 | Path p = new Path(root, f.getName()); |
---|
136 | FSDataOutputStream out = fs.create(p); |
---|
137 | byte[] toWrite = new byte[f.getSize()]; |
---|
138 | new Random(f.getSeed()).nextBytes(toWrite); |
---|
139 | out.write(toWrite); |
---|
140 | out.close(); |
---|
141 | FileSystem.LOG.info("created: " + p + ", size=" + f.getSize()); |
---|
142 | return f; |
---|
143 | } |
---|
144 | |
---|
145 | static MyFile createFile(Path root, FileSystem fs) throws IOException { |
---|
146 | return createFile(root, fs, -1); |
---|
147 | } |
---|
148 | |
---|
149 | private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files |
---|
150 | ) throws IOException { |
---|
151 | return checkFiles(fs, topdir, files, false); |
---|
152 | } |
---|
153 | |
---|
154 | private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files, |
---|
155 | boolean existingOnly) throws IOException { |
---|
156 | Path root = new Path(topdir); |
---|
157 | |
---|
158 | for (int idx = 0; idx < files.length; idx++) { |
---|
159 | Path fPath = new Path(root, files[idx].getName()); |
---|
160 | try { |
---|
161 | fs.getFileStatus(fPath); |
---|
162 | FSDataInputStream in = fs.open(fPath); |
---|
163 | byte[] toRead = new byte[files[idx].getSize()]; |
---|
164 | byte[] toCompare = new byte[files[idx].getSize()]; |
---|
165 | Random rb = new Random(files[idx].getSeed()); |
---|
166 | rb.nextBytes(toCompare); |
---|
167 | assertEquals("Cannnot read file.", toRead.length, in.read(toRead)); |
---|
168 | in.close(); |
---|
169 | for (int i = 0; i < toRead.length; i++) { |
---|
170 | if (toRead[i] != toCompare[i]) { |
---|
171 | return false; |
---|
172 | } |
---|
173 | } |
---|
174 | toRead = null; |
---|
175 | toCompare = null; |
---|
176 | } |
---|
177 | catch(FileNotFoundException fnfe) { |
---|
178 | if (!existingOnly) { |
---|
179 | throw fnfe; |
---|
180 | } |
---|
181 | } |
---|
182 | } |
---|
183 | |
---|
184 | return true; |
---|
185 | } |
---|
186 | |
---|
187 | private static void updateFiles(FileSystem fs, String topdir, MyFile[] files, |
---|
188 | int nupdate) throws IOException { |
---|
189 | assert nupdate <= NFILES; |
---|
190 | |
---|
191 | Path root = new Path(topdir); |
---|
192 | |
---|
193 | for (int idx = 0; idx < nupdate; ++idx) { |
---|
194 | Path fPath = new Path(root, files[idx].getName()); |
---|
195 | // overwrite file |
---|
196 | assertTrue(fPath.toString() + " does not exist", fs.exists(fPath)); |
---|
197 | FSDataOutputStream out = fs.create(fPath); |
---|
198 | files[idx].reset(); |
---|
199 | byte[] toWrite = new byte[files[idx].getSize()]; |
---|
200 | Random rb = new Random(files[idx].getSeed()); |
---|
201 | rb.nextBytes(toWrite); |
---|
202 | out.write(toWrite); |
---|
203 | out.close(); |
---|
204 | } |
---|
205 | } |
---|
206 | |
---|
207 | private static FileStatus[] getFileStatus(FileSystem fs, |
---|
208 | String topdir, MyFile[] files) throws IOException { |
---|
209 | return getFileStatus(fs, topdir, files, false); |
---|
210 | } |
---|
211 | private static FileStatus[] getFileStatus(FileSystem fs, |
---|
212 | String topdir, MyFile[] files, boolean existingOnly) throws IOException { |
---|
213 | Path root = new Path(topdir); |
---|
214 | List<FileStatus> statuses = new ArrayList<FileStatus>(); |
---|
215 | for (int idx = 0; idx < NFILES; ++idx) { |
---|
216 | try { |
---|
217 | statuses.add(fs.getFileStatus(new Path(root, files[idx].getName()))); |
---|
218 | } catch(FileNotFoundException fnfe) { |
---|
219 | if (!existingOnly) { |
---|
220 | throw fnfe; |
---|
221 | } |
---|
222 | } |
---|
223 | } |
---|
224 | return statuses.toArray(new FileStatus[statuses.size()]); |
---|
225 | } |
---|
226 | |
---|
227 | private static boolean checkUpdate(FileSystem fs, FileStatus[] old, |
---|
228 | String topdir, MyFile[] upd, final int nupdate) throws IOException { |
---|
229 | Path root = new Path(topdir); |
---|
230 | |
---|
231 | // overwrote updated files |
---|
232 | for (int idx = 0; idx < nupdate; ++idx) { |
---|
233 | final FileStatus stat = |
---|
234 | fs.getFileStatus(new Path(root, upd[idx].getName())); |
---|
235 | if (stat.getModificationTime() <= old[idx].getModificationTime()) { |
---|
236 | return false; |
---|
237 | } |
---|
238 | } |
---|
239 | // did not overwrite files not updated |
---|
240 | for (int idx = nupdate; idx < NFILES; ++idx) { |
---|
241 | final FileStatus stat = |
---|
242 | fs.getFileStatus(new Path(root, upd[idx].getName())); |
---|
243 | if (stat.getModificationTime() != old[idx].getModificationTime()) { |
---|
244 | return false; |
---|
245 | } |
---|
246 | } |
---|
247 | return true; |
---|
248 | } |
---|
249 | |
---|
250 | /** delete directory and everything underneath it.*/ |
---|
251 | private static void deldir(FileSystem fs, String topdir) throws IOException { |
---|
252 | fs.delete(new Path(topdir), true); |
---|
253 | } |
---|
254 | |
---|
255 | /** copy files from local file system to local file system */ |
---|
256 | public void testCopyFromLocalToLocal() throws Exception { |
---|
257 | Configuration conf = new Configuration(); |
---|
258 | FileSystem localfs = FileSystem.get(LOCAL_FS, conf); |
---|
259 | MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat"); |
---|
260 | ToolRunner.run(new DistCp(new Configuration()), |
---|
261 | new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", |
---|
262 | "file:///"+TEST_ROOT_DIR+"/destdat"}); |
---|
263 | assertTrue("Source and destination directories do not match.", |
---|
264 | checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files)); |
---|
265 | deldir(localfs, TEST_ROOT_DIR+"/destdat"); |
---|
266 | deldir(localfs, TEST_ROOT_DIR+"/srcdat"); |
---|
267 | } |
---|
268 | |
---|
269 | /** copy files from dfs file system to dfs file system */ |
---|
270 | public void testCopyFromDfsToDfs() throws Exception { |
---|
271 | String namenode = null; |
---|
272 | MiniDFSCluster cluster = null; |
---|
273 | try { |
---|
274 | Configuration conf = new Configuration(); |
---|
275 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
276 | final FileSystem hdfs = cluster.getFileSystem(); |
---|
277 | namenode = FileSystem.getDefaultUri(conf).toString(); |
---|
278 | if (namenode.startsWith("hdfs://")) { |
---|
279 | MyFile[] files = createFiles(URI.create(namenode), "/srcdat"); |
---|
280 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
281 | "-log", |
---|
282 | namenode+"/logs", |
---|
283 | namenode+"/srcdat", |
---|
284 | namenode+"/destdat"}); |
---|
285 | assertTrue("Source and destination directories do not match.", |
---|
286 | checkFiles(hdfs, "/destdat", files)); |
---|
287 | FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf); |
---|
288 | assertTrue("Log directory does not exist.", |
---|
289 | fs.exists(new Path(namenode+"/logs"))); |
---|
290 | deldir(hdfs, "/destdat"); |
---|
291 | deldir(hdfs, "/srcdat"); |
---|
292 | deldir(hdfs, "/logs"); |
---|
293 | } |
---|
294 | } finally { |
---|
295 | if (cluster != null) { cluster.shutdown(); } |
---|
296 | } |
---|
297 | } |
---|
298 | |
---|
299 | /** copy files from local file system to dfs file system */ |
---|
300 | public void testCopyFromLocalToDfs() throws Exception { |
---|
301 | MiniDFSCluster cluster = null; |
---|
302 | try { |
---|
303 | Configuration conf = new Configuration(); |
---|
304 | cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
305 | final FileSystem hdfs = cluster.getFileSystem(); |
---|
306 | final String namenode = hdfs.getUri().toString(); |
---|
307 | if (namenode.startsWith("hdfs://")) { |
---|
308 | MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat"); |
---|
309 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
310 | "-log", |
---|
311 | namenode+"/logs", |
---|
312 | "file:///"+TEST_ROOT_DIR+"/srcdat", |
---|
313 | namenode+"/destdat"}); |
---|
314 | assertTrue("Source and destination directories do not match.", |
---|
315 | checkFiles(cluster.getFileSystem(), "/destdat", files)); |
---|
316 | assertTrue("Log directory does not exist.", |
---|
317 | hdfs.exists(new Path(namenode+"/logs"))); |
---|
318 | deldir(hdfs, "/destdat"); |
---|
319 | deldir(hdfs, "/logs"); |
---|
320 | deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat"); |
---|
321 | } |
---|
322 | } finally { |
---|
323 | if (cluster != null) { cluster.shutdown(); } |
---|
324 | } |
---|
325 | } |
---|
326 | |
---|
327 | /** copy files from dfs file system to local file system */ |
---|
328 | public void testCopyFromDfsToLocal() throws Exception { |
---|
329 | MiniDFSCluster cluster = null; |
---|
330 | try { |
---|
331 | Configuration conf = new Configuration(); |
---|
332 | final FileSystem localfs = FileSystem.get(LOCAL_FS, conf); |
---|
333 | cluster = new MiniDFSCluster(conf, 1, true, null); |
---|
334 | final FileSystem hdfs = cluster.getFileSystem(); |
---|
335 | final String namenode = FileSystem.getDefaultUri(conf).toString(); |
---|
336 | if (namenode.startsWith("hdfs://")) { |
---|
337 | MyFile[] files = createFiles(URI.create(namenode), "/srcdat"); |
---|
338 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
339 | "-log", |
---|
340 | "/logs", |
---|
341 | namenode+"/srcdat", |
---|
342 | "file:///"+TEST_ROOT_DIR+"/destdat"}); |
---|
343 | assertTrue("Source and destination directories do not match.", |
---|
344 | checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files)); |
---|
345 | assertTrue("Log directory does not exist.", |
---|
346 | hdfs.exists(new Path("/logs"))); |
---|
347 | deldir(localfs, TEST_ROOT_DIR+"/destdat"); |
---|
348 | deldir(hdfs, "/logs"); |
---|
349 | deldir(hdfs, "/srcdat"); |
---|
350 | } |
---|
351 | } finally { |
---|
352 | if (cluster != null) { cluster.shutdown(); } |
---|
353 | } |
---|
354 | } |
---|
355 | |
---|
356 | public void testCopyDfsToDfsUpdateOverwrite() throws Exception { |
---|
357 | MiniDFSCluster cluster = null; |
---|
358 | try { |
---|
359 | Configuration conf = new Configuration(); |
---|
360 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
361 | final FileSystem hdfs = cluster.getFileSystem(); |
---|
362 | final String namenode = hdfs.getUri().toString(); |
---|
363 | if (namenode.startsWith("hdfs://")) { |
---|
364 | MyFile[] files = createFiles(URI.create(namenode), "/srcdat"); |
---|
365 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
366 | "-p", |
---|
367 | "-log", |
---|
368 | namenode+"/logs", |
---|
369 | namenode+"/srcdat", |
---|
370 | namenode+"/destdat"}); |
---|
371 | assertTrue("Source and destination directories do not match.", |
---|
372 | checkFiles(hdfs, "/destdat", files)); |
---|
373 | FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf); |
---|
374 | assertTrue("Log directory does not exist.", |
---|
375 | fs.exists(new Path(namenode+"/logs"))); |
---|
376 | |
---|
377 | FileStatus[] dchkpoint = getFileStatus(hdfs, "/destdat", files); |
---|
378 | final int nupdate = NFILES>>2; |
---|
379 | updateFiles(cluster.getFileSystem(), "/srcdat", files, nupdate); |
---|
380 | deldir(hdfs, "/logs"); |
---|
381 | |
---|
382 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
383 | "-p", |
---|
384 | "-update", |
---|
385 | "-log", |
---|
386 | namenode+"/logs", |
---|
387 | namenode+"/srcdat", |
---|
388 | namenode+"/destdat"}); |
---|
389 | assertTrue("Source and destination directories do not match.", |
---|
390 | checkFiles(hdfs, "/destdat", files)); |
---|
391 | assertTrue("Update failed to replicate all changes in src", |
---|
392 | checkUpdate(hdfs, dchkpoint, "/destdat", files, nupdate)); |
---|
393 | |
---|
394 | deldir(hdfs, "/logs"); |
---|
395 | ToolRunner.run(new DistCp(conf), new String[] { |
---|
396 | "-p", |
---|
397 | "-overwrite", |
---|
398 | "-log", |
---|
399 | namenode+"/logs", |
---|
400 | namenode+"/srcdat", |
---|
401 | namenode+"/destdat"}); |
---|
402 | assertTrue("Source and destination directories do not match.", |
---|
403 | checkFiles(hdfs, "/destdat", files)); |
---|
404 | assertTrue("-overwrite didn't.", |
---|
405 | checkUpdate(hdfs, dchkpoint, "/destdat", files, NFILES)); |
---|
406 | |
---|
407 | deldir(hdfs, "/destdat"); |
---|
408 | deldir(hdfs, "/srcdat"); |
---|
409 | deldir(hdfs, "/logs"); |
---|
410 | } |
---|
411 | } finally { |
---|
412 | if (cluster != null) { cluster.shutdown(); } |
---|
413 | } |
---|
414 | } |
---|
415 | |
---|
416 | public void testCopyDuplication() throws Exception { |
---|
417 | final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration()); |
---|
418 | try { |
---|
419 | MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat"); |
---|
420 | ToolRunner.run(new DistCp(new Configuration()), |
---|
421 | new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", |
---|
422 | "file:///"+TEST_ROOT_DIR+"/src2/srcdat"}); |
---|
423 | assertTrue("Source and destination directories do not match.", |
---|
424 | checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files)); |
---|
425 | |
---|
426 | assertEquals(DistCp.DuplicationException.ERROR_CODE, |
---|
427 | ToolRunner.run(new DistCp(new Configuration()), |
---|
428 | new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", |
---|
429 | "file:///"+TEST_ROOT_DIR+"/src2/srcdat", |
---|
430 | "file:///"+TEST_ROOT_DIR+"/destdat",})); |
---|
431 | } |
---|
432 | finally { |
---|
433 | deldir(localfs, TEST_ROOT_DIR+"/destdat"); |
---|
434 | deldir(localfs, TEST_ROOT_DIR+"/srcdat"); |
---|
435 | deldir(localfs, TEST_ROOT_DIR+"/src2"); |
---|
436 | } |
---|
437 | } |
---|
438 | |
---|
439 | public void testCopySingleFile() throws Exception { |
---|
440 | FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration()); |
---|
441 | Path root = new Path(TEST_ROOT_DIR+"/srcdat"); |
---|
442 | try { |
---|
443 | MyFile[] files = {createFile(root, fs)}; |
---|
444 | //copy a dir with a single file |
---|
445 | ToolRunner.run(new DistCp(new Configuration()), |
---|
446 | new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat", |
---|
447 | "file:///"+TEST_ROOT_DIR+"/destdat"}); |
---|
448 | assertTrue("Source and destination directories do not match.", |
---|
449 | checkFiles(fs, TEST_ROOT_DIR+"/destdat", files)); |
---|
450 | |
---|
451 | //copy a single file |
---|
452 | String fname = files[0].getName(); |
---|
453 | Path p = new Path(root, fname); |
---|
454 | FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p)); |
---|
455 | ToolRunner.run(new DistCp(new Configuration()), |
---|
456 | new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname, |
---|
457 | "file:///"+TEST_ROOT_DIR+"/dest2/"+fname}); |
---|
458 | assertTrue("Source and destination directories do not match.", |
---|
459 | checkFiles(fs, TEST_ROOT_DIR+"/dest2", files)); |
---|
460 | //copy single file to existing dir |
---|
461 | deldir(fs, TEST_ROOT_DIR+"/dest2"); |
---|
462 | fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2")); |
---|
463 | MyFile[] files2 = {createFile(root, fs, 0)}; |
---|
464 | String sname = files2[0].getName(); |
---|
465 | ToolRunner.run(new DistCp(new Configuration()), |
---|
466 | new String[] {"-update", |
---|
467 | "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, |
---|
468 | "file:///"+TEST_ROOT_DIR+"/dest2/"}); |
---|
469 | assertTrue("Source and destination directories do not match.", |
---|
470 | checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); |
---|
471 | updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1); |
---|
472 | //copy single file to existing dir w/ dst name conflict |
---|
473 | ToolRunner.run(new DistCp(new Configuration()), |
---|
474 | new String[] {"-update", |
---|
475 | "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname, |
---|
476 | "file:///"+TEST_ROOT_DIR+"/dest2/"}); |
---|
477 | assertTrue("Source and destination directories do not match.", |
---|
478 | checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2)); |
---|
479 | } |
---|
480 | finally { |
---|
481 | deldir(fs, TEST_ROOT_DIR+"/destdat"); |
---|
482 | deldir(fs, TEST_ROOT_DIR+"/dest2"); |
---|
483 | deldir(fs, TEST_ROOT_DIR+"/srcdat"); |
---|
484 | } |
---|
485 | } |
---|
486 | |
---|
487 | public void testPreserveOption() throws Exception { |
---|
488 | Configuration conf = new Configuration(); |
---|
489 | MiniDFSCluster cluster = null; |
---|
490 | try { |
---|
491 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
492 | String nnUri = FileSystem.getDefaultUri(conf).toString(); |
---|
493 | FileSystem fs = FileSystem.get(URI.create(nnUri), conf); |
---|
494 | |
---|
495 | {//test preserving user |
---|
496 | MyFile[] files = createFiles(URI.create(nnUri), "/srcdat"); |
---|
497 | FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files); |
---|
498 | for(int i = 0; i < srcstat.length; i++) { |
---|
499 | fs.setOwner(srcstat[i].getPath(), "u" + i, null); |
---|
500 | } |
---|
501 | ToolRunner.run(new DistCp(conf), |
---|
502 | new String[]{"-pu", nnUri+"/srcdat", nnUri+"/destdat"}); |
---|
503 | assertTrue("Source and destination directories do not match.", |
---|
504 | checkFiles(fs, "/destdat", files)); |
---|
505 | |
---|
506 | FileStatus[] dststat = getFileStatus(fs, "/destdat", files); |
---|
507 | for(int i = 0; i < dststat.length; i++) { |
---|
508 | assertEquals("i=" + i, "u" + i, dststat[i].getOwner()); |
---|
509 | } |
---|
510 | deldir(fs, "/destdat"); |
---|
511 | deldir(fs, "/srcdat"); |
---|
512 | } |
---|
513 | |
---|
514 | {//test preserving group |
---|
515 | MyFile[] files = createFiles(URI.create(nnUri), "/srcdat"); |
---|
516 | FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files); |
---|
517 | for(int i = 0; i < srcstat.length; i++) { |
---|
518 | fs.setOwner(srcstat[i].getPath(), null, "g" + i); |
---|
519 | } |
---|
520 | ToolRunner.run(new DistCp(conf), |
---|
521 | new String[]{"-pg", nnUri+"/srcdat", nnUri+"/destdat"}); |
---|
522 | assertTrue("Source and destination directories do not match.", |
---|
523 | checkFiles(fs, "/destdat", files)); |
---|
524 | |
---|
525 | FileStatus[] dststat = getFileStatus(fs, "/destdat", files); |
---|
526 | for(int i = 0; i < dststat.length; i++) { |
---|
527 | assertEquals("i=" + i, "g" + i, dststat[i].getGroup()); |
---|
528 | } |
---|
529 | deldir(fs, "/destdat"); |
---|
530 | deldir(fs, "/srcdat"); |
---|
531 | } |
---|
532 | |
---|
533 | {//test preserving mode |
---|
534 | MyFile[] files = createFiles(URI.create(nnUri), "/srcdat"); |
---|
535 | FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files); |
---|
536 | FsPermission[] permissions = new FsPermission[srcstat.length]; |
---|
537 | for(int i = 0; i < srcstat.length; i++) { |
---|
538 | permissions[i] = new FsPermission((short)(i & 0666)); |
---|
539 | fs.setPermission(srcstat[i].getPath(), permissions[i]); |
---|
540 | } |
---|
541 | |
---|
542 | ToolRunner.run(new DistCp(conf), |
---|
543 | new String[]{"-pp", nnUri+"/srcdat", nnUri+"/destdat"}); |
---|
544 | assertTrue("Source and destination directories do not match.", |
---|
545 | checkFiles(fs, "/destdat", files)); |
---|
546 | |
---|
547 | FileStatus[] dststat = getFileStatus(fs, "/destdat", files); |
---|
548 | for(int i = 0; i < dststat.length; i++) { |
---|
549 | assertEquals("i=" + i, permissions[i], dststat[i].getPermission()); |
---|
550 | } |
---|
551 | deldir(fs, "/destdat"); |
---|
552 | deldir(fs, "/srcdat"); |
---|
553 | } |
---|
554 | } finally { |
---|
555 | if (cluster != null) { cluster.shutdown(); } |
---|
556 | } |
---|
557 | } |
---|
558 | |
---|
559 | public void testMapCount() throws Exception { |
---|
560 | String namenode = null; |
---|
561 | MiniDFSCluster dfs = null; |
---|
562 | MiniMRCluster mr = null; |
---|
563 | try { |
---|
564 | Configuration conf = new Configuration(); |
---|
565 | dfs = new MiniDFSCluster(conf, 3, true, null); |
---|
566 | FileSystem fs = dfs.getFileSystem(); |
---|
567 | final FsShell shell = new FsShell(conf); |
---|
568 | namenode = fs.getUri().toString(); |
---|
569 | mr = new MiniMRCluster(3, namenode, 1); |
---|
570 | MyFile[] files = createFiles(fs.getUri(), "/srcdat"); |
---|
571 | long totsize = 0; |
---|
572 | for (MyFile f : files) { |
---|
573 | totsize += f.getSize(); |
---|
574 | } |
---|
575 | Configuration job = mr.createJobConf(); |
---|
576 | job.setLong("distcp.bytes.per.map", totsize / 3); |
---|
577 | ToolRunner.run(new DistCp(job), |
---|
578 | new String[] {"-m", "100", |
---|
579 | "-log", |
---|
580 | namenode+"/logs", |
---|
581 | namenode+"/srcdat", |
---|
582 | namenode+"/destdat"}); |
---|
583 | assertTrue("Source and destination directories do not match.", |
---|
584 | checkFiles(fs, "/destdat", files)); |
---|
585 | |
---|
586 | String logdir = namenode + "/logs"; |
---|
587 | System.out.println(execCmd(shell, "-lsr", logdir)); |
---|
588 | FileStatus[] logs = fs.listStatus(new Path(logdir)); |
---|
589 | // rare case where splits are exact, logs.length can be 4 |
---|
590 | assertTrue("Unexpected map count, logs.length=" + logs.length, |
---|
591 | logs.length == 5 || logs.length == 4); |
---|
592 | |
---|
593 | deldir(fs, "/destdat"); |
---|
594 | deldir(fs, "/logs"); |
---|
595 | ToolRunner.run(new DistCp(job), |
---|
596 | new String[] {"-m", "1", |
---|
597 | "-log", |
---|
598 | namenode+"/logs", |
---|
599 | namenode+"/srcdat", |
---|
600 | namenode+"/destdat"}); |
---|
601 | |
---|
602 | System.out.println(execCmd(shell, "-lsr", logdir)); |
---|
603 | logs = fs.listStatus(new Path(namenode+"/logs")); |
---|
604 | assertTrue("Unexpected map count, logs.length=" + logs.length, |
---|
605 | logs.length == 2); |
---|
606 | } finally { |
---|
607 | if (dfs != null) { dfs.shutdown(); } |
---|
608 | if (mr != null) { mr.shutdown(); } |
---|
609 | } |
---|
610 | } |
---|
611 | |
---|
612 | public void testLimits() throws Exception { |
---|
613 | Configuration conf = new Configuration(); |
---|
614 | MiniDFSCluster cluster = null; |
---|
615 | try { |
---|
616 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
617 | final String nnUri = FileSystem.getDefaultUri(conf).toString(); |
---|
618 | final FileSystem fs = FileSystem.get(URI.create(nnUri), conf); |
---|
619 | final DistCp distcp = new DistCp(conf); |
---|
620 | final FsShell shell = new FsShell(conf); |
---|
621 | |
---|
622 | final String srcrootdir = "/src_root"; |
---|
623 | final Path srcrootpath = new Path(srcrootdir); |
---|
624 | final String dstrootdir = "/dst_root"; |
---|
625 | final Path dstrootpath = new Path(dstrootdir); |
---|
626 | |
---|
627 | {//test -filelimit |
---|
628 | MyFile[] files = createFiles(URI.create(nnUri), srcrootdir); |
---|
629 | int filelimit = files.length / 2; |
---|
630 | System.out.println("filelimit=" + filelimit); |
---|
631 | |
---|
632 | ToolRunner.run(distcp, |
---|
633 | new String[]{"-filelimit", ""+filelimit, nnUri+srcrootdir, nnUri+dstrootdir}); |
---|
634 | String results = execCmd(shell, "-lsr", dstrootdir); |
---|
635 | results = removePrefix(results, dstrootdir); |
---|
636 | System.out.println("results=" + results); |
---|
637 | |
---|
638 | FileStatus[] dststat = getFileStatus(fs, dstrootdir, files, true); |
---|
639 | assertEquals(filelimit, dststat.length); |
---|
640 | deldir(fs, dstrootdir); |
---|
641 | deldir(fs, srcrootdir); |
---|
642 | } |
---|
643 | |
---|
644 | {//test -sizelimit |
---|
645 | createFiles(URI.create(nnUri), srcrootdir); |
---|
646 | long sizelimit = fs.getContentSummary(srcrootpath).getLength()/2; |
---|
647 | System.out.println("sizelimit=" + sizelimit); |
---|
648 | |
---|
649 | ToolRunner.run(distcp, |
---|
650 | new String[]{"-sizelimit", ""+sizelimit, nnUri+srcrootdir, nnUri+dstrootdir}); |
---|
651 | |
---|
652 | ContentSummary summary = fs.getContentSummary(dstrootpath); |
---|
653 | System.out.println("summary=" + summary); |
---|
654 | assertTrue(summary.getLength() <= sizelimit); |
---|
655 | deldir(fs, dstrootdir); |
---|
656 | deldir(fs, srcrootdir); |
---|
657 | } |
---|
658 | |
---|
659 | {//test update |
---|
660 | final MyFile[] srcs = createFiles(URI.create(nnUri), srcrootdir); |
---|
661 | final long totalsize = fs.getContentSummary(srcrootpath).getLength(); |
---|
662 | System.out.println("src.length=" + srcs.length); |
---|
663 | System.out.println("totalsize =" + totalsize); |
---|
664 | fs.mkdirs(dstrootpath); |
---|
665 | final int parts = RAN.nextInt(NFILES/3 - 1) + 2; |
---|
666 | final int filelimit = srcs.length/parts; |
---|
667 | final long sizelimit = totalsize/parts; |
---|
668 | System.out.println("filelimit=" + filelimit); |
---|
669 | System.out.println("sizelimit=" + sizelimit); |
---|
670 | System.out.println("parts =" + parts); |
---|
671 | final String[] args = {"-filelimit", ""+filelimit, "-sizelimit", ""+sizelimit, |
---|
672 | "-update", nnUri+srcrootdir, nnUri+dstrootdir}; |
---|
673 | |
---|
674 | int dstfilecount = 0; |
---|
675 | long dstsize = 0; |
---|
676 | for(int i = 0; i <= parts; i++) { |
---|
677 | ToolRunner.run(distcp, args); |
---|
678 | |
---|
679 | FileStatus[] dststat = getFileStatus(fs, dstrootdir, srcs, true); |
---|
680 | System.out.println(i + ") dststat.length=" + dststat.length); |
---|
681 | assertTrue(dststat.length - dstfilecount <= filelimit); |
---|
682 | ContentSummary summary = fs.getContentSummary(dstrootpath); |
---|
683 | System.out.println(i + ") summary.getLength()=" + summary.getLength()); |
---|
684 | assertTrue(summary.getLength() - dstsize <= sizelimit); |
---|
685 | assertTrue(checkFiles(fs, dstrootdir, srcs, true)); |
---|
686 | dstfilecount = dststat.length; |
---|
687 | dstsize = summary.getLength(); |
---|
688 | } |
---|
689 | |
---|
690 | deldir(fs, dstrootdir); |
---|
691 | deldir(fs, srcrootdir); |
---|
692 | } |
---|
693 | } finally { |
---|
694 | if (cluster != null) { cluster.shutdown(); } |
---|
695 | } |
---|
696 | } |
---|
697 | |
---|
698 | static final long now = System.currentTimeMillis(); |
---|
699 | |
---|
700 | static UnixUserGroupInformation createUGI(String name, boolean issuper) { |
---|
701 | String username = name + now; |
---|
702 | String group = issuper? "supergroup": username; |
---|
703 | return UnixUserGroupInformation.createImmutable( |
---|
704 | new String[]{username, group}); |
---|
705 | } |
---|
706 | |
---|
707 | static Path createHomeDirectory(FileSystem fs, UserGroupInformation ugi |
---|
708 | ) throws IOException { |
---|
709 | final Path home = new Path("/user/" + ugi.getUserName()); |
---|
710 | fs.mkdirs(home); |
---|
711 | fs.setOwner(home, ugi.getUserName(), ugi.getGroupNames()[0]); |
---|
712 | fs.setPermission(home, new FsPermission((short)0700)); |
---|
713 | return home; |
---|
714 | } |
---|
715 | |
---|
716 | public void testHftpAccessControl() throws Exception { |
---|
717 | MiniDFSCluster cluster = null; |
---|
718 | try { |
---|
719 | final UnixUserGroupInformation DFS_UGI = createUGI("dfs", true); |
---|
720 | final UnixUserGroupInformation USER_UGI = createUGI("user", false); |
---|
721 | |
---|
722 | //start cluster by DFS_UGI |
---|
723 | final Configuration dfsConf = new Configuration(); |
---|
724 | UnixUserGroupInformation.saveToConf(dfsConf, |
---|
725 | UnixUserGroupInformation.UGI_PROPERTY_NAME, DFS_UGI); |
---|
726 | cluster = new MiniDFSCluster(dfsConf, 2, true, null); |
---|
727 | cluster.waitActive(); |
---|
728 | |
---|
729 | final String httpAdd = dfsConf.get("dfs.http.address"); |
---|
730 | final URI nnURI = FileSystem.getDefaultUri(dfsConf); |
---|
731 | final String nnUri = nnURI.toString(); |
---|
732 | final Path home = createHomeDirectory(FileSystem.get(nnURI, dfsConf), USER_UGI); |
---|
733 | |
---|
734 | //now, login as USER_UGI |
---|
735 | final Configuration userConf = new Configuration(); |
---|
736 | UnixUserGroupInformation.saveToConf(userConf, |
---|
737 | UnixUserGroupInformation.UGI_PROPERTY_NAME, USER_UGI); |
---|
738 | final FileSystem fs = FileSystem.get(nnURI, userConf); |
---|
739 | |
---|
740 | final Path srcrootpath = new Path(home, "src_root"); |
---|
741 | final String srcrootdir = srcrootpath.toString(); |
---|
742 | final Path dstrootpath = new Path(home, "dst_root"); |
---|
743 | final String dstrootdir = dstrootpath.toString(); |
---|
744 | final DistCp distcp = new DistCp(userConf); |
---|
745 | |
---|
746 | FileSystem.mkdirs(fs, srcrootpath, new FsPermission((short)0700)); |
---|
747 | final String[] args = {"hftp://"+httpAdd+srcrootdir, nnUri+dstrootdir}; |
---|
748 | |
---|
749 | { //copy with permission 000, should fail |
---|
750 | fs.setPermission(srcrootpath, new FsPermission((short)0)); |
---|
751 | assertEquals(-3, ToolRunner.run(distcp, args)); |
---|
752 | } |
---|
753 | } finally { |
---|
754 | if (cluster != null) { cluster.shutdown(); } |
---|
755 | } |
---|
756 | } |
---|
757 | |
---|
758 | /** test -delete */ |
---|
759 | public void testDelete() throws Exception { |
---|
760 | final Configuration conf = new Configuration(); |
---|
761 | MiniDFSCluster cluster = null; |
---|
762 | try { |
---|
763 | cluster = new MiniDFSCluster(conf, 2, true, null); |
---|
764 | final URI nnURI = FileSystem.getDefaultUri(conf); |
---|
765 | final String nnUri = nnURI.toString(); |
---|
766 | final FileSystem fs = FileSystem.get(URI.create(nnUri), conf); |
---|
767 | |
---|
768 | final DistCp distcp = new DistCp(conf); |
---|
769 | final FsShell shell = new FsShell(conf); |
---|
770 | |
---|
771 | final String srcrootdir = "/src_root"; |
---|
772 | final String dstrootdir = "/dst_root"; |
---|
773 | |
---|
774 | { |
---|
775 | //create source files |
---|
776 | createFiles(nnURI, srcrootdir); |
---|
777 | String srcresults = execCmd(shell, "-lsr", srcrootdir); |
---|
778 | srcresults = removePrefix(srcresults, srcrootdir); |
---|
779 | System.out.println("srcresults=" + srcresults); |
---|
780 | |
---|
781 | //create some files in dst |
---|
782 | createFiles(nnURI, dstrootdir); |
---|
783 | System.out.println("dstrootdir=" + dstrootdir); |
---|
784 | shell.run(new String[]{"-lsr", dstrootdir}); |
---|
785 | |
---|
786 | //run distcp |
---|
787 | ToolRunner.run(distcp, |
---|
788 | new String[]{"-delete", "-update", "-log", "/log", |
---|
789 | nnUri+srcrootdir, nnUri+dstrootdir}); |
---|
790 | |
---|
791 | //make sure src and dst contains the same files |
---|
792 | String dstresults = execCmd(shell, "-lsr", dstrootdir); |
---|
793 | dstresults = removePrefix(dstresults, dstrootdir); |
---|
794 | System.out.println("first dstresults=" + dstresults); |
---|
795 | assertEquals(srcresults, dstresults); |
---|
796 | |
---|
797 | //create additional file in dst |
---|
798 | create(fs, new Path(dstrootdir, "foo")); |
---|
799 | create(fs, new Path(dstrootdir, "foobar")); |
---|
800 | |
---|
801 | //run distcp again |
---|
802 | ToolRunner.run(distcp, |
---|
803 | new String[]{"-delete", "-update", "-log", "/log2", |
---|
804 | nnUri+srcrootdir, nnUri+dstrootdir}); |
---|
805 | |
---|
806 | //make sure src and dst contains the same files |
---|
807 | dstresults = execCmd(shell, "-lsr", dstrootdir); |
---|
808 | dstresults = removePrefix(dstresults, dstrootdir); |
---|
809 | System.out.println("second dstresults=" + dstresults); |
---|
810 | assertEquals(srcresults, dstresults); |
---|
811 | |
---|
812 | //cleanup |
---|
813 | deldir(fs, dstrootdir); |
---|
814 | deldir(fs, srcrootdir); |
---|
815 | } |
---|
816 | } finally { |
---|
817 | if (cluster != null) { cluster.shutdown(); } |
---|
818 | } |
---|
819 | } |
---|
820 | |
---|
821 | static void create(FileSystem fs, Path f) throws IOException { |
---|
822 | FSDataOutputStream out = fs.create(f); |
---|
823 | try { |
---|
824 | byte[] b = new byte[1024 + RAN.nextInt(1024)]; |
---|
825 | RAN.nextBytes(b); |
---|
826 | out.write(b); |
---|
827 | } finally { |
---|
828 | if (out != null) out.close(); |
---|
829 | } |
---|
830 | } |
---|
831 | |
---|
832 | static String execCmd(FsShell shell, String... args) throws Exception { |
---|
833 | ByteArrayOutputStream baout = new ByteArrayOutputStream(); |
---|
834 | PrintStream out = new PrintStream(baout, true); |
---|
835 | PrintStream old = System.out; |
---|
836 | System.setOut(out); |
---|
837 | shell.run(args); |
---|
838 | out.close(); |
---|
839 | System.setOut(old); |
---|
840 | return baout.toString(); |
---|
841 | } |
---|
842 | |
---|
843 | private static String removePrefix(String lines, String prefix) { |
---|
844 | final int prefixlen = prefix.length(); |
---|
845 | final StringTokenizer t = new StringTokenizer(lines, "\n"); |
---|
846 | final StringBuffer results = new StringBuffer(); |
---|
847 | for(; t.hasMoreTokens(); ) { |
---|
848 | String s = t.nextToken(); |
---|
849 | results.append(s.substring(s.indexOf(prefix) + prefixlen) + "\n"); |
---|
850 | } |
---|
851 | return results.toString(); |
---|
852 | } |
---|
853 | } |
---|