[120] | 1 | /** |
---|
| 2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
| 3 | * or more contributor license agreements. See the NOTICE file |
---|
| 4 | * distributed with this work for additional information |
---|
| 5 | * regarding copyright ownership. The ASF licenses this file |
---|
| 6 | * to you under the Apache License, Version 2.0 (the |
---|
| 7 | * "License"); you may not use this file except in compliance |
---|
| 8 | * with the License. You may obtain a copy of the License at |
---|
| 9 | * |
---|
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 11 | * |
---|
| 12 | * Unless required by applicable law or agreed to in writing, software |
---|
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
| 15 | * See the License for the specific language governing permissions and |
---|
| 16 | * limitations under the License. |
---|
| 17 | */ |
---|
| 18 | package org.apache.hadoop.fs.ftp; |
---|
| 19 | |
---|
| 20 | import java.net.URI; |
---|
| 21 | import junit.framework.TestCase; |
---|
| 22 | |
---|
| 23 | import org.apache.ftpserver.DefaultFtpServerContext; |
---|
| 24 | import org.apache.ftpserver.FtpServer; |
---|
| 25 | import org.apache.ftpserver.ftplet.Authority; |
---|
| 26 | import org.apache.ftpserver.ftplet.UserManager; |
---|
| 27 | import org.apache.ftpserver.listener.mina.MinaListener; |
---|
| 28 | import org.apache.ftpserver.usermanager.BaseUser; |
---|
| 29 | import org.apache.ftpserver.usermanager.WritePermission; |
---|
| 30 | import org.apache.hadoop.conf.Configuration; |
---|
| 31 | import org.apache.hadoop.hdfs.DFSTestUtil; |
---|
| 32 | import org.apache.hadoop.fs.FileSystem; |
---|
| 33 | import org.apache.hadoop.fs.FileUtil; |
---|
| 34 | import org.apache.hadoop.fs.Path; |
---|
| 35 | import org.apache.hadoop.mapred.JobConf; |
---|
| 36 | |
---|
| 37 | /** |
---|
| 38 | * Generates a bunch of random files and directories using class 'DFSTestUtil', |
---|
| 39 | * stores them on the FTP file system, copies them and check if all the files |
---|
| 40 | * were retrieved successfully without any data corruption |
---|
| 41 | */ |
---|
| 42 | public class TestFTPFileSystem extends TestCase { |
---|
| 43 | |
---|
| 44 | private Configuration defaultConf = new JobConf(); |
---|
| 45 | private FtpServer server = null; |
---|
| 46 | private FileSystem localFs = null; |
---|
| 47 | private FileSystem ftpFs = null; |
---|
| 48 | |
---|
| 49 | private Path workDir = new Path(new Path(System.getProperty( |
---|
| 50 | "test.build.data", "."), "data"), "TestFTPFileSystem"); |
---|
| 51 | |
---|
| 52 | Path ftpServerRoot = new Path(workDir, "FTPServer"); |
---|
| 53 | Path ftpServerConfig = null; |
---|
| 54 | |
---|
| 55 | private void startServer() { |
---|
| 56 | try { |
---|
| 57 | DefaultFtpServerContext context = new DefaultFtpServerContext(false); |
---|
| 58 | MinaListener listener = new MinaListener(); |
---|
| 59 | // Set port to 0 for OS to give a free port |
---|
| 60 | listener.setPort(0); |
---|
| 61 | context.setListener("default", listener); |
---|
| 62 | |
---|
| 63 | // Create a test user. |
---|
| 64 | UserManager userManager = context.getUserManager(); |
---|
| 65 | BaseUser adminUser = new BaseUser(); |
---|
| 66 | adminUser.setName("admin"); |
---|
| 67 | adminUser.setPassword("admin"); |
---|
| 68 | adminUser.setEnabled(true); |
---|
| 69 | adminUser.setAuthorities(new Authority[] { new WritePermission() }); |
---|
| 70 | |
---|
| 71 | Path adminUserHome = new Path(ftpServerRoot, "user/admin"); |
---|
| 72 | adminUser.setHomeDirectory(adminUserHome.toUri().getPath()); |
---|
| 73 | adminUser.setMaxIdleTime(0); |
---|
| 74 | userManager.save(adminUser); |
---|
| 75 | |
---|
| 76 | // Initialize the server and start. |
---|
| 77 | server = new FtpServer(context); |
---|
| 78 | server.start(); |
---|
| 79 | |
---|
| 80 | } catch (Exception e) { |
---|
| 81 | throw new RuntimeException("FTP server start-up failed", e); |
---|
| 82 | } |
---|
| 83 | } |
---|
| 84 | |
---|
| 85 | private void stopServer() { |
---|
| 86 | if (server != null) { |
---|
| 87 | server.stop(); |
---|
| 88 | } |
---|
| 89 | } |
---|
| 90 | |
---|
| 91 | @Override |
---|
| 92 | public void setUp() throws Exception { |
---|
| 93 | startServer(); |
---|
| 94 | defaultConf = new Configuration(); |
---|
| 95 | localFs = FileSystem.getLocal(defaultConf); |
---|
| 96 | ftpServerConfig = new Path(localFs.getWorkingDirectory(), "res"); |
---|
| 97 | MinaListener listener = (MinaListener) server.getServerContext() |
---|
| 98 | .getListener("default"); |
---|
| 99 | int serverPort = listener.getPort(); |
---|
| 100 | ftpFs = FileSystem.get(URI.create("ftp://admin:admin@localhost:" |
---|
| 101 | + serverPort), defaultConf); |
---|
| 102 | } |
---|
| 103 | |
---|
| 104 | @Override |
---|
| 105 | public void tearDown() throws Exception { |
---|
| 106 | localFs.delete(ftpServerRoot, true); |
---|
| 107 | localFs.delete(ftpServerConfig, true); |
---|
| 108 | localFs.close(); |
---|
| 109 | ftpFs.close(); |
---|
| 110 | stopServer(); |
---|
| 111 | } |
---|
| 112 | |
---|
| 113 | /** |
---|
| 114 | * Tests FTPFileSystem, create(), open(), delete(), mkdirs(), rename(), |
---|
| 115 | * listStatus(), getStatus() APIs. * |
---|
| 116 | * |
---|
| 117 | * @throws Exception |
---|
| 118 | */ |
---|
| 119 | public void testReadWrite() throws Exception { |
---|
| 120 | |
---|
| 121 | DFSTestUtil util = new DFSTestUtil("TestFTPFileSystem", 20, 3, 1024 * 1024); |
---|
| 122 | localFs.setWorkingDirectory(workDir); |
---|
| 123 | Path localData = new Path(workDir, "srcData"); |
---|
| 124 | Path remoteData = new Path("srcData"); |
---|
| 125 | |
---|
| 126 | util.createFiles(localFs, localData.toUri().getPath()); |
---|
| 127 | |
---|
| 128 | boolean dataConsistency = util.checkFiles(localFs, localData.getName()); |
---|
| 129 | assertTrue("Test data corrupted", dataConsistency); |
---|
| 130 | |
---|
| 131 | // Copy files and directories recursively to FTP file system. |
---|
| 132 | boolean filesCopied = FileUtil.copy(localFs, localData, ftpFs, remoteData, |
---|
| 133 | false, defaultConf); |
---|
| 134 | assertTrue("Copying to FTPFileSystem failed", filesCopied); |
---|
| 135 | |
---|
| 136 | // Rename the remote copy |
---|
| 137 | Path renamedData = new Path("Renamed"); |
---|
| 138 | boolean renamed = ftpFs.rename(remoteData, renamedData); |
---|
| 139 | assertTrue("Rename failed", renamed); |
---|
| 140 | |
---|
| 141 | // Copy files and directories from FTP file system and delete remote copy. |
---|
| 142 | filesCopied = FileUtil.copy(ftpFs, renamedData, localFs, workDir, true, |
---|
| 143 | defaultConf); |
---|
| 144 | assertTrue("Copying from FTPFileSystem fails", filesCopied); |
---|
| 145 | |
---|
| 146 | // Check if the data was received completely without any corruption. |
---|
| 147 | dataConsistency = util.checkFiles(localFs, renamedData.getName()); |
---|
| 148 | assertTrue("Invalid or corrupted data recieved from FTP Server!", |
---|
| 149 | dataConsistency); |
---|
| 150 | |
---|
| 151 | // Delete local copies |
---|
| 152 | boolean deleteSuccess = localFs.delete(renamedData, true) |
---|
| 153 | & localFs.delete(localData, true); |
---|
| 154 | assertTrue("Local test data deletion failed", deleteSuccess); |
---|
| 155 | } |
---|
| 156 | } |
---|