1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.fs.loadGenerator; |
---|
20 | |
---|
21 | import java.io.BufferedReader; |
---|
22 | import java.io.File; |
---|
23 | import java.io.FileReader; |
---|
24 | import java.io.IOException; |
---|
25 | |
---|
26 | import org.apache.hadoop.conf.Configuration; |
---|
27 | import org.apache.hadoop.conf.Configured; |
---|
28 | import org.apache.hadoop.fs.FSDataOutputStream; |
---|
29 | import org.apache.hadoop.fs.FileSystem; |
---|
30 | import org.apache.hadoop.fs.Path; |
---|
31 | import org.apache.hadoop.util.Tool; |
---|
32 | import org.apache.hadoop.util.ToolRunner; |
---|
33 | |
---|
34 | /** |
---|
35 | * This program reads the directory structure and file structure from |
---|
36 | * the input directory and creates the namespace in the file system |
---|
37 | * specified by the configuration in the specified root. |
---|
38 | * All the files are filled with 'a'. |
---|
39 | * |
---|
40 | * The synopsis of the command is |
---|
41 | * java DataGenerator |
---|
42 | * -inDir <inDir>: input directory name where directory/file structures |
---|
43 | * are stored. Its default value is the current directory. |
---|
44 | * -root <root>: the name of the root directory which the new namespace |
---|
45 | * is going to be placed under. |
---|
46 | * Its default value is "/testLoadSpace". |
---|
47 | */ |
---|
48 | public class DataGenerator extends Configured implements Tool { |
---|
49 | private File inDir = StructureGenerator.DEFAULT_STRUCTURE_DIRECTORY; |
---|
50 | private Path root = DEFAULT_ROOT; |
---|
51 | private FileSystem fs; |
---|
52 | final static private long BLOCK_SIZE = 10; |
---|
53 | final static private String USAGE = "java DataGenerator " + |
---|
54 | "-inDir <inDir> " + |
---|
55 | "-root <root>"; |
---|
56 | |
---|
57 | /** default name of the root where the test namespace will be placed under */ |
---|
58 | final static Path DEFAULT_ROOT = new Path("/testLoadSpace"); |
---|
59 | |
---|
60 | /** Main function. |
---|
61 | * It first parses the command line arguments. |
---|
62 | * It then reads the directory structure from the input directory |
---|
63 | * structure file and creates directory structure in the file system |
---|
64 | * namespace. Afterwards it reads the file attributes and creates files |
---|
65 | * in the file. All file content is filled with 'a'. |
---|
66 | */ |
---|
67 | public int run(String[] args) throws Exception { |
---|
68 | int exitCode = 0; |
---|
69 | exitCode = init(args); |
---|
70 | if (exitCode != 0) { |
---|
71 | return exitCode; |
---|
72 | } |
---|
73 | genDirStructure(); |
---|
74 | genFiles(); |
---|
75 | return exitCode; |
---|
76 | } |
---|
77 | |
---|
78 | /** Parse the command line arguments and initialize the data */ |
---|
79 | private int init(String[] args) { |
---|
80 | try { // initialize file system handle |
---|
81 | fs = FileSystem.get(getConf()); |
---|
82 | } catch (IOException ioe) { |
---|
83 | System.err.println("Can not initialize the file system: " + |
---|
84 | ioe.getLocalizedMessage()); |
---|
85 | return -1; |
---|
86 | } |
---|
87 | |
---|
88 | for (int i = 0; i < args.length; i++) { // parse command line |
---|
89 | if (args[i].equals("-root")) { |
---|
90 | root = new Path(args[++i]); |
---|
91 | } else if (args[i].equals("-inDir")) { |
---|
92 | inDir = new File(args[++i]); |
---|
93 | } else { |
---|
94 | System.err.println(USAGE); |
---|
95 | ToolRunner.printGenericCommandUsage(System.err); |
---|
96 | System.exit(-1); |
---|
97 | } |
---|
98 | } |
---|
99 | return 0; |
---|
100 | } |
---|
101 | |
---|
102 | /** Read directory structure file under the input directory. |
---|
103 | * Create each directory under the specified root. |
---|
104 | * The directory names are relative to the specified root. |
---|
105 | */ |
---|
106 | private void genDirStructure() throws IOException { |
---|
107 | BufferedReader in = new BufferedReader( |
---|
108 | new FileReader(new File(inDir, |
---|
109 | StructureGenerator.DIR_STRUCTURE_FILE_NAME))); |
---|
110 | String line; |
---|
111 | while ((line=in.readLine()) != null) { |
---|
112 | fs.mkdirs(new Path(root+line)); |
---|
113 | } |
---|
114 | } |
---|
115 | |
---|
116 | /** Read file structure file under the input directory. |
---|
117 | * Create each file under the specified root. |
---|
118 | * The file names are relative to the root. |
---|
119 | */ |
---|
120 | private void genFiles() throws IOException { |
---|
121 | BufferedReader in = new BufferedReader( |
---|
122 | new FileReader(new File(inDir, |
---|
123 | StructureGenerator.FILE_STRUCTURE_FILE_NAME))); |
---|
124 | String line; |
---|
125 | while ((line=in.readLine()) != null) { |
---|
126 | String[] tokens = line.split(" "); |
---|
127 | if (tokens.length != 2) { |
---|
128 | throw new IOException("Expect at most 2 tokens per line: " + line); |
---|
129 | } |
---|
130 | String fileName = root+tokens[0]; |
---|
131 | long fileSize = (long)(BLOCK_SIZE*Double.parseDouble(tokens[1])); |
---|
132 | genFile(new Path(fileName), fileSize); |
---|
133 | } |
---|
134 | } |
---|
135 | |
---|
136 | /** Create a file with the name <code>file</code> and |
---|
137 | * a length of <code>fileSize</code>. The file is filled with character 'a'. |
---|
138 | */ |
---|
139 | private void genFile(Path file, long fileSize) throws IOException { |
---|
140 | FSDataOutputStream out = fs.create(file, true, |
---|
141 | getConf().getInt("io.file.buffer.size", 4096), |
---|
142 | (short)getConf().getInt("dfs.replication", 3), |
---|
143 | fs.getDefaultBlockSize()); |
---|
144 | for(long i=0; i<fileSize; i++) { |
---|
145 | out.writeByte('a'); |
---|
146 | } |
---|
147 | out.close(); |
---|
148 | } |
---|
149 | |
---|
150 | /** Main program. |
---|
151 | * |
---|
152 | * @param args Command line arguments |
---|
153 | * @throws Exception |
---|
154 | */ |
---|
155 | public static void main(String[] args) throws Exception { |
---|
156 | int res = ToolRunner.run(new Configuration(), |
---|
157 | new DataGenerator(), args); |
---|
158 | System.exit(res); |
---|
159 | } |
---|
160 | } |
---|