1 | /* |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.record; |
---|
20 | |
---|
21 | import java.io.ByteArrayInputStream; |
---|
22 | import java.io.ByteArrayOutputStream; |
---|
23 | import java.io.DataInputStream; |
---|
24 | import java.io.DataOutputStream; |
---|
25 | import java.io.IOException; |
---|
26 | import java.lang.reflect.Array; |
---|
27 | import java.lang.reflect.InvocationTargetException; |
---|
28 | import java.lang.reflect.Method; |
---|
29 | import java.util.Random; |
---|
30 | |
---|
31 | /** |
---|
32 | * Benchmark for various types of serializations |
---|
33 | */ |
---|
34 | public class RecordBench { |
---|
35 | |
---|
36 | private static class Times { |
---|
37 | long init; |
---|
38 | long serialize; |
---|
39 | long deserialize; |
---|
40 | long write; |
---|
41 | long readFields; |
---|
42 | }; |
---|
43 | |
---|
44 | private static final long SEED = 0xDEADBEEFL; |
---|
45 | private static final Random rand = new Random(); |
---|
46 | |
---|
47 | /** Do not allow to create a new instance of RecordBench */ |
---|
48 | private RecordBench() {} |
---|
49 | |
---|
50 | private static void initBuffers(Record[] buffers) { |
---|
51 | final int BUFLEN = 32; |
---|
52 | for (int idx = 0; idx < buffers.length; idx++) { |
---|
53 | buffers[idx] = new RecBuffer(); |
---|
54 | int buflen = rand.nextInt(BUFLEN); |
---|
55 | byte[] bytes = new byte[buflen]; |
---|
56 | rand.nextBytes(bytes); |
---|
57 | ((RecBuffer)buffers[idx]).setData(new Buffer(bytes)); |
---|
58 | } |
---|
59 | } |
---|
60 | |
---|
61 | private static void initStrings(Record[] strings) { |
---|
62 | final int STRLEN = 32; |
---|
63 | for (int idx = 0; idx < strings.length; idx++) { |
---|
64 | strings[idx] = new RecString(); |
---|
65 | int strlen = rand.nextInt(STRLEN); |
---|
66 | StringBuilder sb = new StringBuilder(strlen); |
---|
67 | for (int ich = 0; ich < strlen; ich++) { |
---|
68 | int cpt = 0; |
---|
69 | while (true) { |
---|
70 | cpt = rand.nextInt(0x10FFFF+1); |
---|
71 | if (Utils.isValidCodePoint(cpt)) { |
---|
72 | break; |
---|
73 | } |
---|
74 | } |
---|
75 | sb.appendCodePoint(cpt); |
---|
76 | } |
---|
77 | ((RecString)strings[idx]).setData(sb.toString()); |
---|
78 | } |
---|
79 | } |
---|
80 | |
---|
81 | private static void initInts(Record[] ints) { |
---|
82 | for (int idx = 0; idx < ints.length; idx++) { |
---|
83 | ints[idx] = new RecInt(); |
---|
84 | ((RecInt)ints[idx]).setData(rand.nextInt()); |
---|
85 | } |
---|
86 | } |
---|
87 | |
---|
88 | private static Record[] makeArray(String type, int numRecords, Times times) { |
---|
89 | Method init = null; |
---|
90 | try { |
---|
91 | init = RecordBench.class.getDeclaredMethod("init"+ |
---|
92 | toCamelCase(type) + "s", |
---|
93 | new Class[] {Record[].class}); |
---|
94 | } catch (NoSuchMethodException ex) { |
---|
95 | throw new RuntimeException(ex); |
---|
96 | } |
---|
97 | |
---|
98 | Record[] records = new Record[numRecords]; |
---|
99 | times.init = System.nanoTime(); |
---|
100 | try { |
---|
101 | init.invoke(null, new Object[]{records}); |
---|
102 | } catch (Exception ex) { |
---|
103 | throw new RuntimeException(ex); |
---|
104 | } |
---|
105 | times.init = System.nanoTime() - times.init; |
---|
106 | return records; |
---|
107 | } |
---|
108 | |
---|
109 | private static void runBinaryBench(String type, int numRecords, Times times) |
---|
110 | throws IOException { |
---|
111 | Record[] records = makeArray(type, numRecords, times); |
---|
112 | ByteArrayOutputStream bout = new ByteArrayOutputStream(); |
---|
113 | BinaryRecordOutput rout = new BinaryRecordOutput(bout); |
---|
114 | DataOutputStream dout = new DataOutputStream(bout); |
---|
115 | |
---|
116 | for(int idx = 0; idx < numRecords; idx++) { |
---|
117 | records[idx].serialize(rout); |
---|
118 | } |
---|
119 | bout.reset(); |
---|
120 | |
---|
121 | times.serialize = System.nanoTime(); |
---|
122 | for(int idx = 0; idx < numRecords; idx++) { |
---|
123 | records[idx].serialize(rout); |
---|
124 | } |
---|
125 | times.serialize = System.nanoTime() - times.serialize; |
---|
126 | |
---|
127 | byte[] serialized = bout.toByteArray(); |
---|
128 | ByteArrayInputStream bin = new ByteArrayInputStream(serialized); |
---|
129 | BinaryRecordInput rin = new BinaryRecordInput(bin); |
---|
130 | |
---|
131 | times.deserialize = System.nanoTime(); |
---|
132 | for(int idx = 0; idx < numRecords; idx++) { |
---|
133 | records[idx].deserialize(rin); |
---|
134 | } |
---|
135 | times.deserialize = System.nanoTime() - times.deserialize; |
---|
136 | |
---|
137 | bout.reset(); |
---|
138 | |
---|
139 | times.write = System.nanoTime(); |
---|
140 | for(int idx = 0; idx < numRecords; idx++) { |
---|
141 | records[idx].write(dout); |
---|
142 | } |
---|
143 | times.write = System.nanoTime() - times.write; |
---|
144 | |
---|
145 | bin.reset(); |
---|
146 | DataInputStream din = new DataInputStream(bin); |
---|
147 | |
---|
148 | times.readFields = System.nanoTime(); |
---|
149 | for(int idx = 0; idx < numRecords; idx++) { |
---|
150 | records[idx].readFields(din); |
---|
151 | } |
---|
152 | times.readFields = System.nanoTime() - times.readFields; |
---|
153 | } |
---|
154 | |
---|
155 | private static void runCsvBench(String type, int numRecords, Times times) |
---|
156 | throws IOException { |
---|
157 | Record[] records = makeArray(type, numRecords, times); |
---|
158 | ByteArrayOutputStream bout = new ByteArrayOutputStream(); |
---|
159 | CsvRecordOutput rout = new CsvRecordOutput(bout); |
---|
160 | |
---|
161 | for(int idx = 0; idx < numRecords; idx++) { |
---|
162 | records[idx].serialize(rout); |
---|
163 | } |
---|
164 | bout.reset(); |
---|
165 | |
---|
166 | times.serialize = System.nanoTime(); |
---|
167 | for(int idx = 0; idx < numRecords; idx++) { |
---|
168 | records[idx].serialize(rout); |
---|
169 | } |
---|
170 | times.serialize = System.nanoTime() - times.serialize; |
---|
171 | |
---|
172 | byte[] serialized = bout.toByteArray(); |
---|
173 | ByteArrayInputStream bin = new ByteArrayInputStream(serialized); |
---|
174 | CsvRecordInput rin = new CsvRecordInput(bin); |
---|
175 | |
---|
176 | times.deserialize = System.nanoTime(); |
---|
177 | for(int idx = 0; idx < numRecords; idx++) { |
---|
178 | records[idx].deserialize(rin); |
---|
179 | } |
---|
180 | times.deserialize = System.nanoTime() - times.deserialize; |
---|
181 | } |
---|
182 | |
---|
183 | private static void runXmlBench(String type, int numRecords, Times times) |
---|
184 | throws IOException { |
---|
185 | Record[] records = makeArray(type, numRecords, times); |
---|
186 | ByteArrayOutputStream bout = new ByteArrayOutputStream(); |
---|
187 | XmlRecordOutput rout = new XmlRecordOutput(bout); |
---|
188 | |
---|
189 | for(int idx = 0; idx < numRecords; idx++) { |
---|
190 | records[idx].serialize(rout); |
---|
191 | } |
---|
192 | bout.reset(); |
---|
193 | |
---|
194 | bout.write("<records>\n".getBytes()); |
---|
195 | |
---|
196 | times.serialize = System.nanoTime(); |
---|
197 | for(int idx = 0; idx < numRecords; idx++) { |
---|
198 | records[idx].serialize(rout); |
---|
199 | } |
---|
200 | times.serialize = System.nanoTime() - times.serialize; |
---|
201 | |
---|
202 | bout.write("</records>\n".getBytes()); |
---|
203 | |
---|
204 | byte[] serialized = bout.toByteArray(); |
---|
205 | ByteArrayInputStream bin = new ByteArrayInputStream(serialized); |
---|
206 | |
---|
207 | times.deserialize = System.nanoTime(); |
---|
208 | XmlRecordInput rin = new XmlRecordInput(bin); |
---|
209 | for(int idx = 0; idx < numRecords; idx++) { |
---|
210 | records[idx].deserialize(rin); |
---|
211 | } |
---|
212 | times.deserialize = System.nanoTime() - times.deserialize; |
---|
213 | } |
---|
214 | |
---|
215 | private static void printTimes(String type, |
---|
216 | String format, |
---|
217 | int numRecords, |
---|
218 | Times times) { |
---|
219 | System.out.println("Type: " + type + " Format: " + format + |
---|
220 | " #Records: "+numRecords); |
---|
221 | if (times.init != 0) { |
---|
222 | System.out.println("Initialization Time (Per record) : "+ |
---|
223 | times.init/numRecords + " Nanoseconds"); |
---|
224 | } |
---|
225 | |
---|
226 | if (times.serialize != 0) { |
---|
227 | System.out.println("Serialization Time (Per Record) : "+ |
---|
228 | times.serialize/numRecords + " Nanoseconds"); |
---|
229 | } |
---|
230 | |
---|
231 | if (times.deserialize != 0) { |
---|
232 | System.out.println("Deserialization Time (Per Record) : "+ |
---|
233 | times.deserialize/numRecords + " Nanoseconds"); |
---|
234 | } |
---|
235 | |
---|
236 | if (times.write != 0) { |
---|
237 | System.out.println("Write Time (Per Record) : "+ |
---|
238 | times.write/numRecords + " Nanoseconds"); |
---|
239 | } |
---|
240 | |
---|
241 | if (times.readFields != 0) { |
---|
242 | System.out.println("ReadFields Time (Per Record) : "+ |
---|
243 | times.readFields/numRecords + " Nanoseconds"); |
---|
244 | } |
---|
245 | |
---|
246 | System.out.println(); |
---|
247 | } |
---|
248 | |
---|
249 | private static String toCamelCase(String inp) { |
---|
250 | char firstChar = inp.charAt(0); |
---|
251 | if (Character.isLowerCase(firstChar)) { |
---|
252 | return ""+Character.toUpperCase(firstChar) + inp.substring(1); |
---|
253 | } |
---|
254 | return inp; |
---|
255 | } |
---|
256 | |
---|
257 | private static void exitOnError() { |
---|
258 | String usage = "RecordBench {buffer|string|int}"+ |
---|
259 | " {binary|csv|xml} <numRecords>"; |
---|
260 | System.out.println(usage); |
---|
261 | System.exit(1); |
---|
262 | } |
---|
263 | |
---|
264 | /** |
---|
265 | * @param args the command line arguments |
---|
266 | */ |
---|
267 | public static void main(String[] args) throws IOException { |
---|
268 | String version = "RecordBench v0.1"; |
---|
269 | System.out.println(version+"\n"); |
---|
270 | |
---|
271 | if (args.length != 3) { |
---|
272 | exitOnError(); |
---|
273 | } |
---|
274 | |
---|
275 | String typeName = args[0]; |
---|
276 | String format = args[1]; |
---|
277 | int numRecords = Integer.decode(args[2]).intValue(); |
---|
278 | |
---|
279 | Method bench = null; |
---|
280 | try { |
---|
281 | bench = RecordBench.class.getDeclaredMethod("run"+ |
---|
282 | toCamelCase(format) + "Bench", |
---|
283 | new Class[] {String.class, Integer.TYPE, Times.class}); |
---|
284 | } catch (NoSuchMethodException ex) { |
---|
285 | ex.printStackTrace(); |
---|
286 | exitOnError(); |
---|
287 | } |
---|
288 | |
---|
289 | if (numRecords < 0) { |
---|
290 | exitOnError(); |
---|
291 | } |
---|
292 | |
---|
293 | // dry run |
---|
294 | rand.setSeed(SEED); |
---|
295 | Times times = new Times(); |
---|
296 | try { |
---|
297 | bench.invoke(null, new Object[] {typeName, numRecords, times}); |
---|
298 | } catch (Exception ex) { |
---|
299 | ex.printStackTrace(); |
---|
300 | System.exit(1); |
---|
301 | } |
---|
302 | |
---|
303 | // timed run |
---|
304 | rand.setSeed(SEED); |
---|
305 | try { |
---|
306 | bench.invoke(null, new Object[] {typeName, numRecords, times}); |
---|
307 | } catch (Exception ex) { |
---|
308 | ex.printStackTrace(); |
---|
309 | System.exit(1); |
---|
310 | } |
---|
311 | printTimes(typeName, format, numRecords, times); |
---|
312 | } |
---|
313 | } |
---|