1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | #include "hadoop/SerialUtils.hh" |
---|
19 | #include "hadoop/StringUtils.hh" |
---|
20 | |
---|
21 | #include <errno.h> |
---|
22 | #include <rpc/types.h> |
---|
23 | #include <rpc/xdr.h> |
---|
24 | #include <string> |
---|
25 | |
---|
26 | using std::string; |
---|
27 | |
---|
28 | namespace HadoopUtils { |
---|
29 | |
---|
30 | Error::Error(const std::string& msg): error(msg) { |
---|
31 | } |
---|
32 | |
---|
33 | Error::Error(const std::string& msg, |
---|
34 | const std::string& file, int line, |
---|
35 | const std::string& function) { |
---|
36 | error = msg + " at " + file + ":" + toString(line) + |
---|
37 | " in " + function; |
---|
38 | } |
---|
39 | |
---|
40 | const std::string& Error::getMessage() const { |
---|
41 | return error; |
---|
42 | } |
---|
43 | |
---|
44 | FileInStream::FileInStream() |
---|
45 | { |
---|
46 | mFile = NULL; |
---|
47 | isOwned = false; |
---|
48 | } |
---|
49 | |
---|
50 | bool FileInStream::open(const std::string& name) |
---|
51 | { |
---|
52 | mFile = fopen(name.c_str(), "rb"); |
---|
53 | isOwned = true; |
---|
54 | return (mFile != NULL); |
---|
55 | } |
---|
56 | |
---|
57 | bool FileInStream::open(FILE* file) |
---|
58 | { |
---|
59 | mFile = file; |
---|
60 | isOwned = false; |
---|
61 | return (mFile != NULL); |
---|
62 | } |
---|
63 | |
---|
64 | void FileInStream::read(void *buf, size_t len) |
---|
65 | { |
---|
66 | size_t result = fread(buf, len, 1, mFile); |
---|
67 | if (result == 0) { |
---|
68 | if (feof(mFile)) { |
---|
69 | HADOOP_ASSERT(false, "end of file"); |
---|
70 | } else { |
---|
71 | HADOOP_ASSERT(false, string("read error on file: ") + strerror(errno)); |
---|
72 | } |
---|
73 | } |
---|
74 | } |
---|
75 | |
---|
76 | bool FileInStream::skip(size_t nbytes) |
---|
77 | { |
---|
78 | return (0==fseek(mFile, nbytes, SEEK_CUR)); |
---|
79 | } |
---|
80 | |
---|
81 | bool FileInStream::close() |
---|
82 | { |
---|
83 | int ret = 0; |
---|
84 | if (mFile != NULL && isOwned) { |
---|
85 | ret = fclose(mFile); |
---|
86 | } |
---|
87 | mFile = NULL; |
---|
88 | return (ret==0); |
---|
89 | } |
---|
90 | |
---|
91 | FileInStream::~FileInStream() |
---|
92 | { |
---|
93 | if (mFile != NULL) { |
---|
94 | close(); |
---|
95 | } |
---|
96 | } |
---|
97 | |
---|
98 | FileOutStream::FileOutStream() |
---|
99 | { |
---|
100 | mFile = NULL; |
---|
101 | isOwned = false; |
---|
102 | } |
---|
103 | |
---|
104 | bool FileOutStream::open(const std::string& name, bool overwrite) |
---|
105 | { |
---|
106 | if (!overwrite) { |
---|
107 | mFile = fopen(name.c_str(), "rb"); |
---|
108 | if (mFile != NULL) { |
---|
109 | fclose(mFile); |
---|
110 | return false; |
---|
111 | } |
---|
112 | } |
---|
113 | mFile = fopen(name.c_str(), "wb"); |
---|
114 | isOwned = true; |
---|
115 | return (mFile != NULL); |
---|
116 | } |
---|
117 | |
---|
118 | bool FileOutStream::open(FILE* file) |
---|
119 | { |
---|
120 | mFile = file; |
---|
121 | isOwned = false; |
---|
122 | return (mFile != NULL); |
---|
123 | } |
---|
124 | |
---|
125 | void FileOutStream::write(const void* buf, size_t len) |
---|
126 | { |
---|
127 | size_t result = fwrite(buf, len, 1, mFile); |
---|
128 | HADOOP_ASSERT(result == 1, |
---|
129 | string("write error to file: ") + strerror(errno)); |
---|
130 | } |
---|
131 | |
---|
132 | bool FileOutStream::advance(size_t nbytes) |
---|
133 | { |
---|
134 | return (0==fseek(mFile, nbytes, SEEK_CUR)); |
---|
135 | } |
---|
136 | |
---|
137 | bool FileOutStream::close() |
---|
138 | { |
---|
139 | int ret = 0; |
---|
140 | if (mFile != NULL && isOwned) { |
---|
141 | ret = fclose(mFile); |
---|
142 | } |
---|
143 | mFile = NULL; |
---|
144 | return (ret == 0); |
---|
145 | } |
---|
146 | |
---|
147 | void FileOutStream::flush() |
---|
148 | { |
---|
149 | fflush(mFile); |
---|
150 | } |
---|
151 | |
---|
152 | FileOutStream::~FileOutStream() |
---|
153 | { |
---|
154 | if (mFile != NULL) { |
---|
155 | close(); |
---|
156 | } |
---|
157 | } |
---|
158 | |
---|
159 | StringInStream::StringInStream(const std::string& str): buffer(str) { |
---|
160 | itr = buffer.begin(); |
---|
161 | } |
---|
162 | |
---|
163 | void StringInStream::read(void *buf, size_t buflen) { |
---|
164 | size_t bytes = 0; |
---|
165 | char* output = (char*) buf; |
---|
166 | std::string::const_iterator end = buffer.end(); |
---|
167 | while (bytes < buflen) { |
---|
168 | output[bytes++] = *itr; |
---|
169 | ++itr; |
---|
170 | if (itr == end) { |
---|
171 | break; |
---|
172 | } |
---|
173 | } |
---|
174 | HADOOP_ASSERT(bytes == buflen, "unexpected end of string reached"); |
---|
175 | } |
---|
176 | |
---|
177 | void serializeInt(int32_t t, OutStream& stream) { |
---|
178 | serializeLong(t,stream); |
---|
179 | } |
---|
180 | |
---|
181 | void serializeLong(int64_t t, OutStream& stream) |
---|
182 | { |
---|
183 | if (t >= -112 && t <= 127) { |
---|
184 | int8_t b = t; |
---|
185 | stream.write(&b, 1); |
---|
186 | return; |
---|
187 | } |
---|
188 | |
---|
189 | int8_t len = -112; |
---|
190 | if (t < 0) { |
---|
191 | t ^= -1ll; // reset the sign bit |
---|
192 | len = -120; |
---|
193 | } |
---|
194 | |
---|
195 | uint64_t tmp = t; |
---|
196 | while (tmp != 0) { |
---|
197 | tmp = tmp >> 8; |
---|
198 | len--; |
---|
199 | } |
---|
200 | |
---|
201 | stream.write(&len, 1); |
---|
202 | len = (len < -120) ? -(len + 120) : -(len + 112); |
---|
203 | |
---|
204 | for (uint32_t idx = len; idx != 0; idx--) { |
---|
205 | uint32_t shiftbits = (idx - 1) * 8; |
---|
206 | uint64_t mask = 0xFFll << shiftbits; |
---|
207 | uint8_t b = (t & mask) >> shiftbits; |
---|
208 | stream.write(&b, 1); |
---|
209 | } |
---|
210 | } |
---|
211 | |
---|
212 | int32_t deserializeInt(InStream& stream) { |
---|
213 | return deserializeLong(stream); |
---|
214 | } |
---|
215 | |
---|
216 | int64_t deserializeLong(InStream& stream) |
---|
217 | { |
---|
218 | int8_t b; |
---|
219 | stream.read(&b, 1); |
---|
220 | if (b >= -112) { |
---|
221 | return b; |
---|
222 | } |
---|
223 | bool negative; |
---|
224 | int len; |
---|
225 | if (b < -120) { |
---|
226 | negative = true; |
---|
227 | len = -120 - b; |
---|
228 | } else { |
---|
229 | negative = false; |
---|
230 | len = -112 - b; |
---|
231 | } |
---|
232 | uint8_t barr[len]; |
---|
233 | stream.read(barr, len); |
---|
234 | int64_t t = 0; |
---|
235 | for (int idx = 0; idx < len; idx++) { |
---|
236 | t = t << 8; |
---|
237 | t |= (barr[idx] & 0xFF); |
---|
238 | } |
---|
239 | if (negative) { |
---|
240 | t ^= -1ll; |
---|
241 | } |
---|
242 | return t; |
---|
243 | } |
---|
244 | |
---|
245 | void serializeFloat(float t, OutStream& stream) |
---|
246 | { |
---|
247 | char buf[sizeof(float)]; |
---|
248 | XDR xdrs; |
---|
249 | xdrmem_create(&xdrs, buf, sizeof(float), XDR_ENCODE); |
---|
250 | xdr_float(&xdrs, &t); |
---|
251 | stream.write(buf, sizeof(float)); |
---|
252 | } |
---|
253 | |
---|
254 | void deserializeFloat(float& t, InStream& stream) |
---|
255 | { |
---|
256 | char buf[sizeof(float)]; |
---|
257 | stream.read(buf, sizeof(float)); |
---|
258 | XDR xdrs; |
---|
259 | xdrmem_create(&xdrs, buf, sizeof(float), XDR_DECODE); |
---|
260 | xdr_float(&xdrs, &t); |
---|
261 | } |
---|
262 | |
---|
263 | void serializeString(const std::string& t, OutStream& stream) |
---|
264 | { |
---|
265 | serializeInt(t.length(), stream); |
---|
266 | if (t.length() > 0) { |
---|
267 | stream.write(t.data(), t.length()); |
---|
268 | } |
---|
269 | } |
---|
270 | |
---|
271 | void deserializeString(std::string& t, InStream& stream) |
---|
272 | { |
---|
273 | int32_t len = deserializeInt(stream); |
---|
274 | if (len > 0) { |
---|
275 | // resize the string to the right length |
---|
276 | t.resize(len); |
---|
277 | // read into the string in 64k chunks |
---|
278 | const int bufSize = 65536; |
---|
279 | int offset = 0; |
---|
280 | char buf[bufSize]; |
---|
281 | while (len > 0) { |
---|
282 | int chunkLength = len > bufSize ? bufSize : len; |
---|
283 | stream.read(buf, chunkLength); |
---|
284 | t.replace(offset, chunkLength, buf, chunkLength); |
---|
285 | offset += chunkLength; |
---|
286 | len -= chunkLength; |
---|
287 | } |
---|
288 | } else { |
---|
289 | t.clear(); |
---|
290 | } |
---|
291 | } |
---|
292 | |
---|
293 | } |
---|