1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | package org.apache.hadoop.fs; |
---|
19 | |
---|
20 | import java.io.Closeable; |
---|
21 | import java.io.FileNotFoundException; |
---|
22 | import java.io.IOException; |
---|
23 | import java.net.URI; |
---|
24 | import java.util.ArrayList; |
---|
25 | import java.util.Arrays; |
---|
26 | import java.util.Collection; |
---|
27 | import java.util.HashMap; |
---|
28 | import java.util.IdentityHashMap; |
---|
29 | import java.util.Iterator; |
---|
30 | import java.util.List; |
---|
31 | import java.util.Map; |
---|
32 | import java.util.Set; |
---|
33 | import java.util.TreeSet; |
---|
34 | import java.util.concurrent.atomic.AtomicLong; |
---|
35 | import java.util.regex.Pattern; |
---|
36 | |
---|
37 | import javax.security.auth.login.LoginException; |
---|
38 | |
---|
39 | import org.apache.commons.logging.*; |
---|
40 | |
---|
41 | import org.apache.hadoop.conf.*; |
---|
42 | import org.apache.hadoop.net.NetUtils; |
---|
43 | import org.apache.hadoop.util.*; |
---|
44 | import org.apache.hadoop.fs.permission.FsPermission; |
---|
45 | import org.apache.hadoop.io.MultipleIOException; |
---|
46 | import org.apache.hadoop.security.UserGroupInformation; |
---|
47 | |
---|
48 | /**************************************************************** |
---|
49 | * An abstract base class for a fairly generic filesystem. It |
---|
50 | * may be implemented as a distributed filesystem, or as a "local" |
---|
51 | * one that reflects the locally-connected disk. The local version |
---|
52 | * exists for small Hadoop instances and for testing. |
---|
53 | * |
---|
54 | * <p> |
---|
55 | * |
---|
56 | * All user code that may potentially use the Hadoop Distributed |
---|
57 | * File System should be written to use a FileSystem object. The |
---|
58 | * Hadoop DFS is a multi-machine system that appears as a single |
---|
59 | * disk. It's useful because of its fault tolerance and potentially |
---|
60 | * very large capacity. |
---|
61 | * |
---|
62 | * <p> |
---|
63 | * The local implementation is {@link LocalFileSystem} and distributed |
---|
64 | * implementation is DistributedFileSystem. |
---|
65 | *****************************************************************/ |
---|
66 | public abstract class FileSystem extends Configured implements Closeable { |
---|
67 | private static final String FS_DEFAULT_NAME_KEY = "fs.default.name"; |
---|
68 | |
---|
69 | public static final Log LOG = LogFactory.getLog(FileSystem.class); |
---|
70 | |
---|
71 | /** FileSystem cache */ |
---|
72 | private static final Cache CACHE = new Cache(); |
---|
73 | |
---|
74 | /** The key this instance is stored under in the cache. */ |
---|
75 | private Cache.Key key; |
---|
76 | |
---|
77 | /** Recording statistics per a FileSystem class */ |
---|
78 | private static final Map<Class<? extends FileSystem>, Statistics> |
---|
79 | statisticsTable = |
---|
80 | new IdentityHashMap<Class<? extends FileSystem>, Statistics>(); |
---|
81 | |
---|
82 | /** |
---|
83 | * The statistics for this file system. |
---|
84 | */ |
---|
85 | protected Statistics statistics; |
---|
86 | |
---|
87 | /** |
---|
88 | * A cache of files that should be deleted when filsystem is closed |
---|
89 | * or the JVM is exited. |
---|
90 | */ |
---|
91 | private Set<Path> deleteOnExit = new TreeSet<Path>(); |
---|
92 | |
---|
93 | /** Returns the configured filesystem implementation.*/ |
---|
94 | public static FileSystem get(Configuration conf) throws IOException { |
---|
95 | return get(getDefaultUri(conf), conf); |
---|
96 | } |
---|
97 | |
---|
98 | /** Get the default filesystem URI from a configuration. |
---|
99 | * @param conf the configuration to access |
---|
100 | * @return the uri of the default filesystem |
---|
101 | */ |
---|
102 | public static URI getDefaultUri(Configuration conf) { |
---|
103 | return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, "file:///"))); |
---|
104 | } |
---|
105 | |
---|
106 | /** Set the default filesystem URI in a configuration. |
---|
107 | * @param conf the configuration to alter |
---|
108 | * @param uri the new default filesystem uri |
---|
109 | */ |
---|
110 | public static void setDefaultUri(Configuration conf, URI uri) { |
---|
111 | conf.set(FS_DEFAULT_NAME_KEY, uri.toString()); |
---|
112 | } |
---|
113 | |
---|
114 | /** Set the default filesystem URI in a configuration. |
---|
115 | * @param conf the configuration to alter |
---|
116 | * @param uri the new default filesystem uri |
---|
117 | */ |
---|
118 | public static void setDefaultUri(Configuration conf, String uri) { |
---|
119 | setDefaultUri(conf, URI.create(fixName(uri))); |
---|
120 | } |
---|
121 | |
---|
122 | /** Called after a new FileSystem instance is constructed. |
---|
123 | * @param name a uri whose authority section names the host, port, etc. |
---|
124 | * for this FileSystem |
---|
125 | * @param conf the configuration |
---|
126 | */ |
---|
127 | public void initialize(URI name, Configuration conf) throws IOException { |
---|
128 | statistics = getStatistics(name.getScheme(), getClass()); |
---|
129 | } |
---|
130 | |
---|
131 | /** Returns a URI whose scheme and authority identify this FileSystem.*/ |
---|
132 | public abstract URI getUri(); |
---|
133 | |
---|
134 | /** @deprecated call #getUri() instead.*/ |
---|
135 | public String getName() { return getUri().toString(); } |
---|
136 | |
---|
137 | /** @deprecated call #get(URI,Configuration) instead. */ |
---|
138 | public static FileSystem getNamed(String name, Configuration conf) |
---|
139 | throws IOException { |
---|
140 | return get(URI.create(fixName(name)), conf); |
---|
141 | } |
---|
142 | |
---|
143 | /** Update old-format filesystem names, for back-compatibility. This should |
---|
144 | * eventually be replaced with a checkName() method that throws an exception |
---|
145 | * for old-format names. */ |
---|
146 | private static String fixName(String name) { |
---|
147 | // convert old-format name to new-format name |
---|
148 | if (name.equals("local")) { // "local" is now "file:///". |
---|
149 | LOG.warn("\"local\" is a deprecated filesystem name." |
---|
150 | +" Use \"file:///\" instead."); |
---|
151 | name = "file:///"; |
---|
152 | } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://" |
---|
153 | LOG.warn("\""+name+"\" is a deprecated filesystem name." |
---|
154 | +" Use \"hdfs://"+name+"/\" instead."); |
---|
155 | name = "hdfs://"+name; |
---|
156 | } |
---|
157 | return name; |
---|
158 | } |
---|
159 | |
---|
160 | /** |
---|
161 | * Get the local file syste |
---|
162 | * @param conf the configuration to configure the file system with |
---|
163 | * @return a LocalFileSystem |
---|
164 | */ |
---|
165 | public static LocalFileSystem getLocal(Configuration conf) |
---|
166 | throws IOException { |
---|
167 | return (LocalFileSystem)get(LocalFileSystem.NAME, conf); |
---|
168 | } |
---|
169 | |
---|
170 | /** Returns the FileSystem for this URI's scheme and authority. The scheme |
---|
171 | * of the URI determines a configuration property name, |
---|
172 | * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class. |
---|
173 | * The entire URI is passed to the FileSystem instance's initialize method. |
---|
174 | */ |
---|
175 | public static FileSystem get(URI uri, Configuration conf) throws IOException { |
---|
176 | String scheme = uri.getScheme(); |
---|
177 | String authority = uri.getAuthority(); |
---|
178 | |
---|
179 | if (scheme == null) { // no scheme: use default FS |
---|
180 | return get(conf); |
---|
181 | } |
---|
182 | |
---|
183 | if (authority == null) { // no authority |
---|
184 | URI defaultUri = getDefaultUri(conf); |
---|
185 | if (scheme.equals(defaultUri.getScheme()) // if scheme matches default |
---|
186 | && defaultUri.getAuthority() != null) { // & default has authority |
---|
187 | return get(defaultUri, conf); // return default |
---|
188 | } |
---|
189 | } |
---|
190 | |
---|
191 | return CACHE.get(uri, conf); |
---|
192 | } |
---|
193 | |
---|
194 | private static class ClientFinalizer extends Thread { |
---|
195 | public synchronized void run() { |
---|
196 | try { |
---|
197 | FileSystem.closeAll(); |
---|
198 | } catch (IOException e) { |
---|
199 | LOG.info("FileSystem.closeAll() threw an exception:\n" + e); |
---|
200 | } |
---|
201 | } |
---|
202 | } |
---|
203 | private static final ClientFinalizer clientFinalizer = new ClientFinalizer(); |
---|
204 | |
---|
205 | /** |
---|
206 | * Close all cached filesystems. Be sure those filesystems are not |
---|
207 | * used anymore. |
---|
208 | * |
---|
209 | * @throws IOException |
---|
210 | */ |
---|
211 | public static void closeAll() throws IOException { |
---|
212 | CACHE.closeAll(); |
---|
213 | } |
---|
214 | |
---|
215 | /** Make sure that a path specifies a FileSystem. */ |
---|
216 | public Path makeQualified(Path path) { |
---|
217 | checkPath(path); |
---|
218 | return path.makeQualified(this); |
---|
219 | } |
---|
220 | |
---|
221 | /** create a file with the provided permission |
---|
222 | * The permission of the file is set to be the provided permission as in |
---|
223 | * setPermission, not permission&~umask |
---|
224 | * |
---|
225 | * It is implemented using two RPCs. It is understood that it is inefficient, |
---|
226 | * but the implementation is thread-safe. The other option is to change the |
---|
227 | * value of umask in configuration to be 0, but it is not thread-safe. |
---|
228 | * |
---|
229 | * @param fs file system handle |
---|
230 | * @param file the name of the file to be created |
---|
231 | * @param permission the permission of the file |
---|
232 | * @return an output stream |
---|
233 | * @throws IOException |
---|
234 | */ |
---|
235 | public static FSDataOutputStream create(FileSystem fs, |
---|
236 | Path file, FsPermission permission) throws IOException { |
---|
237 | // create the file with default permission |
---|
238 | FSDataOutputStream out = fs.create(file); |
---|
239 | // set its permission to the supplied one |
---|
240 | fs.setPermission(file, permission); |
---|
241 | return out; |
---|
242 | } |
---|
243 | |
---|
244 | /** create a directory with the provided permission |
---|
245 | * The permission of the directory is set to be the provided permission as in |
---|
246 | * setPermission, not permission&~umask |
---|
247 | * |
---|
248 | * @see #create(FileSystem, Path, FsPermission) |
---|
249 | * |
---|
250 | * @param fs file system handle |
---|
251 | * @param dir the name of the directory to be created |
---|
252 | * @param permission the permission of the directory |
---|
253 | * @return true if the directory creation succeeds; false otherwise |
---|
254 | * @throws IOException |
---|
255 | */ |
---|
256 | public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) |
---|
257 | throws IOException { |
---|
258 | // create the directory using the default permission |
---|
259 | boolean result = fs.mkdirs(dir); |
---|
260 | // set its permission to be the supplied one |
---|
261 | fs.setPermission(dir, permission); |
---|
262 | return result; |
---|
263 | } |
---|
264 | |
---|
265 | /////////////////////////////////////////////////////////////// |
---|
266 | // FileSystem |
---|
267 | /////////////////////////////////////////////////////////////// |
---|
268 | |
---|
269 | protected FileSystem() { |
---|
270 | super(null); |
---|
271 | } |
---|
272 | |
---|
273 | /** Check that a Path belongs to this FileSystem. */ |
---|
274 | protected void checkPath(Path path) { |
---|
275 | URI uri = path.toUri(); |
---|
276 | if (uri.getScheme() == null) // fs is relative |
---|
277 | return; |
---|
278 | String thisScheme = this.getUri().getScheme(); |
---|
279 | String thatScheme = uri.getScheme(); |
---|
280 | String thisAuthority = this.getUri().getAuthority(); |
---|
281 | String thatAuthority = uri.getAuthority(); |
---|
282 | //authority and scheme are not case sensitive |
---|
283 | if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match |
---|
284 | if (thisAuthority == thatAuthority || // & authorities match |
---|
285 | (thisAuthority != null && |
---|
286 | thisAuthority.equalsIgnoreCase(thatAuthority))) |
---|
287 | return; |
---|
288 | |
---|
289 | if (thatAuthority == null && // path's authority is null |
---|
290 | thisAuthority != null) { // fs has an authority |
---|
291 | URI defaultUri = getDefaultUri(getConf()); // & is the conf default |
---|
292 | if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) && |
---|
293 | thisAuthority.equalsIgnoreCase(defaultUri.getAuthority())) |
---|
294 | return; |
---|
295 | try { // or the default fs's uri |
---|
296 | defaultUri = get(getConf()).getUri(); |
---|
297 | } catch (IOException e) { |
---|
298 | throw new RuntimeException(e); |
---|
299 | } |
---|
300 | if (thisScheme.equalsIgnoreCase(defaultUri.getScheme()) && |
---|
301 | thisAuthority.equalsIgnoreCase(defaultUri.getAuthority())) |
---|
302 | return; |
---|
303 | } |
---|
304 | } |
---|
305 | throw new IllegalArgumentException("Wrong FS: "+path+ |
---|
306 | ", expected: "+this.getUri()); |
---|
307 | } |
---|
308 | |
---|
309 | /** |
---|
310 | * Return an array containing hostnames, offset and size of |
---|
311 | * portions of the given file. For a nonexistent |
---|
312 | * file or regions, null will be returned. |
---|
313 | * |
---|
314 | * This call is most helpful with DFS, where it returns |
---|
315 | * hostnames of machines that contain the given file. |
---|
316 | * |
---|
317 | * The FileSystem will simply return an elt containing 'localhost'. |
---|
318 | */ |
---|
319 | public BlockLocation[] getFileBlockLocations(FileStatus file, |
---|
320 | long start, long len) throws IOException { |
---|
321 | if (file == null) { |
---|
322 | return null; |
---|
323 | } |
---|
324 | |
---|
325 | if ( (start<0) || (len < 0) ) { |
---|
326 | throw new IllegalArgumentException("Invalid start or len parameter"); |
---|
327 | } |
---|
328 | |
---|
329 | if (file.getLen() < start) { |
---|
330 | return new BlockLocation[0]; |
---|
331 | |
---|
332 | } |
---|
333 | String[] name = { "localhost:50010" }; |
---|
334 | String[] host = { "localhost" }; |
---|
335 | return new BlockLocation[] { new BlockLocation(name, host, 0, file.getLen()) }; |
---|
336 | } |
---|
337 | |
---|
338 | /** |
---|
339 | * Opens an FSDataInputStream at the indicated Path. |
---|
340 | * @param f the file name to open |
---|
341 | * @param bufferSize the size of the buffer to be used. |
---|
342 | */ |
---|
343 | public abstract FSDataInputStream open(Path f, int bufferSize) |
---|
344 | throws IOException; |
---|
345 | |
---|
346 | /** |
---|
347 | * Opens an FSDataInputStream at the indicated Path. |
---|
348 | * @param f the file to open |
---|
349 | */ |
---|
350 | public FSDataInputStream open(Path f) throws IOException { |
---|
351 | return open(f, getConf().getInt("io.file.buffer.size", 4096)); |
---|
352 | } |
---|
353 | |
---|
354 | /** |
---|
355 | * Opens an FSDataOutputStream at the indicated Path. |
---|
356 | * Files are overwritten by default. |
---|
357 | */ |
---|
358 | public FSDataOutputStream create(Path f) throws IOException { |
---|
359 | return create(f, true); |
---|
360 | } |
---|
361 | |
---|
362 | /** |
---|
363 | * Opens an FSDataOutputStream at the indicated Path. |
---|
364 | */ |
---|
365 | public FSDataOutputStream create(Path f, boolean overwrite) |
---|
366 | throws IOException { |
---|
367 | return create(f, overwrite, |
---|
368 | getConf().getInt("io.file.buffer.size", 4096), |
---|
369 | getDefaultReplication(), |
---|
370 | getDefaultBlockSize()); |
---|
371 | } |
---|
372 | |
---|
373 | /** |
---|
374 | * Create an FSDataOutputStream at the indicated Path with write-progress |
---|
375 | * reporting. |
---|
376 | * Files are overwritten by default. |
---|
377 | */ |
---|
378 | public FSDataOutputStream create(Path f, Progressable progress) throws IOException { |
---|
379 | return create(f, true, |
---|
380 | getConf().getInt("io.file.buffer.size", 4096), |
---|
381 | getDefaultReplication(), |
---|
382 | getDefaultBlockSize(), progress); |
---|
383 | } |
---|
384 | |
---|
385 | /** |
---|
386 | * Opens an FSDataOutputStream at the indicated Path. |
---|
387 | * Files are overwritten by default. |
---|
388 | */ |
---|
389 | public FSDataOutputStream create(Path f, short replication) |
---|
390 | throws IOException { |
---|
391 | return create(f, true, |
---|
392 | getConf().getInt("io.file.buffer.size", 4096), |
---|
393 | replication, |
---|
394 | getDefaultBlockSize()); |
---|
395 | } |
---|
396 | |
---|
397 | /** |
---|
398 | * Opens an FSDataOutputStream at the indicated Path with write-progress |
---|
399 | * reporting. |
---|
400 | * Files are overwritten by default. |
---|
401 | */ |
---|
402 | public FSDataOutputStream create(Path f, short replication, Progressable progress) |
---|
403 | throws IOException { |
---|
404 | return create(f, true, |
---|
405 | getConf().getInt("io.file.buffer.size", 4096), |
---|
406 | replication, |
---|
407 | getDefaultBlockSize(), progress); |
---|
408 | } |
---|
409 | |
---|
410 | |
---|
411 | /** |
---|
412 | * Opens an FSDataOutputStream at the indicated Path. |
---|
413 | * @param f the file name to open |
---|
414 | * @param overwrite if a file with this name already exists, then if true, |
---|
415 | * the file will be overwritten, and if false an error will be thrown. |
---|
416 | * @param bufferSize the size of the buffer to be used. |
---|
417 | */ |
---|
418 | public FSDataOutputStream create(Path f, |
---|
419 | boolean overwrite, |
---|
420 | int bufferSize |
---|
421 | ) throws IOException { |
---|
422 | return create(f, overwrite, bufferSize, |
---|
423 | getDefaultReplication(), |
---|
424 | getDefaultBlockSize()); |
---|
425 | } |
---|
426 | |
---|
427 | /** |
---|
428 | * Opens an FSDataOutputStream at the indicated Path with write-progress |
---|
429 | * reporting. |
---|
430 | * @param f the file name to open |
---|
431 | * @param overwrite if a file with this name already exists, then if true, |
---|
432 | * the file will be overwritten, and if false an error will be thrown. |
---|
433 | * @param bufferSize the size of the buffer to be used. |
---|
434 | */ |
---|
435 | public FSDataOutputStream create(Path f, |
---|
436 | boolean overwrite, |
---|
437 | int bufferSize, |
---|
438 | Progressable progress |
---|
439 | ) throws IOException { |
---|
440 | return create(f, overwrite, bufferSize, |
---|
441 | getDefaultReplication(), |
---|
442 | getDefaultBlockSize(), progress); |
---|
443 | } |
---|
444 | |
---|
445 | |
---|
446 | /** |
---|
447 | * Opens an FSDataOutputStream at the indicated Path. |
---|
448 | * @param f the file name to open |
---|
449 | * @param overwrite if a file with this name already exists, then if true, |
---|
450 | * the file will be overwritten, and if false an error will be thrown. |
---|
451 | * @param bufferSize the size of the buffer to be used. |
---|
452 | * @param replication required block replication for the file. |
---|
453 | */ |
---|
454 | public FSDataOutputStream create(Path f, |
---|
455 | boolean overwrite, |
---|
456 | int bufferSize, |
---|
457 | short replication, |
---|
458 | long blockSize |
---|
459 | ) throws IOException { |
---|
460 | return create(f, overwrite, bufferSize, replication, blockSize, null); |
---|
461 | } |
---|
462 | |
---|
463 | /** |
---|
464 | * Opens an FSDataOutputStream at the indicated Path with write-progress |
---|
465 | * reporting. |
---|
466 | * @param f the file name to open |
---|
467 | * @param overwrite if a file with this name already exists, then if true, |
---|
468 | * the file will be overwritten, and if false an error will be thrown. |
---|
469 | * @param bufferSize the size of the buffer to be used. |
---|
470 | * @param replication required block replication for the file. |
---|
471 | */ |
---|
472 | public FSDataOutputStream create(Path f, |
---|
473 | boolean overwrite, |
---|
474 | int bufferSize, |
---|
475 | short replication, |
---|
476 | long blockSize, |
---|
477 | Progressable progress |
---|
478 | ) throws IOException { |
---|
479 | return this.create(f, FsPermission.getDefault(), |
---|
480 | overwrite, bufferSize, replication, blockSize, progress); |
---|
481 | } |
---|
482 | |
---|
483 | /** |
---|
484 | * Opens an FSDataOutputStream at the indicated Path with write-progress |
---|
485 | * reporting. |
---|
486 | * @param f the file name to open |
---|
487 | * @param permission |
---|
488 | * @param overwrite if a file with this name already exists, then if true, |
---|
489 | * the file will be overwritten, and if false an error will be thrown. |
---|
490 | * @param bufferSize the size of the buffer to be used. |
---|
491 | * @param replication required block replication for the file. |
---|
492 | * @param blockSize |
---|
493 | * @param progress |
---|
494 | * @throws IOException |
---|
495 | * @see #setPermission(Path, FsPermission) |
---|
496 | */ |
---|
497 | public abstract FSDataOutputStream create(Path f, |
---|
498 | FsPermission permission, |
---|
499 | boolean overwrite, |
---|
500 | int bufferSize, |
---|
501 | short replication, |
---|
502 | long blockSize, |
---|
503 | Progressable progress) throws IOException; |
---|
504 | |
---|
505 | /** |
---|
506 | * Creates the given Path as a brand-new zero-length file. If |
---|
507 | * create fails, or if it already existed, return false. |
---|
508 | */ |
---|
509 | public boolean createNewFile(Path f) throws IOException { |
---|
510 | if (exists(f)) { |
---|
511 | return false; |
---|
512 | } else { |
---|
513 | create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close(); |
---|
514 | return true; |
---|
515 | } |
---|
516 | } |
---|
517 | |
---|
518 | /** |
---|
519 | * Append to an existing file (optional operation). |
---|
520 | * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) |
---|
521 | * @param f the existing file to be appended. |
---|
522 | * @throws IOException |
---|
523 | */ |
---|
524 | public FSDataOutputStream append(Path f) throws IOException { |
---|
525 | return append(f, getConf().getInt("io.file.buffer.size", 4096), null); |
---|
526 | } |
---|
527 | /** |
---|
528 | * Append to an existing file (optional operation). |
---|
529 | * Same as append(f, bufferSize, null). |
---|
530 | * @param f the existing file to be appended. |
---|
531 | * @param bufferSize the size of the buffer to be used. |
---|
532 | * @throws IOException |
---|
533 | */ |
---|
534 | public FSDataOutputStream append(Path f, int bufferSize) throws IOException { |
---|
535 | return append(f, bufferSize, null); |
---|
536 | } |
---|
537 | |
---|
538 | /** |
---|
539 | * Append to an existing file (optional operation). |
---|
540 | * @param f the existing file to be appended. |
---|
541 | * @param bufferSize the size of the buffer to be used. |
---|
542 | * @param progress for reporting progress if it is not null. |
---|
543 | * @throws IOException |
---|
544 | */ |
---|
545 | public abstract FSDataOutputStream append(Path f, int bufferSize, |
---|
546 | Progressable progress) throws IOException; |
---|
547 | |
---|
548 | /** |
---|
549 | * Get replication. |
---|
550 | * |
---|
551 | * @deprecated Use getFileStatus() instead |
---|
552 | * @param src file name |
---|
553 | * @return file replication |
---|
554 | * @throws IOException |
---|
555 | */ |
---|
556 | @Deprecated |
---|
557 | public short getReplication(Path src) throws IOException { |
---|
558 | return getFileStatus(src).getReplication(); |
---|
559 | } |
---|
560 | |
---|
561 | /** |
---|
562 | * Set replication for an existing file. |
---|
563 | * |
---|
564 | * @param src file name |
---|
565 | * @param replication new replication |
---|
566 | * @throws IOException |
---|
567 | * @return true if successful; |
---|
568 | * false if file does not exist or is a directory |
---|
569 | */ |
---|
570 | public boolean setReplication(Path src, short replication) |
---|
571 | throws IOException { |
---|
572 | return true; |
---|
573 | } |
---|
574 | |
---|
575 | /** |
---|
576 | * Renames Path src to Path dst. Can take place on local fs |
---|
577 | * or remote DFS. |
---|
578 | */ |
---|
579 | public abstract boolean rename(Path src, Path dst) throws IOException; |
---|
580 | |
---|
581 | /** Delete a file. */ |
---|
582 | /** @deprecated Use delete(Path, boolean) instead */ @Deprecated |
---|
583 | public abstract boolean delete(Path f) throws IOException; |
---|
584 | |
---|
585 | /** Delete a file. |
---|
586 | * |
---|
587 | * @param f the path to delete. |
---|
588 | * @param recursive if path is a directory and set to |
---|
589 | * true, the directory is deleted else throws an exception. In |
---|
590 | * case of a file the recursive can be set to either true or false. |
---|
591 | * @return true if delete is successful else false. |
---|
592 | * @throws IOException |
---|
593 | */ |
---|
594 | public abstract boolean delete(Path f, boolean recursive) throws IOException; |
---|
595 | |
---|
596 | /** |
---|
597 | * Mark a path to be deleted when FileSystem is closed. |
---|
598 | * When the JVM shuts down, |
---|
599 | * all FileSystem objects will be closed automatically. |
---|
600 | * Then, |
---|
601 | * the marked path will be deleted as a result of closing the FileSystem. |
---|
602 | * |
---|
603 | * The path has to exist in the file system. |
---|
604 | * |
---|
605 | * @param f the path to delete. |
---|
606 | * @return true if deleteOnExit is successful, otherwise false. |
---|
607 | * @throws IOException |
---|
608 | */ |
---|
609 | public boolean deleteOnExit(Path f) throws IOException { |
---|
610 | if (!exists(f)) { |
---|
611 | return false; |
---|
612 | } |
---|
613 | synchronized (deleteOnExit) { |
---|
614 | deleteOnExit.add(f); |
---|
615 | } |
---|
616 | return true; |
---|
617 | } |
---|
618 | |
---|
619 | /** |
---|
620 | * Delete all files that were marked as delete-on-exit. This recursively |
---|
621 | * deletes all files in the specified paths. |
---|
622 | */ |
---|
623 | protected void processDeleteOnExit() { |
---|
624 | synchronized (deleteOnExit) { |
---|
625 | for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) { |
---|
626 | Path path = iter.next(); |
---|
627 | try { |
---|
628 | delete(path, true); |
---|
629 | } |
---|
630 | catch (IOException e) { |
---|
631 | LOG.info("Ignoring failure to deleteOnExit for path " + path); |
---|
632 | } |
---|
633 | iter.remove(); |
---|
634 | } |
---|
635 | } |
---|
636 | } |
---|
637 | |
---|
638 | /** Check if exists. |
---|
639 | * @param f source file |
---|
640 | */ |
---|
641 | public boolean exists(Path f) throws IOException { |
---|
642 | try { |
---|
643 | return getFileStatus(f) != null; |
---|
644 | } catch (FileNotFoundException e) { |
---|
645 | return false; |
---|
646 | } |
---|
647 | } |
---|
648 | |
---|
649 | /** True iff the named path is a directory. */ |
---|
650 | /** @deprecated Use getFileStatus() instead */ @Deprecated |
---|
651 | public boolean isDirectory(Path f) throws IOException { |
---|
652 | try { |
---|
653 | return getFileStatus(f).isDir(); |
---|
654 | } catch (FileNotFoundException e) { |
---|
655 | return false; // f does not exist |
---|
656 | } |
---|
657 | } |
---|
658 | |
---|
659 | /** True iff the named path is a regular file. */ |
---|
660 | public boolean isFile(Path f) throws IOException { |
---|
661 | try { |
---|
662 | return !getFileStatus(f).isDir(); |
---|
663 | } catch (FileNotFoundException e) { |
---|
664 | return false; // f does not exist |
---|
665 | } |
---|
666 | } |
---|
667 | |
---|
668 | /** The number of bytes in a file. */ |
---|
669 | /** @deprecated Use getFileStatus() instead */ @Deprecated |
---|
670 | public long getLength(Path f) throws IOException { |
---|
671 | return getFileStatus(f).getLen(); |
---|
672 | } |
---|
673 | |
---|
674 | /** Return the {@link ContentSummary} of a given {@link Path}. */ |
---|
675 | public ContentSummary getContentSummary(Path f) throws IOException { |
---|
676 | FileStatus status = getFileStatus(f); |
---|
677 | if (!status.isDir()) { |
---|
678 | // f is a file |
---|
679 | return new ContentSummary(status.getLen(), 1, 0); |
---|
680 | } |
---|
681 | // f is a directory |
---|
682 | long[] summary = {0, 0, 1}; |
---|
683 | for(FileStatus s : listStatus(f)) { |
---|
684 | ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : |
---|
685 | new ContentSummary(s.getLen(), 1, 0); |
---|
686 | summary[0] += c.getLength(); |
---|
687 | summary[1] += c.getFileCount(); |
---|
688 | summary[2] += c.getDirectoryCount(); |
---|
689 | } |
---|
690 | return new ContentSummary(summary[0], summary[1], summary[2]); |
---|
691 | } |
---|
692 | |
---|
693 | final private static PathFilter DEFAULT_FILTER = new PathFilter() { |
---|
694 | public boolean accept(Path file) { |
---|
695 | return true; |
---|
696 | } |
---|
697 | }; |
---|
698 | |
---|
699 | /** |
---|
700 | * List the statuses of the files/directories in the given path if the path is |
---|
701 | * a directory. |
---|
702 | * |
---|
703 | * @param f |
---|
704 | * given path |
---|
705 | * @return the statuses of the files/directories in the given patch |
---|
706 | * @throws IOException |
---|
707 | */ |
---|
708 | public abstract FileStatus[] listStatus(Path f) throws IOException; |
---|
709 | |
---|
710 | /* |
---|
711 | * Filter files/directories in the given path using the user-supplied path |
---|
712 | * filter. Results are added to the given array <code>results</code>. |
---|
713 | */ |
---|
714 | private void listStatus(ArrayList<FileStatus> results, Path f, |
---|
715 | PathFilter filter) throws IOException { |
---|
716 | FileStatus listing[] = listStatus(f); |
---|
717 | if (listing != null) { |
---|
718 | for (int i = 0; i < listing.length; i++) { |
---|
719 | if (filter.accept(listing[i].getPath())) { |
---|
720 | results.add(listing[i]); |
---|
721 | } |
---|
722 | } |
---|
723 | } |
---|
724 | } |
---|
725 | |
---|
726 | /** |
---|
727 | * Filter files/directories in the given path using the user-supplied path |
---|
728 | * filter. |
---|
729 | * |
---|
730 | * @param f |
---|
731 | * a path name |
---|
732 | * @param filter |
---|
733 | * the user-supplied path filter |
---|
734 | * @return an array of FileStatus objects for the files under the given path |
---|
735 | * after applying the filter |
---|
736 | * @throws IOException |
---|
737 | * if encounter any problem while fetching the status |
---|
738 | */ |
---|
739 | public FileStatus[] listStatus(Path f, PathFilter filter) throws IOException { |
---|
740 | ArrayList<FileStatus> results = new ArrayList<FileStatus>(); |
---|
741 | listStatus(results, f, filter); |
---|
742 | return results.toArray(new FileStatus[results.size()]); |
---|
743 | } |
---|
744 | |
---|
745 | /** |
---|
746 | * Filter files/directories in the given list of paths using default |
---|
747 | * path filter. |
---|
748 | * |
---|
749 | * @param files |
---|
750 | * a list of paths |
---|
751 | * @return a list of statuses for the files under the given paths after |
---|
752 | * applying the filter default Path filter |
---|
753 | * @exception IOException |
---|
754 | */ |
---|
755 | public FileStatus[] listStatus(Path[] files) |
---|
756 | throws IOException { |
---|
757 | return listStatus(files, DEFAULT_FILTER); |
---|
758 | } |
---|
759 | |
---|
760 | /** |
---|
761 | * Filter files/directories in the given list of paths using user-supplied |
---|
762 | * path filter. |
---|
763 | * |
---|
764 | * @param files |
---|
765 | * a list of paths |
---|
766 | * @param filter |
---|
767 | * the user-supplied path filter |
---|
768 | * @return a list of statuses for the files under the given paths after |
---|
769 | * applying the filter |
---|
770 | * @exception IOException |
---|
771 | */ |
---|
772 | public FileStatus[] listStatus(Path[] files, PathFilter filter) |
---|
773 | throws IOException { |
---|
774 | ArrayList<FileStatus> results = new ArrayList<FileStatus>(); |
---|
775 | for (int i = 0; i < files.length; i++) { |
---|
776 | listStatus(results, files[i], filter); |
---|
777 | } |
---|
778 | return results.toArray(new FileStatus[results.size()]); |
---|
779 | } |
---|
780 | |
---|
781 | /** |
---|
782 | * <p>Return all the files that match filePattern and are not checksum |
---|
783 | * files. Results are sorted by their names. |
---|
784 | * |
---|
785 | * <p> |
---|
786 | * A filename pattern is composed of <i>regular</i> characters and |
---|
787 | * <i>special pattern matching</i> characters, which are: |
---|
788 | * |
---|
789 | * <dl> |
---|
790 | * <dd> |
---|
791 | * <dl> |
---|
792 | * <p> |
---|
793 | * <dt> <tt> ? </tt> |
---|
794 | * <dd> Matches any single character. |
---|
795 | * |
---|
796 | * <p> |
---|
797 | * <dt> <tt> * </tt> |
---|
798 | * <dd> Matches zero or more characters. |
---|
799 | * |
---|
800 | * <p> |
---|
801 | * <dt> <tt> [<i>abc</i>] </tt> |
---|
802 | * <dd> Matches a single character from character set |
---|
803 | * <tt>{<i>a,b,c</i>}</tt>. |
---|
804 | * |
---|
805 | * <p> |
---|
806 | * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> |
---|
807 | * <dd> Matches a single character from the character range |
---|
808 | * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be |
---|
809 | * lexicographically less than or equal to character <tt><i>b</i></tt>. |
---|
810 | * |
---|
811 | * <p> |
---|
812 | * <dt> <tt> [^<i>a</i>] </tt> |
---|
813 | * <dd> Matches a single character that is not from character set or range |
---|
814 | * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur |
---|
815 | * immediately to the right of the opening bracket. |
---|
816 | * |
---|
817 | * <p> |
---|
818 | * <dt> <tt> \<i>c</i> </tt> |
---|
819 | * <dd> Removes (escapes) any special meaning of character <i>c</i>. |
---|
820 | * |
---|
821 | * <p> |
---|
822 | * <dt> <tt> {ab,cd} </tt> |
---|
823 | * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> |
---|
824 | * |
---|
825 | * <p> |
---|
826 | * <dt> <tt> {ab,c{de,fh}} </tt> |
---|
827 | * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt> |
---|
828 | * |
---|
829 | * </dl> |
---|
830 | * </dd> |
---|
831 | * </dl> |
---|
832 | * |
---|
833 | * @param pathPattern a regular expression specifying a pth pattern |
---|
834 | |
---|
835 | * @return an array of paths that match the path pattern |
---|
836 | * @throws IOException |
---|
837 | */ |
---|
838 | public FileStatus[] globStatus(Path pathPattern) throws IOException { |
---|
839 | return globStatus(pathPattern, DEFAULT_FILTER); |
---|
840 | } |
---|
841 | |
---|
842 | /** |
---|
843 | * Return an array of FileStatus objects whose path names match pathPattern |
---|
844 | * and is accepted by the user-supplied path filter. Results are sorted by |
---|
845 | * their path names. |
---|
846 | * Return null if pathPattern has no glob and the path does not exist. |
---|
847 | * Return an empty array if pathPattern has a glob and no path matches it. |
---|
848 | * |
---|
849 | * @param pathPattern |
---|
850 | * a regular expression specifying the path pattern |
---|
851 | * @param filter |
---|
852 | * a user-supplied path filter |
---|
853 | * @return an array of FileStatus objects |
---|
854 | * @throws IOException if any I/O error occurs when fetching file status |
---|
855 | */ |
---|
856 | public FileStatus[] globStatus(Path pathPattern, PathFilter filter) |
---|
857 | throws IOException { |
---|
858 | String filename = pathPattern.toUri().getPath(); |
---|
859 | List<String> filePatterns = GlobExpander.expand(filename); |
---|
860 | if (filePatterns.size() == 1) { |
---|
861 | return globStatusInternal(pathPattern, filter); |
---|
862 | } else { |
---|
863 | List<FileStatus> results = new ArrayList<FileStatus>(); |
---|
864 | for (String filePattern : filePatterns) { |
---|
865 | FileStatus[] files = globStatusInternal(new Path(filePattern), filter); |
---|
866 | for (FileStatus file : files) { |
---|
867 | results.add(file); |
---|
868 | } |
---|
869 | } |
---|
870 | return results.toArray(new FileStatus[results.size()]); |
---|
871 | } |
---|
872 | } |
---|
873 | |
---|
874 | private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter) |
---|
875 | throws IOException { |
---|
876 | Path[] parents = new Path[1]; |
---|
877 | int level = 0; |
---|
878 | String filename = pathPattern.toUri().getPath(); |
---|
879 | |
---|
880 | // path has only zero component |
---|
881 | if ("".equals(filename) || Path.SEPARATOR.equals(filename)) { |
---|
882 | return getFileStatus(new Path[]{pathPattern}); |
---|
883 | } |
---|
884 | |
---|
885 | // path has at least one component |
---|
886 | String[] components = filename.split(Path.SEPARATOR); |
---|
887 | // get the first component |
---|
888 | if (pathPattern.isAbsolute()) { |
---|
889 | parents[0] = new Path(Path.SEPARATOR); |
---|
890 | level = 1; |
---|
891 | } else { |
---|
892 | parents[0] = new Path(Path.CUR_DIR); |
---|
893 | } |
---|
894 | |
---|
895 | // glob the paths that match the parent path, i.e., [0, components.length-1] |
---|
896 | boolean[] hasGlob = new boolean[]{false}; |
---|
897 | Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob); |
---|
898 | FileStatus[] results; |
---|
899 | if (parentPaths == null || parentPaths.length == 0) { |
---|
900 | results = null; |
---|
901 | } else { |
---|
902 | // Now work on the last component of the path |
---|
903 | GlobFilter fp = new GlobFilter(components[components.length - 1], filter); |
---|
904 | if (fp.hasPattern()) { // last component has a pattern |
---|
905 | // list parent directories and then glob the results |
---|
906 | results = listStatus(parentPaths, fp); |
---|
907 | hasGlob[0] = true; |
---|
908 | } else { // last component does not have a pattern |
---|
909 | // get all the path names |
---|
910 | ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length); |
---|
911 | for (int i = 0; i < parentPaths.length; i++) { |
---|
912 | parentPaths[i] = new Path(parentPaths[i], |
---|
913 | components[components.length - 1]); |
---|
914 | if (fp.accept(parentPaths[i])) { |
---|
915 | filteredPaths.add(parentPaths[i]); |
---|
916 | } |
---|
917 | } |
---|
918 | // get all their statuses |
---|
919 | results = getFileStatus( |
---|
920 | filteredPaths.toArray(new Path[filteredPaths.size()])); |
---|
921 | } |
---|
922 | } |
---|
923 | |
---|
924 | // Decide if the pathPattern contains a glob or not |
---|
925 | if (results == null) { |
---|
926 | if (hasGlob[0]) { |
---|
927 | results = new FileStatus[0]; |
---|
928 | } |
---|
929 | } else { |
---|
930 | if (results.length == 0 ) { |
---|
931 | if (!hasGlob[0]) { |
---|
932 | results = null; |
---|
933 | } |
---|
934 | } else { |
---|
935 | Arrays.sort(results); |
---|
936 | } |
---|
937 | } |
---|
938 | return results; |
---|
939 | } |
---|
940 | |
---|
941 | /* |
---|
942 | * For a path of N components, return a list of paths that match the |
---|
943 | * components [<code>level</code>, <code>N-1</code>]. |
---|
944 | */ |
---|
945 | private Path[] globPathsLevel(Path[] parents, String[] filePattern, |
---|
946 | int level, boolean[] hasGlob) throws IOException { |
---|
947 | if (level == filePattern.length - 1) |
---|
948 | return parents; |
---|
949 | if (parents == null || parents.length == 0) { |
---|
950 | return null; |
---|
951 | } |
---|
952 | GlobFilter fp = new GlobFilter(filePattern[level]); |
---|
953 | if (fp.hasPattern()) { |
---|
954 | parents = FileUtil.stat2Paths(listStatus(parents, fp)); |
---|
955 | hasGlob[0] = true; |
---|
956 | } else { |
---|
957 | for (int i = 0; i < parents.length; i++) { |
---|
958 | parents[i] = new Path(parents[i], filePattern[level]); |
---|
959 | } |
---|
960 | } |
---|
961 | return globPathsLevel(parents, filePattern, level + 1, hasGlob); |
---|
962 | } |
---|
963 | |
---|
964 | /* A class that could decide if a string matches the glob or not */ |
---|
965 | private static class GlobFilter implements PathFilter { |
---|
966 | private PathFilter userFilter = DEFAULT_FILTER; |
---|
967 | private Pattern regex; |
---|
968 | private boolean hasPattern = false; |
---|
969 | |
---|
970 | /** Default pattern character: Escape any special meaning. */ |
---|
971 | private static final char PAT_ESCAPE = '\\'; |
---|
972 | /** Default pattern character: Any single character. */ |
---|
973 | private static final char PAT_ANY = '.'; |
---|
974 | /** Default pattern character: Character set close. */ |
---|
975 | private static final char PAT_SET_CLOSE = ']'; |
---|
976 | |
---|
977 | GlobFilter() { |
---|
978 | } |
---|
979 | |
---|
980 | GlobFilter(String filePattern) throws IOException { |
---|
981 | setRegex(filePattern); |
---|
982 | } |
---|
983 | |
---|
984 | GlobFilter(String filePattern, PathFilter filter) throws IOException { |
---|
985 | userFilter = filter; |
---|
986 | setRegex(filePattern); |
---|
987 | } |
---|
988 | |
---|
989 | private boolean isJavaRegexSpecialChar(char pChar) { |
---|
990 | return pChar == '.' || pChar == '$' || pChar == '(' || pChar == ')' || |
---|
991 | pChar == '|' || pChar == '+'; |
---|
992 | } |
---|
993 | void setRegex(String filePattern) throws IOException { |
---|
994 | int len; |
---|
995 | int setOpen; |
---|
996 | int curlyOpen; |
---|
997 | boolean setRange; |
---|
998 | |
---|
999 | StringBuilder fileRegex = new StringBuilder(); |
---|
1000 | |
---|
1001 | // Validate the pattern |
---|
1002 | len = filePattern.length(); |
---|
1003 | if (len == 0) |
---|
1004 | return; |
---|
1005 | |
---|
1006 | setOpen = 0; |
---|
1007 | setRange = false; |
---|
1008 | curlyOpen = 0; |
---|
1009 | |
---|
1010 | for (int i = 0; i < len; i++) { |
---|
1011 | char pCh; |
---|
1012 | |
---|
1013 | // Examine a single pattern character |
---|
1014 | pCh = filePattern.charAt(i); |
---|
1015 | if (pCh == PAT_ESCAPE) { |
---|
1016 | fileRegex.append(pCh); |
---|
1017 | i++; |
---|
1018 | if (i >= len) |
---|
1019 | error("An escaped character does not present", filePattern, i); |
---|
1020 | pCh = filePattern.charAt(i); |
---|
1021 | } else if (isJavaRegexSpecialChar(pCh)) { |
---|
1022 | fileRegex.append(PAT_ESCAPE); |
---|
1023 | } else if (pCh == '*') { |
---|
1024 | fileRegex.append(PAT_ANY); |
---|
1025 | hasPattern = true; |
---|
1026 | } else if (pCh == '?') { |
---|
1027 | pCh = PAT_ANY; |
---|
1028 | hasPattern = true; |
---|
1029 | } else if (pCh == '{') { |
---|
1030 | fileRegex.append('('); |
---|
1031 | pCh = '('; |
---|
1032 | curlyOpen++; |
---|
1033 | hasPattern = true; |
---|
1034 | } else if (pCh == ',' && curlyOpen > 0) { |
---|
1035 | fileRegex.append(")|"); |
---|
1036 | pCh = '('; |
---|
1037 | } else if (pCh == '}' && curlyOpen > 0) { |
---|
1038 | // End of a group |
---|
1039 | curlyOpen--; |
---|
1040 | fileRegex.append(")"); |
---|
1041 | pCh = ')'; |
---|
1042 | } else if (pCh == '[' && setOpen == 0) { |
---|
1043 | setOpen++; |
---|
1044 | hasPattern = true; |
---|
1045 | } else if (pCh == '^' && setOpen > 0) { |
---|
1046 | } else if (pCh == '-' && setOpen > 0) { |
---|
1047 | // Character set range |
---|
1048 | setRange = true; |
---|
1049 | } else if (pCh == PAT_SET_CLOSE && setRange) { |
---|
1050 | // Incomplete character set range |
---|
1051 | error("Incomplete character set range", filePattern, i); |
---|
1052 | } else if (pCh == PAT_SET_CLOSE && setOpen > 0) { |
---|
1053 | // End of a character set |
---|
1054 | if (setOpen < 2) |
---|
1055 | error("Unexpected end of set", filePattern, i); |
---|
1056 | setOpen = 0; |
---|
1057 | } else if (setOpen > 0) { |
---|
1058 | // Normal character, or the end of a character set range |
---|
1059 | setOpen++; |
---|
1060 | setRange = false; |
---|
1061 | } |
---|
1062 | fileRegex.append(pCh); |
---|
1063 | } |
---|
1064 | |
---|
1065 | // Check for a well-formed pattern |
---|
1066 | if (setOpen > 0 || setRange || curlyOpen > 0) { |
---|
1067 | // Incomplete character set or character range |
---|
1068 | error("Expecting set closure character or end of range, or }", |
---|
1069 | filePattern, len); |
---|
1070 | } |
---|
1071 | regex = Pattern.compile(fileRegex.toString()); |
---|
1072 | } |
---|
1073 | |
---|
1074 | boolean hasPattern() { |
---|
1075 | return hasPattern; |
---|
1076 | } |
---|
1077 | |
---|
1078 | public boolean accept(Path path) { |
---|
1079 | return regex.matcher(path.getName()).matches() && userFilter.accept(path); |
---|
1080 | } |
---|
1081 | |
---|
1082 | private void error(String s, String pattern, int pos) throws IOException { |
---|
1083 | throw new IOException("Illegal file pattern: " |
---|
1084 | +s+ " for glob "+ pattern + " at " + pos); |
---|
1085 | } |
---|
1086 | } |
---|
1087 | |
---|
1088 | /** Return the current user's home directory in this filesystem. |
---|
1089 | * The default implementation returns "/user/$USER/". |
---|
1090 | */ |
---|
1091 | public Path getHomeDirectory() { |
---|
1092 | return new Path("/user/"+System.getProperty("user.name")) |
---|
1093 | .makeQualified(this); |
---|
1094 | } |
---|
1095 | |
---|
1096 | |
---|
1097 | /** |
---|
1098 | * Set the current working directory for the given file system. All relative |
---|
1099 | * paths will be resolved relative to it. |
---|
1100 | * |
---|
1101 | * @param new_dir |
---|
1102 | */ |
---|
1103 | public abstract void setWorkingDirectory(Path new_dir); |
---|
1104 | |
---|
1105 | /** |
---|
1106 | * Get the current working directory for the given file system |
---|
1107 | * @return the directory pathname |
---|
1108 | */ |
---|
1109 | public abstract Path getWorkingDirectory(); |
---|
1110 | |
---|
1111 | /** |
---|
1112 | * Call {@link #mkdirs(Path, FsPermission)} with default permission. |
---|
1113 | */ |
---|
1114 | public boolean mkdirs(Path f) throws IOException { |
---|
1115 | return mkdirs(f, FsPermission.getDefault()); |
---|
1116 | } |
---|
1117 | |
---|
1118 | /** |
---|
1119 | * Make the given file and all non-existent parents into |
---|
1120 | * directories. Has the semantics of Unix 'mkdir -p'. |
---|
1121 | * Existence of the directory hierarchy is not an error. |
---|
1122 | */ |
---|
1123 | public abstract boolean mkdirs(Path f, FsPermission permission |
---|
1124 | ) throws IOException; |
---|
1125 | |
---|
1126 | /** |
---|
1127 | * The src file is on the local disk. Add it to FS at |
---|
1128 | * the given dst name and the source is kept intact afterwards |
---|
1129 | */ |
---|
1130 | public void copyFromLocalFile(Path src, Path dst) |
---|
1131 | throws IOException { |
---|
1132 | copyFromLocalFile(false, src, dst); |
---|
1133 | } |
---|
1134 | |
---|
1135 | /** |
---|
1136 | * The src files is on the local disk. Add it to FS at |
---|
1137 | * the given dst name, removing the source afterwards. |
---|
1138 | */ |
---|
1139 | public void moveFromLocalFile(Path[] srcs, Path dst) |
---|
1140 | throws IOException { |
---|
1141 | copyFromLocalFile(true, true, srcs, dst); |
---|
1142 | } |
---|
1143 | |
---|
1144 | /** |
---|
1145 | * The src file is on the local disk. Add it to FS at |
---|
1146 | * the given dst name, removing the source afterwards. |
---|
1147 | */ |
---|
1148 | public void moveFromLocalFile(Path src, Path dst) |
---|
1149 | throws IOException { |
---|
1150 | copyFromLocalFile(true, src, dst); |
---|
1151 | } |
---|
1152 | |
---|
1153 | /** |
---|
1154 | * The src file is on the local disk. Add it to FS at |
---|
1155 | * the given dst name. |
---|
1156 | * delSrc indicates if the source should be removed |
---|
1157 | */ |
---|
1158 | public void copyFromLocalFile(boolean delSrc, Path src, Path dst) |
---|
1159 | throws IOException { |
---|
1160 | copyFromLocalFile(delSrc, true, src, dst); |
---|
1161 | } |
---|
1162 | |
---|
1163 | /** |
---|
1164 | * The src files are on the local disk. Add it to FS at |
---|
1165 | * the given dst name. |
---|
1166 | * delSrc indicates if the source should be removed |
---|
1167 | */ |
---|
1168 | public void copyFromLocalFile(boolean delSrc, boolean overwrite, |
---|
1169 | Path[] srcs, Path dst) |
---|
1170 | throws IOException { |
---|
1171 | Configuration conf = getConf(); |
---|
1172 | FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf); |
---|
1173 | } |
---|
1174 | |
---|
1175 | /** |
---|
1176 | * The src file is on the local disk. Add it to FS at |
---|
1177 | * the given dst name. |
---|
1178 | * delSrc indicates if the source should be removed |
---|
1179 | */ |
---|
1180 | public void copyFromLocalFile(boolean delSrc, boolean overwrite, |
---|
1181 | Path src, Path dst) |
---|
1182 | throws IOException { |
---|
1183 | Configuration conf = getConf(); |
---|
1184 | FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf); |
---|
1185 | } |
---|
1186 | |
---|
1187 | /** |
---|
1188 | * The src file is under FS, and the dst is on the local disk. |
---|
1189 | * Copy it from FS control to the local dst name. |
---|
1190 | */ |
---|
1191 | public void copyToLocalFile(Path src, Path dst) throws IOException { |
---|
1192 | copyToLocalFile(false, src, dst); |
---|
1193 | } |
---|
1194 | |
---|
1195 | /** |
---|
1196 | * The src file is under FS, and the dst is on the local disk. |
---|
1197 | * Copy it from FS control to the local dst name. |
---|
1198 | * Remove the source afterwards |
---|
1199 | */ |
---|
1200 | public void moveToLocalFile(Path src, Path dst) throws IOException { |
---|
1201 | copyToLocalFile(true, src, dst); |
---|
1202 | } |
---|
1203 | |
---|
1204 | /** |
---|
1205 | * The src file is under FS, and the dst is on the local disk. |
---|
1206 | * Copy it from FS control to the local dst name. |
---|
1207 | * delSrc indicates if the src will be removed or not. |
---|
1208 | */ |
---|
1209 | public void copyToLocalFile(boolean delSrc, Path src, Path dst) |
---|
1210 | throws IOException { |
---|
1211 | FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf()); |
---|
1212 | } |
---|
1213 | |
---|
1214 | /** |
---|
1215 | * Returns a local File that the user can write output to. The caller |
---|
1216 | * provides both the eventual FS target name and the local working |
---|
1217 | * file. If the FS is local, we write directly into the target. If |
---|
1218 | * the FS is remote, we write into the tmp local area. |
---|
1219 | */ |
---|
1220 | public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) |
---|
1221 | throws IOException { |
---|
1222 | return tmpLocalFile; |
---|
1223 | } |
---|
1224 | |
---|
1225 | /** |
---|
1226 | * Called when we're all done writing to the target. A local FS will |
---|
1227 | * do nothing, because we've written to exactly the right place. A remote |
---|
1228 | * FS will copy the contents of tmpLocalFile to the correct target at |
---|
1229 | * fsOutputFile. |
---|
1230 | */ |
---|
1231 | public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) |
---|
1232 | throws IOException { |
---|
1233 | moveFromLocalFile(tmpLocalFile, fsOutputFile); |
---|
1234 | } |
---|
1235 | |
---|
1236 | /** |
---|
1237 | * No more filesystem operations are needed. Will |
---|
1238 | * release any held locks. |
---|
1239 | */ |
---|
1240 | public void close() throws IOException { |
---|
1241 | // delete all files that were marked as delete-on-exit. |
---|
1242 | processDeleteOnExit(); |
---|
1243 | CACHE.remove(this.key, this); |
---|
1244 | } |
---|
1245 | |
---|
1246 | /** Return the total size of all files in the filesystem.*/ |
---|
1247 | public long getUsed() throws IOException{ |
---|
1248 | long used = 0; |
---|
1249 | FileStatus[] files = listStatus(new Path("/")); |
---|
1250 | for(FileStatus file:files){ |
---|
1251 | used += file.getLen(); |
---|
1252 | } |
---|
1253 | return used; |
---|
1254 | } |
---|
1255 | |
---|
1256 | /** |
---|
1257 | * Get the block size for a particular file. |
---|
1258 | * @param f the filename |
---|
1259 | * @return the number of bytes in a block |
---|
1260 | */ |
---|
1261 | /** @deprecated Use getFileStatus() instead */ @Deprecated |
---|
1262 | public long getBlockSize(Path f) throws IOException { |
---|
1263 | return getFileStatus(f).getBlockSize(); |
---|
1264 | } |
---|
1265 | |
---|
1266 | /** Return the number of bytes that large input files should be optimally |
---|
1267 | * be split into to minimize i/o time. */ |
---|
1268 | public long getDefaultBlockSize() { |
---|
1269 | // default to 32MB: large enough to minimize the impact of seeks |
---|
1270 | return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); |
---|
1271 | } |
---|
1272 | |
---|
1273 | /** |
---|
1274 | * Get the default replication. |
---|
1275 | */ |
---|
1276 | public short getDefaultReplication() { return 1; } |
---|
1277 | |
---|
1278 | /** |
---|
1279 | * Return a file status object that represents the path. |
---|
1280 | * @param f The path we want information from |
---|
1281 | * @return a FileStatus object |
---|
1282 | * @throws FileNotFoundException when the path does not exist; |
---|
1283 | * IOException see specific implementation |
---|
1284 | */ |
---|
1285 | public abstract FileStatus getFileStatus(Path f) throws IOException; |
---|
1286 | |
---|
1287 | /** |
---|
1288 | * Get the checksum of a file. |
---|
1289 | * |
---|
1290 | * @param f The file path |
---|
1291 | * @return The file checksum. The default return value is null, |
---|
1292 | * which indicates that no checksum algorithm is implemented |
---|
1293 | * in the corresponding FileSystem. |
---|
1294 | */ |
---|
1295 | public FileChecksum getFileChecksum(Path f) throws IOException { |
---|
1296 | return null; |
---|
1297 | } |
---|
1298 | |
---|
1299 | /** |
---|
1300 | * Set the verify checksum flag. This is only applicable if the |
---|
1301 | * corresponding FileSystem supports checksum. By default doesn't do anything. |
---|
1302 | * @param verifyChecksum |
---|
1303 | */ |
---|
1304 | public void setVerifyChecksum(boolean verifyChecksum) { |
---|
1305 | //doesn't do anything |
---|
1306 | } |
---|
1307 | |
---|
1308 | /** |
---|
1309 | * Return a list of file status objects that corresponds to the list of paths |
---|
1310 | * excluding those non-existent paths. |
---|
1311 | * |
---|
1312 | * @param paths |
---|
1313 | * the list of paths we want information from |
---|
1314 | * @return a list of FileStatus objects |
---|
1315 | * @throws IOException |
---|
1316 | * see specific implementation |
---|
1317 | */ |
---|
1318 | private FileStatus[] getFileStatus(Path[] paths) throws IOException { |
---|
1319 | if (paths == null) { |
---|
1320 | return null; |
---|
1321 | } |
---|
1322 | ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length); |
---|
1323 | for (int i = 0; i < paths.length; i++) { |
---|
1324 | try { |
---|
1325 | results.add(getFileStatus(paths[i])); |
---|
1326 | } catch (FileNotFoundException e) { // do nothing |
---|
1327 | } |
---|
1328 | } |
---|
1329 | return results.toArray(new FileStatus[results.size()]); |
---|
1330 | } |
---|
1331 | |
---|
1332 | /** |
---|
1333 | * Set permission of a path. |
---|
1334 | * @param p |
---|
1335 | * @param permission |
---|
1336 | */ |
---|
1337 | public void setPermission(Path p, FsPermission permission |
---|
1338 | ) throws IOException { |
---|
1339 | } |
---|
1340 | |
---|
1341 | /** |
---|
1342 | * Set owner of a path (i.e. a file or a directory). |
---|
1343 | * The parameters username and groupname cannot both be null. |
---|
1344 | * @param p The path |
---|
1345 | * @param username If it is null, the original username remains unchanged. |
---|
1346 | * @param groupname If it is null, the original groupname remains unchanged. |
---|
1347 | */ |
---|
1348 | public void setOwner(Path p, String username, String groupname |
---|
1349 | ) throws IOException { |
---|
1350 | } |
---|
1351 | |
---|
1352 | /** |
---|
1353 | * Set access time of a file |
---|
1354 | * @param p The path |
---|
1355 | * @param mtime Set the modification time of this file. |
---|
1356 | * The number of milliseconds since Jan 1, 1970. |
---|
1357 | * A value of -1 means that this call should not set modification time. |
---|
1358 | * @param atime Set the access time of this file. |
---|
1359 | * The number of milliseconds since Jan 1, 1970. |
---|
1360 | * A value of -1 means that this call should not set access time. |
---|
1361 | */ |
---|
1362 | public void setTimes(Path p, long mtime, long atime |
---|
1363 | ) throws IOException { |
---|
1364 | } |
---|
1365 | |
---|
1366 | private static FileSystem createFileSystem(URI uri, Configuration conf |
---|
1367 | ) throws IOException { |
---|
1368 | Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null); |
---|
1369 | if (clazz == null) { |
---|
1370 | throw new IOException("No FileSystem for scheme: " + uri.getScheme()); |
---|
1371 | } |
---|
1372 | FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf); |
---|
1373 | fs.initialize(uri, conf); |
---|
1374 | return fs; |
---|
1375 | } |
---|
1376 | |
---|
1377 | /** Caching FileSystem objects */ |
---|
1378 | static class Cache { |
---|
1379 | private final Map<Key, FileSystem> map = new HashMap<Key, FileSystem>(); |
---|
1380 | |
---|
1381 | synchronized FileSystem get(URI uri, Configuration conf) throws IOException{ |
---|
1382 | Key key = new Key(uri, conf); |
---|
1383 | FileSystem fs = map.get(key); |
---|
1384 | if (fs == null) { |
---|
1385 | fs = createFileSystem(uri, conf); |
---|
1386 | if (map.isEmpty() && !clientFinalizer.isAlive()) { |
---|
1387 | Runtime.getRuntime().addShutdownHook(clientFinalizer); |
---|
1388 | } |
---|
1389 | fs.key = key; |
---|
1390 | map.put(key, fs); |
---|
1391 | } |
---|
1392 | return fs; |
---|
1393 | } |
---|
1394 | |
---|
1395 | synchronized void remove(Key key, FileSystem fs) { |
---|
1396 | if (map.containsKey(key) && fs == map.get(key)) { |
---|
1397 | map.remove(key); |
---|
1398 | if (map.isEmpty() && !clientFinalizer.isAlive()) { |
---|
1399 | if (!Runtime.getRuntime().removeShutdownHook(clientFinalizer)) { |
---|
1400 | LOG.info("Could not cancel cleanup thread, though no " + |
---|
1401 | "FileSystems are open"); |
---|
1402 | } |
---|
1403 | } |
---|
1404 | } |
---|
1405 | } |
---|
1406 | |
---|
1407 | synchronized void closeAll() throws IOException { |
---|
1408 | List<IOException> exceptions = new ArrayList<IOException>(); |
---|
1409 | for(; !map.isEmpty(); ) { |
---|
1410 | Map.Entry<Key, FileSystem> e = map.entrySet().iterator().next(); |
---|
1411 | final Key key = e.getKey(); |
---|
1412 | final FileSystem fs = e.getValue(); |
---|
1413 | |
---|
1414 | //remove from cache |
---|
1415 | remove(key, fs); |
---|
1416 | |
---|
1417 | if (fs != null) { |
---|
1418 | try { |
---|
1419 | fs.close(); |
---|
1420 | } |
---|
1421 | catch(IOException ioe) { |
---|
1422 | exceptions.add(ioe); |
---|
1423 | } |
---|
1424 | } |
---|
1425 | } |
---|
1426 | |
---|
1427 | if (!exceptions.isEmpty()) { |
---|
1428 | throw MultipleIOException.createIOException(exceptions); |
---|
1429 | } |
---|
1430 | } |
---|
1431 | |
---|
1432 | /** FileSystem.Cache.Key */ |
---|
1433 | static class Key { |
---|
1434 | final String scheme; |
---|
1435 | final String authority; |
---|
1436 | final String username; |
---|
1437 | |
---|
1438 | Key(URI uri, Configuration conf) throws IOException { |
---|
1439 | scheme = uri.getScheme()==null?"":uri.getScheme().toLowerCase(); |
---|
1440 | authority = uri.getAuthority()==null?"":uri.getAuthority().toLowerCase(); |
---|
1441 | UserGroupInformation ugi = UserGroupInformation.readFrom(conf); |
---|
1442 | if (ugi == null) { |
---|
1443 | try { |
---|
1444 | ugi = UserGroupInformation.login(conf); |
---|
1445 | } catch(LoginException e) { |
---|
1446 | LOG.warn("uri=" + uri, e); |
---|
1447 | } |
---|
1448 | } |
---|
1449 | username = ugi == null? null: ugi.getUserName(); |
---|
1450 | } |
---|
1451 | |
---|
1452 | /** {@inheritDoc} */ |
---|
1453 | public int hashCode() { |
---|
1454 | return (scheme + authority + username).hashCode(); |
---|
1455 | } |
---|
1456 | |
---|
1457 | static boolean isEqual(Object a, Object b) { |
---|
1458 | return a == b || (a != null && a.equals(b)); |
---|
1459 | } |
---|
1460 | |
---|
1461 | /** {@inheritDoc} */ |
---|
1462 | public boolean equals(Object obj) { |
---|
1463 | if (obj == this) { |
---|
1464 | return true; |
---|
1465 | } |
---|
1466 | if (obj != null && obj instanceof Key) { |
---|
1467 | Key that = (Key)obj; |
---|
1468 | return isEqual(this.scheme, that.scheme) |
---|
1469 | && isEqual(this.authority, that.authority) |
---|
1470 | && isEqual(this.username, that.username); |
---|
1471 | } |
---|
1472 | return false; |
---|
1473 | } |
---|
1474 | |
---|
1475 | /** {@inheritDoc} */ |
---|
1476 | public String toString() { |
---|
1477 | return username + "@" + scheme + "://" + authority; |
---|
1478 | } |
---|
1479 | } |
---|
1480 | } |
---|
1481 | |
---|
1482 | public static final class Statistics { |
---|
1483 | private final String scheme; |
---|
1484 | private AtomicLong bytesRead = new AtomicLong(); |
---|
1485 | private AtomicLong bytesWritten = new AtomicLong(); |
---|
1486 | |
---|
1487 | public Statistics(String scheme) { |
---|
1488 | this.scheme = scheme; |
---|
1489 | } |
---|
1490 | |
---|
1491 | /** |
---|
1492 | * Increment the bytes read in the statistics |
---|
1493 | * @param newBytes the additional bytes read |
---|
1494 | */ |
---|
1495 | public void incrementBytesRead(long newBytes) { |
---|
1496 | bytesRead.getAndAdd(newBytes); |
---|
1497 | } |
---|
1498 | |
---|
1499 | /** |
---|
1500 | * Increment the bytes written in the statistics |
---|
1501 | * @param newBytes the additional bytes written |
---|
1502 | */ |
---|
1503 | public void incrementBytesWritten(long newBytes) { |
---|
1504 | bytesWritten.getAndAdd(newBytes); |
---|
1505 | } |
---|
1506 | |
---|
1507 | /** |
---|
1508 | * Get the total number of bytes read |
---|
1509 | * @return the number of bytes |
---|
1510 | */ |
---|
1511 | public long getBytesRead() { |
---|
1512 | return bytesRead.get(); |
---|
1513 | } |
---|
1514 | |
---|
1515 | /** |
---|
1516 | * Get the total number of bytes written |
---|
1517 | * @return the number of bytes |
---|
1518 | */ |
---|
1519 | public long getBytesWritten() { |
---|
1520 | return bytesWritten.get(); |
---|
1521 | } |
---|
1522 | |
---|
1523 | public String toString() { |
---|
1524 | return bytesRead + " bytes read and " + bytesWritten + |
---|
1525 | " bytes written"; |
---|
1526 | } |
---|
1527 | |
---|
1528 | /** |
---|
1529 | * Reset the counts of bytes to 0. |
---|
1530 | */ |
---|
1531 | public void reset() { |
---|
1532 | bytesWritten.set(0); |
---|
1533 | bytesRead.set(0); |
---|
1534 | } |
---|
1535 | |
---|
1536 | /** |
---|
1537 | * Get the uri scheme associated with this statistics object. |
---|
1538 | * @return the schema associated with this set of statistics |
---|
1539 | */ |
---|
1540 | public String getScheme() { |
---|
1541 | return scheme; |
---|
1542 | } |
---|
1543 | } |
---|
1544 | |
---|
1545 | /** |
---|
1546 | * Get the Map of Statistics object indexed by URI Scheme. |
---|
1547 | * @return a Map having a key as URI scheme and value as Statistics object |
---|
1548 | * @deprecated use {@link #getAllStatistics} instead |
---|
1549 | */ |
---|
1550 | public static synchronized Map<String, Statistics> getStatistics() { |
---|
1551 | Map<String, Statistics> result = new HashMap<String, Statistics>(); |
---|
1552 | for(Statistics stat: statisticsTable.values()) { |
---|
1553 | result.put(stat.getScheme(), stat); |
---|
1554 | } |
---|
1555 | return result; |
---|
1556 | } |
---|
1557 | |
---|
1558 | /** |
---|
1559 | * Return the FileSystem classes that have Statistics |
---|
1560 | */ |
---|
1561 | public static synchronized List<Statistics> getAllStatistics() { |
---|
1562 | return new ArrayList<Statistics>(statisticsTable.values()); |
---|
1563 | } |
---|
1564 | |
---|
1565 | /** |
---|
1566 | * Get the statistics for a particular file system |
---|
1567 | * @param cls the class to lookup |
---|
1568 | * @return a statistics object |
---|
1569 | */ |
---|
1570 | public static synchronized |
---|
1571 | Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { |
---|
1572 | Statistics result = statisticsTable.get(cls); |
---|
1573 | if (result == null) { |
---|
1574 | result = new Statistics(scheme); |
---|
1575 | statisticsTable.put(cls, result); |
---|
1576 | } |
---|
1577 | return result; |
---|
1578 | } |
---|
1579 | |
---|
1580 | public static synchronized void clearStatistics() { |
---|
1581 | for(Statistics stat: statisticsTable.values()) { |
---|
1582 | stat.reset(); |
---|
1583 | } |
---|
1584 | } |
---|
1585 | |
---|
1586 | public static synchronized |
---|
1587 | void printStatistics() throws IOException { |
---|
1588 | for (Map.Entry<Class<? extends FileSystem>, Statistics> pair: |
---|
1589 | statisticsTable.entrySet()) { |
---|
1590 | System.out.println(" FileSystem " + pair.getKey().getName() + |
---|
1591 | ": " + pair.getValue()); |
---|
1592 | } |
---|
1593 | } |
---|
1594 | } |
---|