1 | /** |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | package org.apache.hadoop.hdfs.server.namenode; |
---|
19 | |
---|
20 | import junit.framework.TestCase; |
---|
21 | import java.io.*; |
---|
22 | import java.util.Collection; |
---|
23 | import java.util.Iterator; |
---|
24 | import java.util.Random; |
---|
25 | import org.apache.hadoop.conf.Configuration; |
---|
26 | import org.apache.hadoop.fs.FileSystem; |
---|
27 | import org.apache.hadoop.fs.permission.*; |
---|
28 | |
---|
29 | import org.apache.hadoop.hdfs.MiniDFSCluster; |
---|
30 | import org.apache.hadoop.io.ArrayWritable; |
---|
31 | import org.apache.hadoop.io.UTF8; |
---|
32 | import org.apache.hadoop.io.Writable; |
---|
33 | import org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream; |
---|
34 | import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; |
---|
35 | import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType; |
---|
36 | import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile; |
---|
37 | |
---|
38 | /** |
---|
39 | * This class tests the creation and validation of a checkpoint. |
---|
40 | */ |
---|
41 | public class TestEditLog extends TestCase { |
---|
42 | static final int numDatanodes = 1; |
---|
43 | |
---|
44 | // This test creates numThreads threads and each thread does |
---|
45 | // 2 * numberTransactions Transactions concurrently. |
---|
46 | int numberTransactions = 100; |
---|
47 | int numThreads = 100; |
---|
48 | |
---|
49 | // |
---|
50 | // an object that does a bunch of transactions |
---|
51 | // |
---|
52 | static class Transactions implements Runnable { |
---|
53 | FSEditLog editLog; |
---|
54 | int numTransactions; |
---|
55 | short replication = 3; |
---|
56 | long blockSize = 64; |
---|
57 | |
---|
58 | Transactions(FSEditLog editlog, int num) { |
---|
59 | editLog = editlog; |
---|
60 | numTransactions = num; |
---|
61 | } |
---|
62 | |
---|
63 | // add a bunch of transactions. |
---|
64 | public void run() { |
---|
65 | PermissionStatus p = FSNamesystem.getFSNamesystem( |
---|
66 | ).createFsOwnerPermissions(new FsPermission((short)0777)); |
---|
67 | |
---|
68 | for (int i = 0; i < numTransactions; i++) { |
---|
69 | try { |
---|
70 | INodeFileUnderConstruction inode = new INodeFileUnderConstruction( |
---|
71 | p, replication, blockSize, 0, "", "", null); |
---|
72 | editLog.logOpenFile("/filename" + i, inode); |
---|
73 | editLog.logCloseFile("/filename" + i, inode); |
---|
74 | editLog.logSync(); |
---|
75 | } catch (IOException e) { |
---|
76 | System.out.println("Transaction " + i + " encountered exception " + |
---|
77 | e); |
---|
78 | } |
---|
79 | } |
---|
80 | } |
---|
81 | } |
---|
82 | |
---|
83 | /** |
---|
84 | * Tests transaction logging in dfs. |
---|
85 | */ |
---|
86 | public void testEditLog() throws IOException { |
---|
87 | |
---|
88 | // start a cluster |
---|
89 | |
---|
90 | Collection<File> namedirs = null; |
---|
91 | Collection<File> editsdirs = null; |
---|
92 | Configuration conf = new Configuration(); |
---|
93 | MiniDFSCluster cluster = new MiniDFSCluster(0, conf, numDatanodes, |
---|
94 | true, true, null, null); |
---|
95 | cluster.waitActive(); |
---|
96 | FileSystem fileSys = cluster.getFileSystem(); |
---|
97 | int numdirs = 0; |
---|
98 | |
---|
99 | try { |
---|
100 | namedirs = cluster.getNameDirs(); |
---|
101 | editsdirs = cluster.getNameEditsDirs(); |
---|
102 | } finally { |
---|
103 | fileSys.close(); |
---|
104 | cluster.shutdown(); |
---|
105 | } |
---|
106 | |
---|
107 | for (Iterator it = namedirs.iterator(); it.hasNext(); ) { |
---|
108 | File dir = (File)it.next(); |
---|
109 | System.out.println(dir); |
---|
110 | numdirs++; |
---|
111 | } |
---|
112 | |
---|
113 | FSImage fsimage = new FSImage(namedirs, editsdirs); |
---|
114 | FSEditLog editLog = fsimage.getEditLog(); |
---|
115 | |
---|
116 | // set small size of flush buffer |
---|
117 | editLog.setBufferCapacity(2048); |
---|
118 | editLog.close(); |
---|
119 | editLog.open(); |
---|
120 | |
---|
121 | // Create threads and make them run transactions concurrently. |
---|
122 | Thread threadId[] = new Thread[numThreads]; |
---|
123 | for (int i = 0; i < numThreads; i++) { |
---|
124 | Transactions trans = new Transactions(editLog, numberTransactions); |
---|
125 | threadId[i] = new Thread(trans, "TransactionThread-" + i); |
---|
126 | threadId[i].start(); |
---|
127 | } |
---|
128 | |
---|
129 | // wait for all transactions to get over |
---|
130 | for (int i = 0; i < numThreads; i++) { |
---|
131 | try { |
---|
132 | threadId[i].join(); |
---|
133 | } catch (InterruptedException e) { |
---|
134 | i--; // retry |
---|
135 | } |
---|
136 | } |
---|
137 | |
---|
138 | editLog.close(); |
---|
139 | |
---|
140 | // Verify that we can read in all the transactions that we have written. |
---|
141 | // If there were any corruptions, it is likely that the reading in |
---|
142 | // of these transactions will throw an exception. |
---|
143 | // |
---|
144 | for (Iterator<StorageDirectory> it = |
---|
145 | fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) { |
---|
146 | File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS); |
---|
147 | System.out.println("Verifying file: " + editFile); |
---|
148 | int numEdits = FSEditLog.loadFSEdits(new EditLogFileInputStream(editFile)); |
---|
149 | int numLeases = FSNamesystem.getFSNamesystem().leaseManager.countLease(); |
---|
150 | System.out.println("Number of outstanding leases " + numLeases); |
---|
151 | assertEquals(0, numLeases); |
---|
152 | assertTrue("Verification for " + editFile + " failed. " + |
---|
153 | "Expected " + (numThreads * 2 * numberTransactions) + " transactions. "+ |
---|
154 | "Found " + numEdits + " transactions.", |
---|
155 | numEdits == numThreads * 2 * numberTransactions); |
---|
156 | |
---|
157 | } |
---|
158 | } |
---|
159 | } |
---|