[120] | 1 | /** |
---|
| 2 | * Licensed to the Apache Software Foundation (ASF) under one or more |
---|
| 3 | * contributor license agreements. See the NOTICE file distributed with this |
---|
| 4 | * work for additional information regarding copyright ownership. The ASF |
---|
| 5 | * licenses this file to you under the Apache License, Version 2.0 (the |
---|
| 6 | * "License"); you may not use this file except in compliance with the License. |
---|
| 7 | * You may obtain a copy of the License at |
---|
| 8 | * |
---|
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
| 10 | * |
---|
| 11 | * Unless required by applicable law or agreed to in writing, software |
---|
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
---|
| 13 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
---|
| 14 | * License for the specific language governing permissions and limitations under |
---|
| 15 | * the License. |
---|
| 16 | */ |
---|
| 17 | |
---|
| 18 | package org.apache.hadoop.io.file.tfile; |
---|
| 19 | |
---|
| 20 | import java.io.IOException; |
---|
| 21 | |
---|
| 22 | import junit.framework.Assert; |
---|
| 23 | import junit.framework.TestCase; |
---|
| 24 | |
---|
| 25 | import org.apache.hadoop.conf.Configuration; |
---|
| 26 | import org.apache.hadoop.fs.FSDataOutputStream; |
---|
| 27 | import org.apache.hadoop.fs.FileSystem; |
---|
| 28 | import org.apache.hadoop.fs.Path; |
---|
| 29 | import org.apache.hadoop.io.file.tfile.TFile.Reader; |
---|
| 30 | import org.apache.hadoop.io.file.tfile.TFile.Writer; |
---|
| 31 | import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner; |
---|
| 32 | |
---|
| 33 | public class TestTFileUnsortedByteArrays extends TestCase { |
---|
| 34 | private static String ROOT = |
---|
| 35 | System.getProperty("test.build.data", "/tmp/tfile-test"); |
---|
| 36 | |
---|
| 37 | |
---|
| 38 | private final static int BLOCK_SIZE = 512; |
---|
| 39 | private final static int BUF_SIZE = 64; |
---|
| 40 | |
---|
| 41 | private FileSystem fs; |
---|
| 42 | private Configuration conf; |
---|
| 43 | private Path path; |
---|
| 44 | private FSDataOutputStream out; |
---|
| 45 | private Writer writer; |
---|
| 46 | |
---|
| 47 | private String compression = Compression.Algorithm.GZ.getName(); |
---|
| 48 | private String outputFile = "TFileTestUnsorted"; |
---|
| 49 | /* |
---|
| 50 | * pre-sampled numbers of records in one block, based on the given the |
---|
| 51 | * generated key and value strings |
---|
| 52 | */ |
---|
| 53 | private int records1stBlock = 4314; |
---|
| 54 | private int records2ndBlock = 4108; |
---|
| 55 | |
---|
| 56 | public void init(String compression, String outputFile, |
---|
| 57 | int numRecords1stBlock, int numRecords2ndBlock) { |
---|
| 58 | this.compression = compression; |
---|
| 59 | this.outputFile = outputFile; |
---|
| 60 | this.records1stBlock = numRecords1stBlock; |
---|
| 61 | this.records2ndBlock = numRecords2ndBlock; |
---|
| 62 | } |
---|
| 63 | |
---|
| 64 | @Override |
---|
| 65 | public void setUp() throws IOException { |
---|
| 66 | conf = new Configuration(); |
---|
| 67 | path = new Path(ROOT, outputFile); |
---|
| 68 | fs = path.getFileSystem(conf); |
---|
| 69 | out = fs.create(path); |
---|
| 70 | writer = new Writer(out, BLOCK_SIZE, compression, null, conf); |
---|
| 71 | writer.append("keyZ".getBytes(), "valueZ".getBytes()); |
---|
| 72 | writer.append("keyM".getBytes(), "valueM".getBytes()); |
---|
| 73 | writer.append("keyN".getBytes(), "valueN".getBytes()); |
---|
| 74 | writer.append("keyA".getBytes(), "valueA".getBytes()); |
---|
| 75 | closeOutput(); |
---|
| 76 | } |
---|
| 77 | |
---|
| 78 | @Override |
---|
| 79 | public void tearDown() throws IOException { |
---|
| 80 | fs.delete(path, true); |
---|
| 81 | } |
---|
| 82 | |
---|
| 83 | // we still can scan records in an unsorted TFile |
---|
| 84 | public void testFailureScannerWithKeys() throws IOException { |
---|
| 85 | Reader reader = |
---|
| 86 | new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); |
---|
| 87 | Assert.assertFalse(reader.isSorted()); |
---|
| 88 | Assert.assertEquals((int) reader.getEntryCount(), 4); |
---|
| 89 | |
---|
| 90 | try { |
---|
| 91 | Scanner scanner = |
---|
| 92 | reader.createScanner("aaa".getBytes(), "zzz".getBytes()); |
---|
| 93 | Assert |
---|
| 94 | .fail("Failed to catch creating scanner with keys on unsorted file."); |
---|
| 95 | } |
---|
| 96 | catch (RuntimeException e) { |
---|
| 97 | } |
---|
| 98 | finally { |
---|
| 99 | reader.close(); |
---|
| 100 | } |
---|
| 101 | } |
---|
| 102 | |
---|
| 103 | // we still can scan records in an unsorted TFile |
---|
| 104 | public void testScan() throws IOException { |
---|
| 105 | Reader reader = |
---|
| 106 | new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); |
---|
| 107 | Assert.assertFalse(reader.isSorted()); |
---|
| 108 | Assert.assertEquals((int) reader.getEntryCount(), 4); |
---|
| 109 | |
---|
| 110 | Scanner scanner = reader.createScanner(); |
---|
| 111 | |
---|
| 112 | try { |
---|
| 113 | |
---|
| 114 | // read key and value |
---|
| 115 | byte[] kbuf = new byte[BUF_SIZE]; |
---|
| 116 | int klen = scanner.entry().getKeyLength(); |
---|
| 117 | scanner.entry().getKey(kbuf); |
---|
| 118 | Assert.assertEquals(new String(kbuf, 0, klen), "keyZ"); |
---|
| 119 | |
---|
| 120 | byte[] vbuf = new byte[BUF_SIZE]; |
---|
| 121 | int vlen = scanner.entry().getValueLength(); |
---|
| 122 | scanner.entry().getValue(vbuf); |
---|
| 123 | Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ"); |
---|
| 124 | |
---|
| 125 | scanner.advance(); |
---|
| 126 | |
---|
| 127 | // now try get value first |
---|
| 128 | vbuf = new byte[BUF_SIZE]; |
---|
| 129 | vlen = scanner.entry().getValueLength(); |
---|
| 130 | scanner.entry().getValue(vbuf); |
---|
| 131 | Assert.assertEquals(new String(vbuf, 0, vlen), "valueM"); |
---|
| 132 | |
---|
| 133 | kbuf = new byte[BUF_SIZE]; |
---|
| 134 | klen = scanner.entry().getKeyLength(); |
---|
| 135 | scanner.entry().getKey(kbuf); |
---|
| 136 | Assert.assertEquals(new String(kbuf, 0, klen), "keyM"); |
---|
| 137 | } |
---|
| 138 | finally { |
---|
| 139 | scanner.close(); |
---|
| 140 | reader.close(); |
---|
| 141 | } |
---|
| 142 | } |
---|
| 143 | |
---|
| 144 | // we still can scan records in an unsorted TFile |
---|
| 145 | public void testScanRange() throws IOException { |
---|
| 146 | Reader reader = |
---|
| 147 | new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); |
---|
| 148 | Assert.assertFalse(reader.isSorted()); |
---|
| 149 | Assert.assertEquals((int) reader.getEntryCount(), 4); |
---|
| 150 | |
---|
| 151 | Scanner scanner = reader.createScanner(); |
---|
| 152 | |
---|
| 153 | try { |
---|
| 154 | |
---|
| 155 | // read key and value |
---|
| 156 | byte[] kbuf = new byte[BUF_SIZE]; |
---|
| 157 | int klen = scanner.entry().getKeyLength(); |
---|
| 158 | scanner.entry().getKey(kbuf); |
---|
| 159 | Assert.assertEquals(new String(kbuf, 0, klen), "keyZ"); |
---|
| 160 | |
---|
| 161 | byte[] vbuf = new byte[BUF_SIZE]; |
---|
| 162 | int vlen = scanner.entry().getValueLength(); |
---|
| 163 | scanner.entry().getValue(vbuf); |
---|
| 164 | Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ"); |
---|
| 165 | |
---|
| 166 | scanner.advance(); |
---|
| 167 | |
---|
| 168 | // now try get value first |
---|
| 169 | vbuf = new byte[BUF_SIZE]; |
---|
| 170 | vlen = scanner.entry().getValueLength(); |
---|
| 171 | scanner.entry().getValue(vbuf); |
---|
| 172 | Assert.assertEquals(new String(vbuf, 0, vlen), "valueM"); |
---|
| 173 | |
---|
| 174 | kbuf = new byte[BUF_SIZE]; |
---|
| 175 | klen = scanner.entry().getKeyLength(); |
---|
| 176 | scanner.entry().getKey(kbuf); |
---|
| 177 | Assert.assertEquals(new String(kbuf, 0, klen), "keyM"); |
---|
| 178 | } |
---|
| 179 | finally { |
---|
| 180 | scanner.close(); |
---|
| 181 | reader.close(); |
---|
| 182 | } |
---|
| 183 | } |
---|
| 184 | |
---|
| 185 | public void testFailureSeek() throws IOException { |
---|
| 186 | Reader reader = |
---|
| 187 | new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf); |
---|
| 188 | Scanner scanner = reader.createScanner(); |
---|
| 189 | |
---|
| 190 | try { |
---|
| 191 | // can't find ceil |
---|
| 192 | try { |
---|
| 193 | scanner.lowerBound("keyN".getBytes()); |
---|
| 194 | Assert.fail("Cannot search in a unsorted TFile!"); |
---|
| 195 | } |
---|
| 196 | catch (Exception e) { |
---|
| 197 | // noop, expecting excetions |
---|
| 198 | } |
---|
| 199 | finally { |
---|
| 200 | } |
---|
| 201 | |
---|
| 202 | // can't find higher |
---|
| 203 | try { |
---|
| 204 | scanner.upperBound("keyA".getBytes()); |
---|
| 205 | Assert.fail("Cannot search higher in a unsorted TFile!"); |
---|
| 206 | } |
---|
| 207 | catch (Exception e) { |
---|
| 208 | // noop, expecting excetions |
---|
| 209 | } |
---|
| 210 | finally { |
---|
| 211 | } |
---|
| 212 | |
---|
| 213 | // can't seek |
---|
| 214 | try { |
---|
| 215 | scanner.seekTo("keyM".getBytes()); |
---|
| 216 | Assert.fail("Cannot search a unsorted TFile!"); |
---|
| 217 | } |
---|
| 218 | catch (Exception e) { |
---|
| 219 | // noop, expecting excetions |
---|
| 220 | } |
---|
| 221 | finally { |
---|
| 222 | } |
---|
| 223 | } |
---|
| 224 | finally { |
---|
| 225 | scanner.close(); |
---|
| 226 | reader.close(); |
---|
| 227 | } |
---|
| 228 | } |
---|
| 229 | |
---|
| 230 | private void closeOutput() throws IOException { |
---|
| 231 | if (writer != null) { |
---|
| 232 | writer.close(); |
---|
| 233 | writer = null; |
---|
| 234 | out.close(); |
---|
| 235 | out = null; |
---|
| 236 | } |
---|
| 237 | } |
---|
| 238 | } |
---|