1 | /* |
---|
2 | * Licensed to the Apache Software Foundation (ASF) under one |
---|
3 | * or more contributor license agreements. See the NOTICE file |
---|
4 | * distributed with this work for additional information |
---|
5 | * regarding copyright ownership. The ASF licenses this file |
---|
6 | * to you under the Apache License, Version 2.0 (the |
---|
7 | * "License"); you may not use this file except in compliance |
---|
8 | * with the License. You may obtain a copy of the License at |
---|
9 | * |
---|
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
---|
11 | * |
---|
12 | * Unless required by applicable law or agreed to in writing, software |
---|
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
---|
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
---|
15 | * See the License for the specific language governing permissions and |
---|
16 | * limitations under the License. |
---|
17 | */ |
---|
18 | |
---|
19 | package org.apache.hadoop.io.compress; |
---|
20 | |
---|
21 | import java.io.IOException; |
---|
22 | import java.io.OutputStream; |
---|
23 | |
---|
24 | /** |
---|
25 | * A {@link org.apache.hadoop.io.compress.CompressorStream} which works |
---|
26 | * with 'block-based' based compression algorithms, as opposed to |
---|
27 | * 'stream-based' compression algorithms. |
---|
28 | * |
---|
29 | * It should be noted that this wrapper does not guarantee that blocks will |
---|
30 | * be sized for the compressor. If the |
---|
31 | * {@link org.apache.hadoop.io.compress.Compressor} requires buffering to |
---|
32 | * effect meaningful compression, it is responsible for it. |
---|
33 | */ |
---|
34 | public class BlockCompressorStream extends CompressorStream { |
---|
35 | |
---|
36 | // The 'maximum' size of input data to be compressed, to account |
---|
37 | // for the overhead of the compression algorithm. |
---|
38 | private final int MAX_INPUT_SIZE; |
---|
39 | |
---|
40 | /** |
---|
41 | * Create a {@link BlockCompressorStream}. |
---|
42 | * |
---|
43 | * @param out stream |
---|
44 | * @param compressor compressor to be used |
---|
45 | * @param bufferSize size of buffer |
---|
46 | * @param compressionOverhead maximum 'overhead' of the compression |
---|
47 | * algorithm with given bufferSize |
---|
48 | */ |
---|
49 | public BlockCompressorStream(OutputStream out, Compressor compressor, |
---|
50 | int bufferSize, int compressionOverhead) { |
---|
51 | super(out, compressor, bufferSize); |
---|
52 | MAX_INPUT_SIZE = bufferSize - compressionOverhead; |
---|
53 | } |
---|
54 | |
---|
55 | /** |
---|
56 | * Create a {@link BlockCompressorStream} with given output-stream and |
---|
57 | * compressor. |
---|
58 | * Use default of 512 as bufferSize and compressionOverhead of |
---|
59 | * (1% of bufferSize + 12 bytes) = 18 bytes (zlib algorithm). |
---|
60 | * |
---|
61 | * @param out stream |
---|
62 | * @param compressor compressor to be used |
---|
63 | */ |
---|
64 | public BlockCompressorStream(OutputStream out, Compressor compressor) { |
---|
65 | this(out, compressor, 512, 18); |
---|
66 | } |
---|
67 | |
---|
68 | /** |
---|
69 | * Write the data provided to the compression codec, compressing no more |
---|
70 | * than the buffer size less the compression overhead as specified during |
---|
71 | * construction for each block. |
---|
72 | * |
---|
73 | * Each block contains the uncompressed length for the block, followed by |
---|
74 | * one or more length-prefixed blocks of compressed data. |
---|
75 | */ |
---|
76 | public void write(byte[] b, int off, int len) throws IOException { |
---|
77 | // Sanity checks |
---|
78 | if (compressor.finished()) { |
---|
79 | throw new IOException("write beyond end of stream"); |
---|
80 | } |
---|
81 | if (b == null) { |
---|
82 | throw new NullPointerException(); |
---|
83 | } else if ((off < 0) || (off > b.length) || (len < 0) || |
---|
84 | ((off + len) > b.length)) { |
---|
85 | throw new IndexOutOfBoundsException(); |
---|
86 | } else if (len == 0) { |
---|
87 | return; |
---|
88 | } |
---|
89 | |
---|
90 | long limlen = compressor.getBytesRead(); |
---|
91 | if (len + limlen > MAX_INPUT_SIZE && limlen > 0) { |
---|
92 | // Adding this segment would exceed the maximum size. |
---|
93 | // Flush data if we have it. |
---|
94 | finish(); |
---|
95 | compressor.reset(); |
---|
96 | } |
---|
97 | |
---|
98 | if (len > MAX_INPUT_SIZE) { |
---|
99 | // The data we're given exceeds the maximum size. Any data |
---|
100 | // we had have been flushed, so we write out this chunk in segments |
---|
101 | // not exceeding the maximum size until it is exhausted. |
---|
102 | rawWriteInt(len); |
---|
103 | do { |
---|
104 | int bufLen = Math.min(len, MAX_INPUT_SIZE); |
---|
105 | |
---|
106 | compressor.setInput(b, off, bufLen); |
---|
107 | compressor.finish(); |
---|
108 | while (!compressor.finished()) { |
---|
109 | compress(); |
---|
110 | } |
---|
111 | compressor.reset(); |
---|
112 | off += bufLen; |
---|
113 | len -= bufLen; |
---|
114 | } while (len > 0); |
---|
115 | return; |
---|
116 | } |
---|
117 | |
---|
118 | // Give data to the compressor |
---|
119 | compressor.setInput(b, off, len); |
---|
120 | if (!compressor.needsInput()) { |
---|
121 | // compressor buffer size might be smaller than the maximum |
---|
122 | // size, so we permit it to flush if required. |
---|
123 | rawWriteInt((int)compressor.getBytesRead()); |
---|
124 | do { |
---|
125 | compress(); |
---|
126 | } while (!compressor.needsInput()); |
---|
127 | } |
---|
128 | } |
---|
129 | |
---|
130 | public void finish() throws IOException { |
---|
131 | if (!compressor.finished()) { |
---|
132 | rawWriteInt((int)compressor.getBytesRead()); |
---|
133 | compressor.finish(); |
---|
134 | while (!compressor.finished()) { |
---|
135 | compress(); |
---|
136 | } |
---|
137 | } |
---|
138 | } |
---|
139 | |
---|
140 | protected void compress() throws IOException { |
---|
141 | int len = compressor.compress(buffer, 0, buffer.length); |
---|
142 | if (len > 0) { |
---|
143 | // Write out the compressed chunk |
---|
144 | rawWriteInt(len); |
---|
145 | out.write(buffer, 0, len); |
---|
146 | } |
---|
147 | } |
---|
148 | |
---|
149 | private void rawWriteInt(int v) throws IOException { |
---|
150 | out.write((v >>> 24) & 0xFF); |
---|
151 | out.write((v >>> 16) & 0xFF); |
---|
152 | out.write((v >>> 8) & 0xFF); |
---|
153 | out.write((v >>> 0) & 0xFF); |
---|
154 | } |
---|
155 | |
---|
156 | } |
---|