source: proiecte/HadoopJUnit/hadoop-0.20.1/src/test/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java @ 120

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with this
 * work for additional information regarding copyright ownership. The ASF
 * licenses this file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package org.apache.hadoop.io.file.tfile;

import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Random;
import java.util.StringTokenizer;

import junit.framework.TestCase;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner.Entry;

public class TestTFileSeqFileComparison extends TestCase {
  MyOptions options;

  private FileSystem fs;
  private Configuration conf;
  private long startTimeEpoch;
  private long finishTimeEpoch;
  private DateFormat formatter;
  byte[][] dictionary;

  @Override
  public void setUp() throws IOException {
    if (options == null) {
      options = new MyOptions(new String[0]);
    }

    conf = new Configuration();
    conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSize);
    conf.setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSize);
    Path path = new Path(options.rootDir);
    fs = path.getFileSystem(conf);
    formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    setUpDictionary();
  }

  private void setUpDictionary() {
    Random rng = new Random();
    dictionary = new byte[options.dictSize][];
    for (int i = 0; i < options.dictSize; ++i) {
      int len =
          rng.nextInt(options.maxWordLen - options.minWordLen)
              + options.minWordLen;
      dictionary[i] = new byte[len];
      rng.nextBytes(dictionary[i]);
    }
  }

  @Override
  public void tearDown() throws IOException {
    // do nothing
  }

  public void startTime() throws IOException {
    startTimeEpoch = System.currentTimeMillis();
    System.out.println(formatTime() + " Started timing.");
  }

  public void stopTime() throws IOException {
    finishTimeEpoch = System.currentTimeMillis();
    System.out.println(formatTime() + " Stopped timing.");
  }

  public long getIntervalMillis() throws IOException {
    return finishTimeEpoch - startTimeEpoch;
  }

  public void printlnWithTimestamp(String message) throws IOException {
    System.out.println(formatTime() + "  " + message);
  }

  /*
   * Format millis into minutes and seconds.
   */
  public String formatTime(long milis) {
    return formatter.format(milis);
  }

  public String formatTime() {
    return formatTime(System.currentTimeMillis());
  }

  private interface KVAppendable {
    public void append(BytesWritable key, BytesWritable value)
        throws IOException;

    public void close() throws IOException;
  }

  private interface KVReadable {
    public byte[] getKey();

    public byte[] getValue();

    public int getKeyLength();

    public int getValueLength();

    public boolean next() throws IOException;

    public void close() throws IOException;
  }

  static class TFileAppendable implements KVAppendable {
    private FSDataOutputStream fsdos;
    private TFile.Writer writer;

    public TFileAppendable(FileSystem fs, Path path, String compress,
        int minBlkSize, int osBufferSize, Configuration conf)
        throws IOException {
      this.fsdos = fs.create(path, true, osBufferSize);
      this.writer = new TFile.Writer(fsdos, minBlkSize, compress, null, conf);
    }

    public void append(BytesWritable key, BytesWritable value)
        throws IOException {
      writer.append(key.get(), 0, key.getSize(), value.get(), 0, value
          .getSize());
    }

    public void close() throws IOException {
      writer.close();
      fsdos.close();
    }
  }

  static class TFileReadable implements KVReadable {
    private FSDataInputStream fsdis;
    private TFile.Reader reader;
    private TFile.Reader.Scanner scanner;
    private byte[] keyBuffer;
    private int keyLength;
    private byte[] valueBuffer;
    private int valueLength;

    public TFileReadable(FileSystem fs, Path path, int osBufferSize,
        Configuration conf) throws IOException {
      this.fsdis = fs.open(path, osBufferSize);
      this.reader =
          new TFile.Reader(fsdis, fs.getFileStatus(path).getLen(), conf);
      this.scanner = reader.createScanner();
      keyBuffer = new byte[32];
      valueBuffer = new byte[32];
    }

    private void checkKeyBuffer(int size) {
      if (size <= keyBuffer.length) {
        return;
      }
      keyBuffer =
          new byte[Math.max(2 * keyBuffer.length, 2 * size - keyBuffer.length)];
    }

    private void checkValueBuffer(int size) {
      if (size <= valueBuffer.length) {
        return;
      }
      valueBuffer =
          new byte[Math.max(2 * valueBuffer.length, 2 * size
              - valueBuffer.length)];
    }

    public byte[] getKey() {
      return keyBuffer;
    }

    public int getKeyLength() {
      return keyLength;
    }

    public byte[] getValue() {
      return valueBuffer;
    }

    public int getValueLength() {
      return valueLength;
    }

    public boolean next() throws IOException {
      if (scanner.atEnd()) return false;
      Entry entry = scanner.entry();
      keyLength = entry.getKeyLength();
      checkKeyBuffer(keyLength);
      entry.getKey(keyBuffer);
      valueLength = entry.getValueLength();
      checkValueBuffer(valueLength);
      entry.getValue(valueBuffer);
      scanner.advance();
      return true;
    }

    public void close() throws IOException {
      scanner.close();
      reader.close();
      fsdis.close();
    }
  }

  static class SeqFileAppendable implements KVAppendable {
    private FSDataOutputStream fsdos;
    private SequenceFile.Writer writer;

    public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize,
        String compress, int minBlkSize) throws IOException {
      Configuration conf = new Configuration();
      conf.setBoolean("hadoop.native.lib", true);

      CompressionCodec codec = null;
      if ("lzo".equals(compress)) {
        codec = Compression.Algorithm.LZO.getCodec();
      }
      else if ("gz".equals(compress)) {
        codec = Compression.Algorithm.GZ.getCodec();
      }
      else if (!"none".equals(compress))
        throw new IOException("Codec not supported.");

      this.fsdos = fs.create(path, true, osBufferSize);

      if (!"none".equals(compress)) {
        writer =
            SequenceFile.createWriter(conf, fsdos, BytesWritable.class,
                BytesWritable.class, SequenceFile.CompressionType.BLOCK, codec);
      }
      else {
        writer =
            SequenceFile.createWriter(conf, fsdos, BytesWritable.class,
                BytesWritable.class, SequenceFile.CompressionType.NONE, null);
      }
    }

    public void append(BytesWritable key, BytesWritable value)
        throws IOException {
      writer.append(key, value);
    }

    public void close() throws IOException {
      writer.close();
      fsdos.close();
    }
  }

  static class SeqFileReadable implements KVReadable {
    private SequenceFile.Reader reader;
    private BytesWritable key;
    private BytesWritable value;

    public SeqFileReadable(FileSystem fs, Path path, int osBufferSize)
        throws IOException {
      Configuration conf = new Configuration();
      conf.setInt("io.file.buffer.size", osBufferSize);
      reader = new SequenceFile.Reader(fs, path, conf);
      key = new BytesWritable();
      value = new BytesWritable();
    }

    public byte[] getKey() {
      return key.get();
    }

    public int getKeyLength() {
      return key.getSize();
    }

    public byte[] getValue() {
      return value.get();
    }

    public int getValueLength() {
      return value.getSize();
    }

    public boolean next() throws IOException {
      return reader.next(key, value);
    }

    public void close() throws IOException {
      reader.close();
    }
  }

  private void reportStats(Path path, long totalBytes) throws IOException {
    long duration = getIntervalMillis();
    long fsize = fs.getFileStatus(path).getLen();
    printlnWithTimestamp(String.format(
        "Duration: %dms...total size: %.2fMB...raw thrpt: %.2fMB/s", duration,
        (double) totalBytes / 1024 / 1024, (double) totalBytes / duration
            * 1000 / 1024 / 1024));
    printlnWithTimestamp(String.format(
        "Compressed size: %.2fMB...compressed thrpt: %.2fMB/s.",
        (double) fsize / 1024 / 1024, (double) fsize / duration * 1000 / 1024
            / 1024));
  }

  private void fillBuffer(Random rng, BytesWritable bw, byte[] tmp, int len) {
    int n = 0;
    while (n < len) {
      byte[] word = dictionary[rng.nextInt(dictionary.length)];
      int l = Math.min(word.length, len - n);
      System.arraycopy(word, 0, tmp, n, l);
      n += l;
    }
    bw.set(tmp, 0, len);
  }

  private void timeWrite(Path path, KVAppendable appendable, int baseKlen,
      int baseVlen, long fileSize) throws IOException {
    int maxKlen = baseKlen * 2;
    int maxVlen = baseVlen * 2;
    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    byte[] keyBuffer = new byte[maxKlen];
    byte[] valueBuffer = new byte[maxVlen];
    Random rng = new Random(options.seed);
    long totalBytes = 0;
    printlnWithTimestamp("Start writing: " + path.getName() + "...");
    startTime();

    for (long i = 0; true; ++i) {
      if (i % 1000 == 0) { // test the size for every 1000 rows.
        if (fs.getFileStatus(path).getLen() >= fileSize) {
          break;
        }
      }
      int klen = rng.nextInt(baseKlen) + baseKlen;
      int vlen = rng.nextInt(baseVlen) + baseVlen;
      fillBuffer(rng, key, keyBuffer, klen);
      fillBuffer(rng, value, valueBuffer, vlen);
      key.set(keyBuffer, 0, klen);
      value.set(valueBuffer, 0, vlen);
      appendable.append(key, value);
      totalBytes += klen;
      totalBytes += vlen;
    }
    stopTime();
    appendable.close();
    reportStats(path, totalBytes);
  }

  private void timeRead(Path path, KVReadable readable) throws IOException {
    printlnWithTimestamp("Start reading: " + path.getName() + "...");
    long totalBytes = 0;
    startTime();
    for (; readable.next();) {
      totalBytes += readable.getKeyLength();
      totalBytes += readable.getValueLength();
    }
    stopTime();
    readable.close();
    reportStats(path, totalBytes);
  }

  private void createTFile(String parameters, String compress)
      throws IOException {
    System.out.println("=== TFile: Creation (" + parameters + ") === ");
    Path path = new Path(options.rootDir, "TFile.Performance");
    KVAppendable appendable =
        new TFileAppendable(fs, path, compress, options.minBlockSize,
            options.osOutputBufferSize, conf);
    timeWrite(path, appendable, options.keyLength, options.valueLength,
        options.fileSize);
  }

  private void readTFile(String parameters, boolean delFile) throws IOException {
    System.out.println("=== TFile: Reading (" + parameters + ") === ");
    {
      Path path = new Path(options.rootDir, "TFile.Performance");
      KVReadable readable =
          new TFileReadable(fs, path, options.osInputBufferSize, conf);
      timeRead(path, readable);
      if (delFile) {
        if (fs.exists(path)) {
          fs.delete(path, true);
        }
      }
    }
  }

  private void createSeqFile(String parameters, String compress)
      throws IOException {
    System.out.println("=== SeqFile: Creation (" + parameters + ") === ");
    Path path = new Path(options.rootDir, "SeqFile.Performance");
    KVAppendable appendable =
        new SeqFileAppendable(fs, path, options.osOutputBufferSize, compress,
            options.minBlockSize);
    timeWrite(path, appendable, options.keyLength, options.valueLength,
        options.fileSize);
  }

  private void readSeqFile(String parameters, boolean delFile)
      throws IOException {
    System.out.println("=== SeqFile: Reading (" + parameters + ") === ");
    Path path = new Path(options.rootDir, "SeqFile.Performance");
    KVReadable readable =
        new SeqFileReadable(fs, path, options.osInputBufferSize);
    timeRead(path, readable);
    if (delFile) {
      if (fs.exists(path)) {
        fs.delete(path, true);
      }
    }
  }

  private void compareRun(String compress) throws IOException {
    String[] supported = TFile.getSupportedCompressionAlgorithms();
    boolean proceed = false;
    for (String c : supported) {
      if (c.equals(compress)) {
        proceed = true;
        break;
      }
    }

    if (!proceed) {
      System.out.println("Skipped for " + compress);
      return;
    }
    
    options.compress = compress;
    String parameters = parameters2String(options);
    createSeqFile(parameters, compress);
    readSeqFile(parameters, true);
    createTFile(parameters, compress);
    readTFile(parameters, true);
    createTFile(parameters, compress);
    readTFile(parameters, true);
    createSeqFile(parameters, compress);
    readSeqFile(parameters, true);
  }

  public void testRunComparisons() throws IOException {
    String[] compresses = new String[] { "none", "lzo", "gz" };
    for (String compress : compresses) {
      if (compress.equals("none")) {
        conf
            .setInt("tfile.fs.input.buffer.size", options.fsInputBufferSizeNone);
        conf.setInt("tfile.fs.output.buffer.size",
            options.fsOutputBufferSizeNone);
      }
      else if (compress.equals("lzo")) {
        conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSizeLzo);
        conf.setInt("tfile.fs.output.buffer.size",
            options.fsOutputBufferSizeLzo);
      }
      else {
        conf.setInt("tfile.fs.input.buffer.size", options.fsInputBufferSizeGz);
        conf
            .setInt("tfile.fs.output.buffer.size", options.fsOutputBufferSizeGz);
      }
      compareRun(compress);
    }
  }

  private static String parameters2String(MyOptions options) {
    return String
        .format(
            "KLEN: %d-%d... VLEN: %d-%d...MinBlkSize: %.2fKB...Target Size: %.2fMB...Compression: ...%s",
            options.keyLength, options.keyLength * 2, options.valueLength,
            options.valueLength * 2, (double) options.minBlockSize / 1024,
            (double) options.fileSize / 1024 / 1024, options.compress);
  }

  private static class MyOptions {
    String rootDir =
        System
            .getProperty("test.build.data", "/tmp/tfile-test");
    String compress = "gz";
    String format = "tfile";
    int dictSize = 1000;
    int minWordLen = 5;
    int maxWordLen = 20;
    int keyLength = 50;
    int valueLength = 100;
    int minBlockSize = 256 * 1024;
    int fsOutputBufferSize = 1;
    int fsInputBufferSize = 0;
    // special variable only for unit testing.
    int fsInputBufferSizeNone = 0;
    int fsInputBufferSizeGz = 0;
    int fsInputBufferSizeLzo = 0;
    int fsOutputBufferSizeNone = 1;
    int fsOutputBufferSizeGz = 1;
    int fsOutputBufferSizeLzo = 1;

    // un-exposed parameters.
    int osInputBufferSize = 64 * 1024;
    int osOutputBufferSize = 64 * 1024;

    long fileSize = 3 * 1024 * 1024;
    long seed;

    static final int OP_CREATE = 1;
    static final int OP_READ = 2;
    int op = OP_READ;

    boolean proceed = false;

    public MyOptions(String[] args) {
      seed = System.nanoTime();

      try {
        Options opts = buildOptions();
        CommandLineParser parser = new GnuParser();
        CommandLine line = parser.parse(opts, args, true);
        processOptions(line, opts);
        validateOptions();
      }
      catch (ParseException e) {
        System.out.println(e.getMessage());
        System.out.println("Try \"--help\" option for details.");
        setStopProceed();
      }
    }

    public boolean proceed() {
      return proceed;
    }

    private Options buildOptions() {
      Option compress =
          OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz]")
              .hasArg().withDescription("compression scheme").create('c');

      Option ditSize =
          OptionBuilder.withLongOpt("dict").withArgName("size").hasArg()
              .withDescription("number of dictionary entries").create('d');

      Option fileSize =
          OptionBuilder.withLongOpt("file-size").withArgName("size-in-MB")
              .hasArg().withDescription("target size of the file (in MB).")
              .create('s');

      Option format =
          OptionBuilder.withLongOpt("format").withArgName("[tfile|seqfile]")
              .hasArg().withDescription("choose TFile or SeqFile").create('f');

      Option fsInputBufferSz =
          OptionBuilder.withLongOpt("fs-input-buffer").withArgName("size")
              .hasArg().withDescription(
                  "size of the file system input buffer (in bytes).").create(
                  'i');

      Option fsOutputBufferSize =
          OptionBuilder.withLongOpt("fs-output-buffer").withArgName("size")
              .hasArg().withDescription(
                  "size of the file system output buffer (in bytes).").create(
                  'o');

      Option keyLen =
          OptionBuilder
              .withLongOpt("key-length")
              .withArgName("length")
              .hasArg()
              .withDescription(
                  "base length of the key (in bytes), actual length varies in [base, 2*base)")
              .create('k');

      Option valueLen =
          OptionBuilder
              .withLongOpt("value-length")
              .withArgName("length")
              .hasArg()
              .withDescription(
                  "base length of the value (in bytes), actual length varies in [base, 2*base)")
              .create('v');

      Option wordLen =
          OptionBuilder.withLongOpt("word-length").withArgName("min,max")
              .hasArg().withDescription(
                  "range of dictionary word length (in bytes)").create('w');

      Option blockSz =
          OptionBuilder.withLongOpt("block").withArgName("size-in-KB").hasArg()
              .withDescription("minimum block size (in KB)").create('b');

      Option seed =
          OptionBuilder.withLongOpt("seed").withArgName("long-int").hasArg()
              .withDescription("specify the seed").create('S');

      Option operation =
          OptionBuilder.withLongOpt("operation").withArgName("r|w|rw").hasArg()
              .withDescription(
                  "action: read-only, create-only, read-after-create").create(
                  'x');

      Option rootDir =
          OptionBuilder.withLongOpt("root-dir").withArgName("path").hasArg()
              .withDescription(
                  "specify root directory where files will be created.")
              .create('r');

      Option help =
          OptionBuilder.withLongOpt("help").hasArg(false).withDescription(
              "show this screen").create("h");

      return new Options().addOption(compress).addOption(ditSize).addOption(
          fileSize).addOption(format).addOption(fsInputBufferSz).addOption(
          fsOutputBufferSize).addOption(keyLen).addOption(wordLen).addOption(
          blockSz).addOption(rootDir).addOption(valueLen).addOption(operation)
          .addOption(help);

    }

    private void processOptions(CommandLine line, Options opts)
        throws ParseException {
      // --help -h and --version -V must be processed first.
      if (line.hasOption('h')) {
        HelpFormatter formatter = new HelpFormatter();
        System.out.println("TFile and SeqFile benchmark.");
        System.out.println();
        formatter.printHelp(100,
            "java ... TestTFileSeqFileComparison [options]",
            "\nSupported options:", opts, "");
        return;
      }

      if (line.hasOption('c')) {
        compress = line.getOptionValue('c');
      }

      if (line.hasOption('d')) {
        dictSize = Integer.parseInt(line.getOptionValue('d'));
      }

      if (line.hasOption('s')) {
        fileSize = Long.parseLong(line.getOptionValue('s')) * 1024 * 1024;
      }

      if (line.hasOption('f')) {
        format = line.getOptionValue('f');
      }

      if (line.hasOption('i')) {
        fsInputBufferSize = Integer.parseInt(line.getOptionValue('i'));
      }

      if (line.hasOption('o')) {
        fsOutputBufferSize = Integer.parseInt(line.getOptionValue('o'));
      }

      if (line.hasOption('k')) {
        keyLength = Integer.parseInt(line.getOptionValue('k'));
      }

      if (line.hasOption('v')) {
        valueLength = Integer.parseInt(line.getOptionValue('v'));
      }

      if (line.hasOption('b')) {
        minBlockSize = Integer.parseInt(line.getOptionValue('b')) * 1024;
      }

      if (line.hasOption('r')) {
        rootDir = line.getOptionValue('r');
      }

      if (line.hasOption('S')) {
        seed = Long.parseLong(line.getOptionValue('S'));
      }

      if (line.hasOption('w')) {
        String min_max = line.getOptionValue('w');
        StringTokenizer st = new StringTokenizer(min_max, " \t,");
        if (st.countTokens() != 2) {
          throw new ParseException("Bad word length specification: " + min_max);
        }
        minWordLen = Integer.parseInt(st.nextToken());
        maxWordLen = Integer.parseInt(st.nextToken());
      }

      if (line.hasOption('x')) {
        String strOp = line.getOptionValue('x');
        if (strOp.equals("r")) {
          op = OP_READ;
        }
        else if (strOp.equals("w")) {
          op = OP_CREATE;
        }
        else if (strOp.equals("rw")) {
          op = OP_CREATE | OP_READ;
        }
        else {
          throw new ParseException("Unknown action specifier: " + strOp);
        }
      }

      proceed = true;
    }

    private void validateOptions() throws ParseException {
      if (!compress.equals("none") && !compress.equals("lzo")
          && !compress.equals("gz")) {
        throw new ParseException("Unknown compression scheme: " + compress);
      }

      if (!format.equals("tfile") && !format.equals("seqfile")) {
        throw new ParseException("Unknown file format: " + format);
      }

      if (minWordLen >= maxWordLen) {
        throw new ParseException(
            "Max word length must be greater than min word length.");
      }
      return;
    }

    private void setStopProceed() {
      proceed = false;
    }

    public boolean doCreate() {
      return (op & OP_CREATE) != 0;
    }

    public boolean doRead() {
      return (op & OP_READ) != 0;
    }
  }

  public static void main(String[] args) throws IOException {
    TestTFileSeqFileComparison testCase = new TestTFileSeqFileComparison();
    MyOptions options = new MyOptions(args);
    if (options.proceed == false) {
      return;
    }
    testCase.options = options;
    String parameters = parameters2String(options);

    testCase.setUp();
    if (testCase.options.format.equals("tfile")) {
      if (options.doCreate()) {
        testCase.createTFile(parameters, options.compress);
      }
      if (options.doRead()) {
        testCase.readTFile(parameters, options.doCreate());
      }
    }
    else {
      if (options.doCreate()) {
        testCase.createSeqFile(parameters, options.compress);
      }
      if (options.doRead()) {
        testCase.readSeqFile(parameters, options.doCreate());
      }
    }
    testCase.tearDown();
  }
}