[120] | 1 | #!/usr/bin/env bash |
---|
| 2 | |
---|
| 3 | |
---|
| 4 | ## Environment configuration |
---|
| 5 | # Hadoop installation |
---|
| 6 | # set var only if it has not already been set externally |
---|
| 7 | if [ -z "${HADOOP_HOME}" ] ; then |
---|
| 8 | export HADOOP_HOME= |
---|
| 9 | fi |
---|
| 10 | # Base directory for gridmix install |
---|
| 11 | # set var only if it has not already been set externally |
---|
| 12 | if [ -z "${GRID_MIX_HOME}" ] ; then |
---|
| 13 | export GRID_MIX_HOME=${GRID_DIR} |
---|
| 14 | fi |
---|
| 15 | # Hadoop example jar |
---|
| 16 | # set var only if it has not already been set externally |
---|
| 17 | if [ -z "${EXAMPLE_JAR}" ] ; then |
---|
| 18 | export EXAMPLE_JAR="${HADOOP_HOME}/hadoop-*examples.jar" |
---|
| 19 | fi |
---|
| 20 | # Hadoop test jar |
---|
| 21 | # set var only if it has not already been set externally |
---|
| 22 | if [ -z "${APP_JAR}" ] ; then |
---|
| 23 | export APP_JAR="${HADOOP_HOME}/hadoop-*test.jar" |
---|
| 24 | fi |
---|
| 25 | # Hadoop streaming jar |
---|
| 26 | # set var only if it has not already been set externally |
---|
| 27 | if [ -z "${STREAM_JAR}" ] ; then |
---|
| 28 | export STREAM_JAR="${HADOOP_HOME}/contrib/streaming/hadoop-*streaming.jar" |
---|
| 29 | fi |
---|
| 30 | # Location on default filesystem for writing gridmix data (usually HDFS) |
---|
| 31 | # Default: /gridmix/data |
---|
| 32 | # set var only if it has not already been set externally |
---|
| 33 | if [ -z "${GRID_MIX_DATA}" ] ; then |
---|
| 34 | export GRID_MIX_DATA=/gridmix/data |
---|
| 35 | fi |
---|
| 36 | # Location of executables in default filesystem (usually HDFS) |
---|
| 37 | # Default: /gridmix/programs |
---|
| 38 | # set var only if it has not already been set externally |
---|
| 39 | if [ -z "${GRID_MIX_PROG}" ] ; then |
---|
| 40 | export GRID_MIX_PROG=/gridmix/programs |
---|
| 41 | fi |
---|
| 42 | |
---|
| 43 | ## Data sources |
---|
| 44 | # Variable length key, value compressed SequenceFile |
---|
| 45 | export VARCOMPSEQ=${GRID_MIX_DATA}/WebSimulationBlockCompressed |
---|
| 46 | # Fixed length key, value compressed SequenceFile |
---|
| 47 | export FIXCOMPSEQ=${GRID_MIX_DATA}/MonsterQueryBlockCompressed |
---|
| 48 | # Variable length key, value uncompressed Text File |
---|
| 49 | export VARINFLTEXT=${GRID_MIX_DATA}/SortUncompressed |
---|
| 50 | # Fixed length key, value compressed Text File |
---|
| 51 | export FIXCOMPTEXT=${GRID_MIX_DATA}/EntropySimulationCompressed |
---|
| 52 | |
---|
| 53 | ## Job sizing |
---|
| 54 | export NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS=5 |
---|
| 55 | export NUM_OF_LARGE_JOBS_PER_CLASS=3 |
---|
| 56 | export NUM_OF_MEDIUM_JOBS_PER_CLASS=20 |
---|
| 57 | export NUM_OF_SMALL_JOBS_PER_CLASS=40 |
---|
| 58 | |
---|
| 59 | export NUM_OF_REDUCERS_FOR_LARGE_JOB=370 |
---|
| 60 | export NUM_OF_REDUCERS_FOR_MEDIUM_JOB=170 |
---|
| 61 | export NUM_OF_REDUCERS_FOR_SMALL_JOB=15 |
---|
| 62 | |
---|
| 63 | ## Throttling |
---|
| 64 | export INTERVAL_BETWEEN_SUBMITION=20 |
---|
| 65 | |
---|
| 66 | ## Hod |
---|
| 67 | #export HOD_OPTIONS="" |
---|
| 68 | |
---|
| 69 | export CLUSTER_DIR_BASE=$GRID_MIX_HOME/CLUSTER_DIR_BASE |
---|
| 70 | export HOD_CONFIG= |
---|
| 71 | export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}" |
---|
| 72 | export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 5" |
---|
| 73 | export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 50" |
---|
| 74 | export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 100" |
---|
| 75 | |
---|