1 | #!/usr/bin/env bash |
---|
2 | |
---|
3 | |
---|
4 | ## Environment configuration |
---|
5 | # Hadoop installation |
---|
6 | # set var only if it has not already been set externally |
---|
7 | if [ -z "${HADOOP_HOME}" ] ; then |
---|
8 | export HADOOP_HOME= |
---|
9 | fi |
---|
10 | # Base directory for gridmix install |
---|
11 | # set var only if it has not already been set externally |
---|
12 | if [ -z "${GRID_MIX_HOME}" ] ; then |
---|
13 | export GRID_MIX_HOME=${GRID_DIR} |
---|
14 | fi |
---|
15 | # Hadoop example jar |
---|
16 | # set var only if it has not already been set externally |
---|
17 | if [ -z "${EXAMPLE_JAR}" ] ; then |
---|
18 | export EXAMPLE_JAR="${HADOOP_HOME}/hadoop-*examples.jar" |
---|
19 | fi |
---|
20 | # Hadoop test jar |
---|
21 | # set var only if it has not already been set externally |
---|
22 | if [ -z "${APP_JAR}" ] ; then |
---|
23 | export APP_JAR="${HADOOP_HOME}/hadoop-*test.jar" |
---|
24 | fi |
---|
25 | # Hadoop streaming jar |
---|
26 | # set var only if it has not already been set externally |
---|
27 | if [ -z "${STREAM_JAR}" ] ; then |
---|
28 | export STREAM_JAR="${HADOOP_HOME}/contrib/streaming/hadoop-*streaming.jar" |
---|
29 | fi |
---|
30 | # Location on default filesystem for writing gridmix data (usually HDFS) |
---|
31 | # Default: /gridmix/data |
---|
32 | # set var only if it has not already been set externally |
---|
33 | if [ -z "${GRID_MIX_DATA}" ] ; then |
---|
34 | export GRID_MIX_DATA=/gridmix/data |
---|
35 | fi |
---|
36 | # Location of executables in default filesystem (usually HDFS) |
---|
37 | # Default: /gridmix/programs |
---|
38 | # set var only if it has not already been set externally |
---|
39 | if [ -z "${GRID_MIX_PROG}" ] ; then |
---|
40 | export GRID_MIX_PROG=/gridmix/programs |
---|
41 | fi |
---|
42 | |
---|
43 | ## Data sources |
---|
44 | # Variable length key, value compressed SequenceFile |
---|
45 | export VARCOMPSEQ=${GRID_MIX_DATA}/WebSimulationBlockCompressed |
---|
46 | # Fixed length key, value compressed SequenceFile |
---|
47 | export FIXCOMPSEQ=${GRID_MIX_DATA}/MonsterQueryBlockCompressed |
---|
48 | # Variable length key, value uncompressed Text File |
---|
49 | export VARINFLTEXT=${GRID_MIX_DATA}/SortUncompressed |
---|
50 | # Fixed length key, value compressed Text File |
---|
51 | export FIXCOMPTEXT=${GRID_MIX_DATA}/EntropySimulationCompressed |
---|
52 | |
---|
53 | ## Job sizing |
---|
54 | export NUM_OF_LARGE_JOBS_FOR_ENTROPY_CLASS=5 |
---|
55 | export NUM_OF_LARGE_JOBS_PER_CLASS=3 |
---|
56 | export NUM_OF_MEDIUM_JOBS_PER_CLASS=20 |
---|
57 | export NUM_OF_SMALL_JOBS_PER_CLASS=40 |
---|
58 | |
---|
59 | export NUM_OF_REDUCERS_FOR_LARGE_JOB=370 |
---|
60 | export NUM_OF_REDUCERS_FOR_MEDIUM_JOB=170 |
---|
61 | export NUM_OF_REDUCERS_FOR_SMALL_JOB=15 |
---|
62 | |
---|
63 | ## Throttling |
---|
64 | export INTERVAL_BETWEEN_SUBMITION=20 |
---|
65 | |
---|
66 | ## Hod |
---|
67 | #export HOD_OPTIONS="" |
---|
68 | |
---|
69 | export CLUSTER_DIR_BASE=$GRID_MIX_HOME/CLUSTER_DIR_BASE |
---|
70 | export HOD_CONFIG= |
---|
71 | export ALL_HOD_OPTIONS="-c ${HOD_CONFIG} ${HOD_OPTIONS}" |
---|
72 | export SMALL_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 5" |
---|
73 | export MEDIUM_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 50" |
---|
74 | export LARGE_JOB_HOD_OPTIONS="$ALL_HOD_OPTIONS -n 100" |
---|
75 | |
---|