From 14f9200a6b1ae36897db8c1705e04dca07e3d2f4 Mon Sep 17 00:00:00 2001
From: hannes <hannes@cwi.nl>
Date: Mon, 7 Apr 2014 16:54:16 +0200
Subject: [PATCH] mc
---
ehannes.sh | 186 ++++++++++++++++++++++++++++++++++-------------------
1 file changed, 121 insertions(+), 65 deletions(-)
diff --git a/ehannes.sh b/ehannes.sh
index 0b5e81c..807f0e6 100755
--- a/ehannes.sh
+++ b/ehannes.sh
@@ -6,6 +6,8 @@
PGVER=9.3.4
# MonetDB
MVER=11.17.13
+# MariaDB
+MAVER=10.0.10
# protobuf stuff, does probably not need to change as fast
PBVER=2.5.0
@@ -17,78 +19,114 @@ IDIR=$DIR/.install
PINS=$IDIR/postgresql-$PGVER/
MINS=$IDIR/monetdb-$MVER/
+MAINS=$IDIR/mariadb-$MAVER/
PBCINS=$IDIR/protobuf-c-$PBCVER/
PBINS=$IDIR/protobuf-$PBVER/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PBINS/lib:$PBCINS/lib
-if [ ! -f $MINS/bin/mserver5 ] || [ ! -f $PINS/bin/postgres ] || [ ! -f $PINS/lib/cstore_fdw.so ] ; then
- mkdir -p $SDIR
- mkdir -p $IDIR
+mkdir -p $SDIR
+mkdir -p $IDIR
- # download sources
- PGURL=http://ftp.postgresql.org/pub/source/v$PGVER/postgresql-$PGVER.tar.gz
- MURL=http://www.monetdb.org/downloads/sources/Latest/MonetDB-$MVER.tar.bz2
-
- rm -rf $SDIR/*
-
- wget $PGURL -P $SDIR
- wget $MURL -P $SDIR --no-check-certificate
- git clone https://github.com/citusdata/cstore_fdw/ $SDIR/cstore_fdw
+DROPCACHE="echo 3 | sudo /usr/bin/tee /proc/sys/vm/drop_caches"
- # download, unpack and install protobuf & protobuf-c, needed for cstore_fdw
- wget https://protobuf.googlecode.com/files/protobuf-$PBVER.tar.gz -P $SDIR
- wget https://protobuf-c.googlecode.com/files/protobuf-c-$PBCVER.tar.gz -P $SDIR
- wget http://www.tpc.org/tpch/spec/tpch_2_16_1.zip -P $SDIR
+# clean up source dir first
+rm -rf $SDIR/*
- rm -rf $IDIR/*
+# remove this thing to force a rebuild of the citusdata extension, it might change quickly
+rm $PINS/lib/cstore_fdw.so
- # unpack everything
- for i in $SDIR/*.tar.*; do tar xvf $i -C $SDIR; done
- unzip $SDIR/tpch_*.zip -d $SDIR
-
- # make TPCH dbgen
- cd $SDIR/tpch_2_16_1/dbgen
- sed -e 's/DATABASE\s*=/DATABASE=DB2/' -e 's/MACHINE\s*=/MACHINE=LINUX/' -e 's/WORKLOAD\s*=/WORKLOAD=TPCH/' -e 's/CC\s*=/CC=gcc/' makefile.suite > Makefile
- make
- mkdir $IDIR/dbgen/
- cp dbgen dists.dss $IDIR/dbgen/
+# MonetDB installer
+if [ ! -f $MINS/bin/mserver5 ] ; then
+ rm -rf $MINS
+ MURL=http://www.monetdb.org/downloads/sources/Latest/MonetDB-$MVER.tar.bz2
+ wget $MURL -P $SDIR --no-check-certificate
+ tar xvf $SDIR/MonetDB-*.tar.* -C $SDIR
+ MSRC=$SDIR/MonetDB-$MVER/
+ cd $MSRC
+ ./configure --prefix=$MINS --enable-rubygem=no --enable-python3=no --enable-python2=no --enable-perl=no --enable-geos=no --enable-python=no --enable-geom=no --enable-fits=no --enable-jaql=no --enable-gsl=no --enable-odbc=no --enable-jdbc=no --enable-merocontrol=no
+ make -j install
+ cd $DIR
+ rm -rf $MSRC $SDIR/MonetDB-*.tar.*
+fi
- # okay, now compile everything
+# PostgreSQL installer
+if [ ! -f $PINS/bin/postgres ] ; then
+ rm -rf $PINS
+ PGURL=http://ftp.postgresql.org/pub/source/v$PGVER/postgresql-$PGVER.tar.gz
+ wget $PGURL -P $SDIR
+ tar xvf $SDIR/postgresql-*.tar.* -C $SDIR
PSRC=$SDIR/postgresql-$PGVER/
cd $PSRC
./configure --prefix=$PINS
make
make install
+ cd $DIR
+ rm -rf $PSRC $SDIR/postgresql-*.tar.*
+fi
- MSRC=$SDIR/MonetDB-$MVER/
- cd $MSRC
- ./configure --prefix=$MINS --enable-rubygem=no --enable-python3=no --enable-python2=no --enable-perl=no --enable-geos=no --enable-python=no --enable-geom=no --enable-fits=no --enable-jaql=no --enable-gsl=no --enable-odbc=no --enable-jdbc=no --enable-merocontrol=no
- make -j install
-
- # protobuf and protbuf-c are dependencies of cstore
- PBSRC=$SDIR/protobuf-$PBVER/
- cd $PBSRC
- ./configure --prefix=$PBINS
- make -j install
-
- PBCSRC=$SDIR/protobuf-c-$PBCVER/
- cd $PBCSRC
- ./configure --prefix=$PBCINS CXXFLAGS=-I$IDIR/protobuf-$PBVER/include LDFLAGS=-L$IDIR/protobuf-$PBVER/lib PATH=$PATH:$PBINS/bin/
- make -j install
-
- # cstore is a pgplugin, therefore it has to be built last
+# Citusdata installer
+if [ ! -f $PINS/lib/cstore_fdw.so ] ; then
+ git clone https://github.com/citusdata/cstore_fdw/ $SDIR/cstore_fdw
+ if [ ! -f $PBINS/bin/protoc ] || [ ! -f $PBCINS/bin/protoc-c ] ; then
+ wget https://protobuf.googlecode.com/files/protobuf-$PBVER.tar.gz -P $SDIR
+ wget https://protobuf-c.googlecode.com/files/protobuf-c-$PBCVER.tar.gz -P $SDIR
+ tar xvf $SDIR/protobuf-$PBVER.tar.gz -C $SDIR
+ tar xvf $SDIR/protobuf-c-$PBCVER.tar.gz -C $SDIR
+
+ # protobuf and protbuf-c are dependencies of citusdb-store
+ PBSRC=$SDIR/protobuf-$PBVER/
+ cd $PBSRC
+ ./configure --prefix=$PBINS
+ make -j install
+
+ PBCSRC=$SDIR/protobuf-c-$PBCVER/
+ cd $PBCSRC
+ ./configure --prefix=$PBCINS CXXFLAGS=-I$IDIR/protobuf-$PBVER/include LDFLAGS=-L$IDIR/protobuf-$PBVER/lib PATH=$PATH:$PBINS/bin/
+ make -j install
+ fi
+ # cstore is a pgplugin
CSRC=$SDIR/cstore_fdw
cd $CSRC
+ # some funny include path messing
PATH=$PATH:$PINS/bin/:$PBCINS/bin/ CPATH=$CPATH:$PBCINS/include LIBRARY_PATH=$LIBRARY_PATH:$PBCINS/lib make -j install
+ cd $DIR
+ rm -rf $CSRC $PBCSRC $PBSRC $SDIR/protobuf-*.tar.*
+fi
- # this should be it, check for certain files
+# MariaDB installer
+if [ ! -f $MAINS/bin/mysqld ] ; then
+ rm -rf $MAINS
+ MAURL=http://mariadb.mirror.triple-it.nl//mariadb-$MAVER/kvm-tarbake-jaunty-x86/mariadb-$MAVER.tar.gz
+ wget $MAURL -P $SDIR
+ tar xvf $SDIR/mariadb-*.tar.* -C $SDIR
+ MASRC=$SDIR/mariadb-$MAVER/
+ cd $MASRC
+ cmake -DCMAKE_INSTALL_PREFIX:PATH=$MAINS .
+ make
+ make install
+ cd $DIR
+ rm -rf $MASRC $SDIR/mariadb-*.tar.*
+fi
- rm -rf $SDIR/*
+# TPC-H dbgen installer
+if [ ! -f $IDIR/dbgen/dbgen ] ; then
+ rm -rf $IDIR/dbgen/
+ wget http://www.tpc.org/tpch/spec/tpch_2_16_1.zip -P $SDIR
+ unzip $SDIR/tpch_*.zip -d $SDIR
+ cd $SDIR/tpch_2_16_1/dbgen
+ sed -e 's/DATABASE\s*=/DATABASE=DB2/' -e 's/MACHINE\s*=/MACHINE=LINUX/' -e 's/WORKLOAD\s*=/WORKLOAD=TPCH/' -e 's/CC\s*=/CC=gcc/' makefile.suite > Makefile
+ make
+ mkdir $IDIR/dbgen/
+ cp dbgen dists.dss $IDIR/dbgen/
+ rm -rf $SDIR/tpch_*
fi
+# in case something was left
+rm -rf $SDIR/*
+
-# some sys setup for PostgreSQL according to Dr. Kyzirakos
+# some sys setup for PostgreSQL according to Dr. Kyzirakos (TM)
### RAM
## 4 GB of RAM
@@ -165,10 +203,12 @@ PORT=51337
mkdir -p $DDIR
mkdir -p $FARM
mkdir -p $QRDIR
+BMARK="tpch"
-TIMINGCMD="/usr/bin/time -o $DIR/.time -f %e"
+TIMINGCMD="/usr/bin/time -o $DIR/.time -f %e "
+TIMEOUTCMD="timeout -k 35m 30m "
-for SF in 1 # 5 10 # 1 30
+for SF in 1 5 10
do
# check if we have data
SFDDIR=$DDIR/sf-$SF/
@@ -182,7 +222,7 @@ do
mv *.tbl $SFDDIR
fi
cd $DIR
- for DB in citusdata postgres monetdb
+ for DB in postgres # citusdata postgres monetdb mariadb
do
DBNAME=$DB-sf$SF
DBFARM=$FARM/$DBNAME/
@@ -193,6 +233,7 @@ do
CLIENTCMD="$MINS/bin/mclient -p $PORT "
INITFCMD="echo "
CREATEDBCMD="echo createdb"
+ DBVER=$MVER
fi
if [ "$DB" == "postgres" ] || [ "$DB" == "citusdata" ]; then
SERVERCMD="$PINS/bin/postgres -p $PORT \
@@ -212,11 +253,24 @@ do
CLIENTCMD="$PINS/bin/psql -p $PORT tpch -t -A -F , -f "
INITFCMD="$PINS/bin/initdb -D "
CREATEDBCMD="$PINS/bin/createdb -p $PORT tpch"
+ DBVER=$PGVER
+ fi
+ if [ "$DB" == "citusdata" ]; then
+ DBVER=snapshot-`date +"%Y-%m-%d"`
fi
+
+ if [ "$DB" == "mariadb" ] ; then
+ SERVERCMD="$MAINS/bin/mysqld --basedir=$MAINS -P $PORT --datadir="
+ CLIENTCMD="$MAINS/bin/mysql -P $PORT -N tpcd -B <"
+ INITFCMD=""
+ CREATEDBCMD=""
+ DBVER=$MAVER
+ fi
+
if [ ! -d $DBFARM ] ; then
# clear caches (fair loading)
- #sudo bash -c "echo 3 > /proc/sys/vm/drop_caches"
+ echo 3 | sudo /usr/bin/tee /proc/sys/vm/drop_caches
mkdir -p $DBFARM
# initialize db directory
@@ -237,17 +291,17 @@ do
sed -e "s|DIR|$SFDDIR|" $SCDIR/$DB.load.sql > $DIR/.$DB.load.sql.local
$TIMINGCMD $CLIENTCMD $DIR/.$DB.load.sql.local > /dev/null
LDTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\tload\t\t$LDTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\tload\t\t\t$LDTIME" >> $RESFL
# constraints
$TIMINGCMD $CLIENTCMD $SCDIR/$DB.constraints.sql > /dev/null
CTTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\tconstraints\t\t$CTTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\tconstraints\t\t\t$CTTIME" >> $RESFL
# analyze/vacuum
$TIMINGCMD $CLIENTCMD $SCDIR/$DB.analyze.sql > /dev/null
AZTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\tanalyze\t\t$AZTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\tanalyze\t\t\t$AZTIME" >> $RESFL
# aand restart
kill `jobs -p`
@@ -255,20 +309,22 @@ do
fi
# we start with cold runs
# clear caches (fair loading)
- for coldrun in {1..2}
+ for REP in {1..5}
do
for i in $QYDIR/q??.sql
do
- #sudo bash -c "echo 3 > /proc/sys/vm/drop_caches"
+ echo 3 | sudo /usr/bin/tee /proc/sys/vm/drop_caches
$SERVERCMD$DBFARM > /dev/null &
sleep 5
q=${i%.sql}
qn=`basename $q`
- $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-coldrun$coldrun-q$qn.out
+ $TIMEOUTCMD $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-coldrun$coldrun-q$qn.out
QTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\tcoldruns\t$qn\t$QTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\tcoldruns\t$qn\t$REP\t$QTIME" >> $RESFL
kill `jobs -p`
sleep 10
+ kill -9 `jobs -p`
+ sleep 10
done
done
@@ -276,28 +332,28 @@ do
# warmup...
$SERVERCMD$DBFARM > /dev/null &
sleep 5
- for warmup in {1..3}
+ for REP in {1..2}
do
for i in $QYDIR/q??.sql
do
q=${i%.sql}
qn=`basename $q`
- $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-warmup$warmup-q$qn.out
+ $TIMEOUTCMD $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-warmup$warmup-q$qn.out
QTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\twarmup\t$qn\t$QTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\twarmup\t$qn\t$REP\t$QTIME" >> $RESFL
done
done
# hot runs!
- for hotrun in {1..5}
+ for REP in {1..5}
do
for i in $QYDIR/q??.sql
do
q=${i%.sql}
qn=`basename $q`
- $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-hotrun$hotrun-q$qn.out
+ $TIMEOUTCMD $TIMINGCMD $CLIENTCMD $i > $QRDIR/$DB-SF$SF-hotrun$hotrun-q$qn.out
QTIME=`cat $DIR/.time`
- echo -e "$DB\t$SF\thotruns\t$qn\t$QTIME" >> $RESFL
+ echo -e "$DB\t$DBVER\t$BMARK\t$SF\thotruns\t$qn\t$REP\t$QTIME" >> $RESFL
done
done
kill `jobs -p`
--
GitLab